feat(ui): revise filter implementation

This commit is contained in:
psychedelicious 2024-08-21 19:01:48 +10:00
parent 92906a9575
commit 9465ff450b
10 changed files with 134 additions and 126 deletions

View File

@ -6,20 +6,23 @@ import { InpaintMaskList } from 'features/controlLayers/components/InpaintMask/I
import { IPAdapterList } from 'features/controlLayers/components/IPAdapter/IPAdapterList';
import { RasterLayerEntityList } from 'features/controlLayers/components/RasterLayer/RasterLayerEntityList';
import { RegionalGuidanceEntityList } from 'features/controlLayers/components/RegionalGuidance/RegionalGuidanceEntityList';
import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
import { memo } from 'react';
export const CanvasEntityList = memo(() => {
return (
<ScrollableContent>
<Flex flexDir="column" gap={4} pt={2} data-testid="control-layers-layer-list">
<CanvasEntityOpacity />
<InpaintMaskList />
<RegionalGuidanceEntityList />
<IPAdapterList />
<ControlLayerEntityList />
<RasterLayerEntityList />
</Flex>
</ScrollableContent>
<CanvasManagerProviderGate>
<ScrollableContent>
<Flex flexDir="column" gap={4} pt={2} data-testid="control-layers-layer-list">
<CanvasEntityOpacity />
<InpaintMaskList />
<RegionalGuidanceEntityList />
<IPAdapterList />
<ControlLayerEntityList />
<RasterLayerEntityList />
</Flex>
</ScrollableContent>
</CanvasManagerProviderGate>
);
});

View File

@ -1,8 +1,8 @@
import { Combobox, FormControl, Tooltip } from '@invoke-ai/ui-library';
import { useAppSelector } from 'app/store/storeHooks';
import { useGroupedModelCombobox } from 'common/hooks/useGroupedModelCombobox';
import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
import { $filterConfig, $filteringEntity } from 'features/controlLayers/store/canvasV2Slice';
import { IMAGE_FILTERS, isFilterType } from 'features/controlLayers/store/types';
import { memo, useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
@ -17,6 +17,7 @@ type Props = {
export const ControlLayerControlAdapterModel = memo(({ modelKey, onChange: onChangeModel }: Props) => {
const { t } = useTranslation();
const entityIdentifier = useEntityIdentifierContext();
const canvasManager = useCanvasManager();
const currentBaseModel = useAppSelector((s) => s.canvasV2.params.model?.base);
const [modelConfigs, { isLoading }] = useControlNetAndT2IAdapterModels();
const selectedModel = useMemo(() => modelConfigs.find((m) => m.key === modelKey), [modelConfigs, modelKey]);
@ -35,19 +36,21 @@ export const ControlLayerControlAdapterModel = memo(({ modelKey, onChange: onCha
return;
}
// Update the filter, preferring the model's default
if (isFilterType(modelConfig.default_settings?.preprocessor)) {
$filterConfig.set(IMAGE_FILTERS[modelConfig.default_settings.preprocessor].buildDefaults(modelConfig.base));
} else {
$filterConfig.set(IMAGE_FILTERS.canny_image_processor.buildDefaults(modelConfig.base));
}
// Open the filter popup by setting this entity as the filtering entity
if (!$filteringEntity.get()) {
$filteringEntity.set(entityIdentifier);
if (!canvasManager.filter.$adapter.get()) {
// Update the filter, preferring the model's default
if (isFilterType(modelConfig.default_settings?.preprocessor)) {
canvasManager.filter.$config.set(
IMAGE_FILTERS[modelConfig.default_settings.preprocessor].buildDefaults(modelConfig.base)
);
} else {
canvasManager.filter.$config.set(IMAGE_FILTERS.canny_image_processor.buildDefaults(modelConfig.base));
}
canvasManager.filter.initialize(entityIdentifier);
canvasManager.filter.previewFilter();
}
},
[entityIdentifier, modelKey, onChangeModel]
[canvasManager.filter, entityIdentifier, modelKey, onChangeModel]
);
const getIsDisabled = useCallback(

View File

@ -6,6 +6,7 @@ import { ControlLayersToolbar } from 'features/controlLayers/components/ControlL
import { Filter } from 'features/controlLayers/components/Filters/Filter';
import { StageComponent } from 'features/controlLayers/components/StageComponent';
import { StagingAreaToolbar } from 'features/controlLayers/components/StagingArea/StagingAreaToolbar';
import { CanvasManagerProviderGate } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
import { memo, useRef } from 'react';
export const CanvasEditor = memo(() => {
@ -33,7 +34,9 @@ export const CanvasEditor = memo(() => {
<StagingAreaToolbar />
</Flex>
<Flex position="absolute" bottom={16}>
<Filter />
<CanvasManagerProviderGate>
<Filter />
</CanvasManagerProviderGate>
</Flex>
<CanvasDropArea />
</Flex>

View File

@ -2,8 +2,7 @@ import { Button, ButtonGroup, Flex, Heading } from '@invoke-ai/ui-library';
import { useStore } from '@nanostores/react';
import { FilterSettings } from 'features/controlLayers/components/Filters/FilterSettings';
import { FilterTypeSelect } from 'features/controlLayers/components/Filters/FilterTypeSelect';
import { $canvasManager } from 'features/controlLayers/konva/CanvasManager';
import { $filterConfig, $filteringEntity, $isProcessingFilter } from 'features/controlLayers/store/canvasV2Slice';
import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
import { type FilterConfig, IMAGE_FILTERS } from 'features/controlLayers/store/types';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
@ -11,64 +10,38 @@ import { PiCheckBold, PiShootingStarBold, PiXBold } from 'react-icons/pi';
export const Filter = memo(() => {
const { t } = useTranslation();
const filteringEntity = useStore($filteringEntity);
const filterConfig = useStore($filterConfig);
const isProcessingFilter = useStore($isProcessingFilter);
const canvasManager = useCanvasManager();
const adapter = useStore(canvasManager.filter.$adapter);
const config = useStore(canvasManager.filter.$config);
const isProcessing = useStore(canvasManager.filter.$isProcessing);
const previewFilter = useCallback(() => {
if (!filteringEntity) {
return;
}
const canvasManager = $canvasManager.get();
if (!canvasManager) {
return;
}
const entity = canvasManager.stateApi.getEntity(filteringEntity);
if (!entity || (entity.type !== 'raster_layer' && entity.type !== 'control_layer')) {
return;
}
entity.adapter.filter.previewFilter();
}, [filteringEntity]);
canvasManager.filter.previewFilter();
}, [canvasManager.filter]);
const applyFilter = useCallback(() => {
if (!filteringEntity) {
return;
}
const canvasManager = $canvasManager.get();
if (!canvasManager) {
return;
}
const entity = canvasManager.stateApi.getEntity(filteringEntity);
if (!entity || (entity.type !== 'raster_layer' && entity.type !== 'control_layer')) {
return;
}
entity.adapter.filter.applyFilter();
}, [filteringEntity]);
canvasManager.filter.applyFilter();
}, [canvasManager.filter]);
const cancelFilter = useCallback(() => {
if (!filteringEntity) {
return;
}
const canvasManager = $canvasManager.get();
if (!canvasManager) {
return;
}
const entity = canvasManager.stateApi.getEntity(filteringEntity);
if (!entity || (entity.type !== 'raster_layer' && entity.type !== 'control_layer')) {
return;
}
entity.adapter.filter.cancelFilter();
}, [filteringEntity]);
canvasManager.filter.cancelFilter();
}, [canvasManager.filter]);
const onChangeFilterConfig = useCallback((filterConfig: FilterConfig) => {
$filterConfig.set(filterConfig);
}, []);
const onChangeFilterConfig = useCallback(
(filterConfig: FilterConfig) => {
canvasManager.filter.$config.set(filterConfig);
},
[canvasManager.filter.$config]
);
const onChangeFilterType = useCallback((filterType: FilterConfig['type']) => {
$filterConfig.set(IMAGE_FILTERS[filterType].buildDefaults());
}, []);
const onChangeFilterType = useCallback(
(filterType: FilterConfig['type']) => {
canvasManager.filter.$config.set(IMAGE_FILTERS[filterType].buildDefaults());
},
[canvasManager.filter.$config]
);
if (!filteringEntity || !filterConfig) {
if (!adapter) {
return null;
}
@ -88,13 +61,13 @@ export const Filter = memo(() => {
<Heading size="md" color="base.300" userSelect="none">
{t('controlLayers.filter.filter')}
</Heading>
<FilterTypeSelect filterType={filterConfig.type} onChange={onChangeFilterType} />
<FilterSettings filterConfig={filterConfig} onChange={onChangeFilterConfig} />
<FilterTypeSelect filterType={config.type} onChange={onChangeFilterType} />
<FilterSettings filterConfig={config} onChange={onChangeFilterConfig} />
<ButtonGroup isAttached={false} size="sm" alignSelf="self-end">
<Button
leftIcon={<PiShootingStarBold />}
onClick={previewFilter}
isLoading={isProcessingFilter}
isLoading={isProcessing}
loadingText={t('controlLayers.filter.preview')}
>
{t('controlLayers.filter.preview')}
@ -102,7 +75,7 @@ export const Filter = memo(() => {
<Button
leftIcon={<PiCheckBold />}
onClick={applyFilter}
isLoading={isProcessingFilter}
isLoading={isProcessing}
loadingText={t('controlLayers.filter.apply')}
>
{t('controlLayers.filter.apply')}
@ -110,7 +83,7 @@ export const Filter = memo(() => {
<Button
leftIcon={<PiXBold />}
onClick={cancelFilter}
isLoading={isProcessingFilter}
isLoading={isProcessing}
loadingText={t('controlLayers.filter.cancel')}
>
{t('controlLayers.filter.cancel')}

View File

@ -1,19 +1,21 @@
import { MenuItem } from '@invoke-ai/ui-library';
import { useCanvasManager } from 'features/controlLayers/contexts/CanvasManagerProviderGate';
import { useEntityIdentifierContext } from 'features/controlLayers/contexts/EntityIdentifierContext';
import { $filteringEntity } from 'features/controlLayers/store/canvasV2Slice';
import { memo, useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { PiShootingStarBold } from 'react-icons/pi';
export const CanvasEntityMenuItemsFilter = memo(() => {
const { t } = useTranslation();
const canvasManager = useCanvasManager();
const entityIdentifier = useEntityIdentifierContext();
const filter = useCallback(() => {
$filteringEntity.set(entityIdentifier);
}, [entityIdentifier]);
const onClick = useCallback(() => {
canvasManager.filter.initialize(entityIdentifier);
}, [entityIdentifier, canvasManager.filter]);
return (
<MenuItem onClick={filter} icon={<PiShootingStarBold />}>
<MenuItem onClick={onClick} icon={<PiShootingStarBold />}>
{t('controlLayers.filter.filter')}
</MenuItem>
);

View File

@ -2,8 +2,9 @@ import type { JSONObject, SerializableObject } from 'common/types';
import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import { getPrefixedId } from 'features/controlLayers/konva/util';
import type { CanvasImageState } from 'features/controlLayers/store/types';
import type { CanvasEntityIdentifier, CanvasImageState, FilterConfig } from 'features/controlLayers/store/types';
import { IMAGE_FILTERS, imageDTOToImageObject } from 'features/controlLayers/store/types';
import { atom } from 'nanostores';
import type { Logger } from 'roarr';
import { getImageDTO } from 'services/api/endpoints/images';
import { queueApi } from 'services/api/endpoints/queue';
@ -18,27 +19,48 @@ export class CanvasFilter {
id: string;
path: string[];
parent: CanvasLayerAdapter;
manager: CanvasManager;
log: Logger;
imageState: CanvasImageState | null = null;
constructor(parent: CanvasLayerAdapter) {
$adapter = atom<CanvasLayerAdapter | null>(null);
$isProcessing = atom<boolean>(false);
$config = atom<FilterConfig>(IMAGE_FILTERS.canny_image_processor.buildDefaults());
constructor(manager: CanvasManager) {
this.id = getPrefixedId(this.type);
this.parent = parent;
this.manager = parent.manager;
this.path = this.parent.path.concat(this.id);
this.manager = manager;
this.path = this.manager.path.concat(this.id);
this.log = this.manager.buildLogger(this.getLoggingContext);
this.log.trace('Creating filter');
}
initialize = (entityIdentifier: CanvasEntityIdentifier) => {
this.log.trace('Initializing filter');
const entity = this.manager.stateApi.getEntity(entityIdentifier);
if (!entity) {
this.log.warn({ entityIdentifier }, 'Unable to find entity');
return;
}
if (entity.type !== 'raster_layer' && entity.type !== 'control_layer') {
this.log.warn({ entityIdentifier }, 'Unsupported entity type');
return;
}
this.$adapter.set(entity.adapter);
};
previewFilter = async () => {
const config = this.manager.stateApi.$filterConfig.get();
const adapter = this.$adapter.get();
if (!adapter) {
this.log.warn('Cannot preview filter without an adapter');
return;
}
const config = this.$config.get();
this.log.trace({ config }, 'Previewing filter');
const dispatch = this.manager.stateApi._store.dispatch;
const rect = this.parent.transformer.getRelativeRect();
const imageDTO = await this.parent.renderer.rasterize(rect, false);
const rect = adapter.transformer.getRelativeRect();
const imageDTO = await adapter.renderer.rasterize(rect, false);
// TODO(psyche): I can't get TS to be happy, it thinkgs `config` is `never` but it should be inferred from the generic... I'll just cast it for now
const filterNode = IMAGE_FILTERS[config.type].buildNode(imageDTO, config as never);
const enqueueBatchArg: BatchConfig = {
@ -76,19 +98,19 @@ export class CanvasFilter {
assert(imageDTO, "Failed to fetch processor output's image DTO");
this.imageState = imageDTOToImageObject(imageDTO);
this.parent.renderer.clearBuffer();
adapter.renderer.clearBuffer();
await this.parent.renderer.setBuffer(this.imageState);
await adapter.renderer.setBuffer(this.imageState);
this.parent.renderer.hideObjects();
this.manager.stateApi.$isProcessingFilter.set(false);
adapter.renderer.hideObjects();
this.$isProcessing.set(false);
};
this.manager.socket.on('invocation_complete', listener);
this.log.trace({ enqueueBatchArg } as SerializableObject, 'Enqueuing filter batch');
this.manager.stateApi.$isProcessingFilter.set(true);
this.$isProcessing.set(true);
dispatch(
queueApi.endpoints.enqueueBatch.initiate(enqueueBatchArg, {
fixedCacheKey: 'enqueueBatch',
@ -97,36 +119,47 @@ export class CanvasFilter {
};
applyFilter = () => {
this.log.trace('Applying filter');
if (!this.imageState) {
const imageState = this.imageState;
const adapter = this.$adapter.get();
if (!imageState) {
this.log.warn('No image state to apply filter to');
return;
}
this.parent.renderer.commitBuffer();
const rect = this.parent.transformer.getRelativeRect();
if (!adapter) {
this.log.warn('Cannot apply filter without an adapter');
return;
}
this.log.trace('Applying filter');
adapter.renderer.commitBuffer();
const rect = adapter.transformer.getRelativeRect();
this.manager.stateApi.rasterizeEntity({
entityIdentifier: this.parent.getEntityIdentifier(),
imageObject: this.imageState,
entityIdentifier: adapter.getEntityIdentifier(),
imageObject: imageState,
rect: {
x: Math.round(rect.x),
y: Math.round(rect.y),
width: this.imageState.image.height,
height: this.imageState.image.width,
width: imageState.image.height,
height: imageState.image.width,
},
replaceObjects: true,
});
this.parent.renderer.showObjects();
this.manager.stateApi.$filteringEntity.set(null);
adapter.renderer.showObjects();
this.imageState = null;
this.$adapter.set(null);
};
cancelFilter = () => {
this.log.trace('Cancelling filter');
this.parent.renderer.clearBuffer();
this.parent.renderer.showObjects();
this.manager.stateApi.$filteringEntity.set(null);
const adapter = this.$adapter.get();
if (adapter) {
adapter.renderer.clearBuffer();
adapter.renderer.showObjects();
this.$adapter.set(null);
}
this.imageState = null;
this.manager.stateApi.$isProcessingFilter.set(false);
this.$isProcessing.set(false);
};
destroy = () => {
@ -141,6 +174,6 @@ export class CanvasFilter {
};
getLoggingContext = (): JSONObject => {
return { ...this.parent.getLoggingContext(), path: this.path.join('.') };
return { ...this.manager.getLoggingContext(), path: this.path.join('.') };
};
}

View File

@ -1,6 +1,5 @@
import type { JSONObject } from 'common/types';
import { deepClone } from 'common/util/deepClone';
import { CanvasFilter } from 'features/controlLayers/konva/CanvasFilter';
import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer';
import { CanvasTransformer } from 'features/controlLayers/konva/CanvasTransformer';
@ -31,7 +30,6 @@ export class CanvasLayerAdapter {
};
transformer: CanvasTransformer;
renderer: CanvasObjectRenderer;
filter: CanvasFilter;
isFirstRender: boolean = true;
@ -56,7 +54,6 @@ export class CanvasLayerAdapter {
this.renderer = new CanvasObjectRenderer(this);
this.transformer = new CanvasTransformer(this);
this.filter = new CanvasFilter(this);
}
/**

View File

@ -2,6 +2,7 @@ import type { AppSocket } from 'app/hooks/useSocketIO';
import { logger } from 'app/logging/logger';
import type { AppStore } from 'app/store/store';
import type { JSONObject } from 'common/types';
import { CanvasFilter } from 'features/controlLayers/konva/CanvasFilter';
import { MAX_CANVAS_SCALE, MIN_CANVAS_SCALE } from 'features/controlLayers/konva/constants';
import {
canvasToBlob,
@ -53,6 +54,7 @@ export class CanvasManager {
stateApi: CanvasStateApi;
preview: CanvasPreview;
background: CanvasBackground;
filter: CanvasFilter;
log: Logger;
socket: AppSocket;
@ -92,6 +94,8 @@ export class CanvasManager {
this.background = new CanvasBackground(this);
this.stage.add(this.background.konva.layer);
this.filter = new CanvasFilter(this);
this._worker.onmessage = (event: MessageEvent<ExtentsResult | WorkerLogMessage>) => {
const { type, data } = event.data;
if (type === 'log') {

View File

@ -4,11 +4,8 @@ import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLaye
import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
import {
$filterConfig,
$filteringEntity,
$isDrawing,
$isMouseDown,
$isProcessingFilter,
$lastAddedPoint,
$lastCursorPos,
$lastMouseDownPos,
@ -233,9 +230,6 @@ export class CanvasStateApi {
};
$transformingEntity = $transformingEntity;
$filteringEntity = $filteringEntity;
$filterConfig = $filterConfig;
$isProcessingFilter = $isProcessingFilter;
$toolState: WritableAtom<CanvasV2State['tool']> = atom();
$currentFill: WritableAtom<RgbaColor> = atom();

View File

@ -38,10 +38,9 @@ import type {
EntityMovedPayload,
EntityRasterizedPayload,
EntityRectAddedPayload,
FilterConfig,
StageAttrs,
} from './types';
import { getEntityIdentifier, IMAGE_FILTERS, isDrawableEntity } from './types';
import { getEntityIdentifier, isDrawableEntity } from './types';
const initialState: CanvasV2State = {
_version: 3,
@ -702,9 +701,6 @@ export const $lastMouseDownPos = atom<Coordinate | null>(null);
export const $lastCursorPos = atom<Coordinate | null>(null);
export const $spaceKey = atom<boolean>(false);
export const $transformingEntity = atom<CanvasEntityIdentifier | null>(null);
export const $filteringEntity = atom<CanvasEntityIdentifier | null>(null);
export const $filterConfig = atom<FilterConfig>(IMAGE_FILTERS.canny_image_processor.buildDefaults());
export const $isProcessingFilter = atom(false);
export const canvasV2PersistConfig: PersistConfig<CanvasV2State> = {
name: canvasV2Slice.name,