Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): region mask rendering

Browse Source
This commit is contained in:
psychedelicious 2024-08-06 15:56:30 +10:00
parent 83e786bd1e
commit d26095dfa1
11 changed files with 200 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
invokeai/frontend/web/src/features/controlLayers/components/HeadsUpDisplay.tsx
View File

@ -24,10 +24,10 @@ export const HeadsUpDisplay = memo(() => {
return ( return (
<Flex flexDir="column" bg="blackAlpha.400" borderBottomEndRadius="base" p={2} minW={64} gap={2}> <Flex flexDir="column" bg="blackAlpha.400" borderBottomEndRadius="base" p={2} minW={64} gap={2}>
<HUDItem label="Zoom" value={`${round(stageAttrs.scale * 100, 2)}%`} /> <HUDItem label="Zoom" value={`${round(stageAttrs.scale * 100, 2)}%`} />
<HUDItem label="Stage Pos" value={`${round(stageAttrs.position.x, 3)}, ${round(stageAttrs.position.y, 3)}`} /> <HUDItem label="Stage Pos" value={`${round(stageAttrs.x, 3)}, ${round(stageAttrs.y, 3)}`} />
<HUDItem <HUDItem
label="Stage Size" label="Stage Size"
value={`${round(stageAttrs.dimensions.width / stageAttrs.scale, 2)}×${round(stageAttrs.dimensions.height / stageAttrs.scale, 2)} px`} value={`${round(stageAttrs.width / stageAttrs.scale, 2)}×${round(stageAttrs.height / stageAttrs.scale, 2)} px`}
/> />
<HUDItem label="BBox Size" value={`${bbox.rect.width}×${bbox.rect.height} px`} /> <HUDItem label="BBox Size" value={`${bbox.rect.width}×${bbox.rect.height} px`} />
<HUDItem label="BBox Position" value={`${bbox.rect.x}, ${bbox.rect.y}`} /> <HUDItem label="BBox Position" value={`${bbox.rect.x}, ${bbox.rect.y}`} />

21
invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayer.ts → invokeai/frontend/web/src/features/controlLayers/konva/CanvasLayerAdapter.ts
View File

@ -7,13 +7,9 @@ import Konva from 'konva';
import { get } from 'lodash-es'; import { get } from 'lodash-es';
import type { Logger } from 'roarr'; import type { Logger } from 'roarr';
export class CanvasLayer { export class CanvasLayerAdapter {
static TYPE = 'layer' as const;
static KONVA_LAYER_NAME = `${CanvasLayer.TYPE}_layer`;
static KONVA_OBJECT_GROUP_NAME = `${CanvasLayer.TYPE}_object-group`;
id: string; id: string;
type = CanvasLayer.TYPE; type: CanvasLayerState['type'];
manager: CanvasManager; manager: CanvasManager;
log: Logger; log: Logger;
getLoggingContext: GetLoggingContext; getLoggingContext: GetLoggingContext;
@ -29,8 +25,9 @@ export class CanvasLayer {
isFirstRender: boolean = true; isFirstRender: boolean = true;
bboxNeedsUpdate: boolean = true; bboxNeedsUpdate: boolean = true;
constructor(state: CanvasLayerState, manager: CanvasManager) { constructor(state: CanvasLayerAdapter['state'], manager: CanvasLayerAdapter['manager']) {
this.id = state.id; this.id = state.id;
this.type = state.type;
this.manager = manager; this.manager = manager;
this.getLoggingContext = this.manager.buildGetLoggingContext(this); this.getLoggingContext = this.manager.buildGetLoggingContext(this);
this.log = this.manager.buildLogger(this.getLoggingContext); this.log = this.manager.buildLogger(this.getLoggingContext);
@ -39,7 +36,7 @@ export class CanvasLayer {
this.konva = { this.konva = {
layer: new Konva.Layer({ layer: new Konva.Layer({
id: this.id, id: this.id,
name: CanvasLayer.KONVA_LAYER_NAME, name: `${this.type}:layer`,
listening: false, listening: false,
imageSmoothingEnabled: false, imageSmoothingEnabled: false,
}), }),
@ -59,7 +56,11 @@ export class CanvasLayer {
this.konva.layer.destroy(); this.konva.layer.destroy();
}; };
update = async (arg?: { state: CanvasLayerState; toolState: CanvasV2State['tool']; isSelected: boolean }) => { update = async (arg?: {
state: CanvasLayerAdapter['state'];
toolState: CanvasV2State['tool'];
isSelected: boolean;
}) => {
const state = get(arg, 'state', this.state); const state = get(arg, 'state', this.state);
if (!this.isFirstRender && state === this.state) { if (!this.isFirstRender && state === this.state) {
@ -119,7 +120,7 @@ export class CanvasLayer {
repr = () => { repr = () => {
return { return {
id: this.id, id: this.id,
type: CanvasLayer.TYPE, type: this.type,
state: deepClone(this.state), state: deepClone(this.state),
bboxNeedsUpdate: this.bboxNeedsUpdate, bboxNeedsUpdate: this.bboxNeedsUpdate,
transformer: this.transformer.repr(), transformer: this.transformer.repr(),

123
invokeai/frontend/web/src/features/controlLayers/konva/CanvasManager.ts
View File

@ -26,7 +26,6 @@ import { $lastProgressEvent, $shouldShowStagedImage } from 'features/controlLaye
import { import {
type CanvasControlAdapterState, type CanvasControlAdapterState,
type CanvasEntityIdentifier, type CanvasEntityIdentifier,
type CanvasEntityState,
type CanvasInpaintMaskState, type CanvasInpaintMaskState,
type CanvasLayerState, type CanvasLayerState,
type CanvasRegionalGuidanceState, type CanvasRegionalGuidanceState,
@ -42,15 +41,14 @@ import { atom } from 'nanostores';
import type { Logger } from 'roarr'; import type { Logger } from 'roarr';
import { getImageDTO as defaultGetImageDTO, uploadImage as defaultUploadImage } from 'services/api/endpoints/images'; import { getImageDTO as defaultGetImageDTO, uploadImage as defaultUploadImage } from 'services/api/endpoints/images';
import type { ImageCategory, ImageDTO } from 'services/api/types'; import type { ImageCategory, ImageDTO } from 'services/api/types';
import { assert } from 'tsafe';
import { CanvasBackground } from './CanvasBackground'; import { CanvasBackground } from './CanvasBackground';
import { CanvasBbox } from './CanvasBbox'; import { CanvasBbox } from './CanvasBbox';
import { CanvasControlAdapter } from './CanvasControlAdapter'; import { CanvasControlAdapter } from './CanvasControlAdapter';
import { CanvasInpaintMask } from './CanvasInpaintMask'; import { CanvasLayerAdapter } from './CanvasLayerAdapter';
import { CanvasLayer } from './CanvasLayer'; import { CanvasMaskAdapter } from './CanvasMaskAdapter';
import { CanvasPreview } from './CanvasPreview'; import { CanvasPreview } from './CanvasPreview';
import { CanvasRegion } from './CanvasRegion'; import type { CanvasRegion } from './CanvasRegion';
import { CanvasStagingArea } from './CanvasStagingArea'; import { CanvasStagingArea } from './CanvasStagingArea';
import { CanvasStateApi } from './CanvasStateApi'; import { CanvasStateApi } from './CanvasStateApi';
import { CanvasTool } from './CanvasTool'; import { CanvasTool } from './CanvasTool';
@ -86,20 +84,28 @@ type Util = {
type EntityStateAndAdapter = type EntityStateAndAdapter =
| { | {
id: string;
type: CanvasLayerState['type'];
state: CanvasLayerState; state: CanvasLayerState;
adapter: CanvasLayer; adapter: CanvasLayerAdapter;
} }
| { | {
id: string;
type: CanvasInpaintMaskState['type'];
state: CanvasInpaintMaskState; state: CanvasInpaintMaskState;
adapter: CanvasInpaintMask; adapter: CanvasMaskAdapter;
} }
| { | {
id: string;
type: CanvasControlAdapterState['type'];
state: CanvasControlAdapterState; state: CanvasControlAdapterState;
adapter: CanvasControlAdapter; adapter: CanvasControlAdapter;
} }
| { | {
id: string;
type: CanvasRegionalGuidanceState['type'];
state: CanvasRegionalGuidanceState; state: CanvasRegionalGuidanceState;
adapter: CanvasRegion; adapter: CanvasMaskAdapter;
}; };
export const $canvasManager = atom<CanvasManager | null>(null); export const $canvasManager = atom<CanvasManager | null>(null);
@ -111,9 +117,9 @@ export class CanvasManager {
stage: Konva.Stage; stage: Konva.Stage;
container: HTMLDivElement; container: HTMLDivElement;
controlAdapters: Map<string, CanvasControlAdapter>; controlAdapters: Map<string, CanvasControlAdapter>;
layers: Map<string, CanvasLayer>; layers: Map<string, CanvasLayerAdapter>;
regions: Map<string, CanvasRegion>; regions: Map<string, CanvasMaskAdapter>;
inpaintMask: CanvasInpaintMask; inpaintMask: CanvasMaskAdapter;
initialImage: CanvasInitialImage; initialImage: CanvasInitialImage;
util: Util; util: Util;
stateApi: CanvasStateApi; stateApi: CanvasStateApi;
@ -221,7 +227,7 @@ export class CanvasManager {
(a, b) => a?.state === b?.state && a?.adapter === b?.adapter (a, b) => a?.state === b?.state && a?.adapter === b?.adapter
); );
this.inpaintMask = new CanvasInpaintMask(this.stateApi.getInpaintMaskState(), this); this.inpaintMask = new CanvasMaskAdapter(this.stateApi.getInpaintMaskState(), this);
this.stage.add(this.inpaintMask.konva.layer); this.stage.add(this.inpaintMask.konva.layer);
} }
@ -248,28 +254,6 @@ export class CanvasManager {
await this.initialImage.render(this.stateApi.getInitialImageState()); await this.initialImage.render(this.stateApi.getInitialImageState());
} }
async renderRegions() {
const { entities } = this.stateApi.getRegionsState();
// Destroy the konva nodes for nonexistent entities
for (const canvasRegion of this.regions.values()) {
if (!entities.find((rg) => rg.id === canvasRegion.id)) {
canvasRegion.destroy();
this.regions.delete(canvasRegion.id);
}
}
for (const entity of entities) {
let adapter = this.regions.get(entity.id);
if (!adapter) {
adapter = new CanvasRegion(entity, this);
this.regions.set(adapter.id, adapter);
this.stage.add(adapter.konva.layer);
}
await adapter.render(entity);
}
}
async renderProgressPreview() { async renderProgressPreview() {
await this.preview.progressPreview.render(this.stateApi.$lastProgressEvent.get()); await this.preview.progressPreview.render(this.stateApi.$lastProgressEvent.get());
} }
@ -320,8 +304,10 @@ export class CanvasManager {
this.stage.width(this.container.offsetWidth); this.stage.width(this.container.offsetWidth);
this.stage.height(this.container.offsetHeight); this.stage.height(this.container.offsetHeight);
this.stateApi.$stageAttrs.set({ this.stateApi.$stageAttrs.set({
position: { x: this.stage.x(), y: this.stage.y() }, x: this.stage.x(),
dimensions: { width: this.stage.width(), height: this.stage.height() }, y: this.stage.y(),
width: this.stage.width(),
height: this.stage.height(),
scale: this.stage.scaleX(), scale: this.stage.scaleX(),
}); });
this.background.render(); this.background.render();
@ -330,8 +316,13 @@ export class CanvasManager {
getEntity(identifier: CanvasEntityIdentifier): EntityStateAndAdapter | null { getEntity(identifier: CanvasEntityIdentifier): EntityStateAndAdapter | null {
const state = this.stateApi.getState(); const state = this.stateApi.getState();
let entityState: CanvasEntityState | null = null; let entityState:
let entityAdapter: CanvasLayer | CanvasRegion | CanvasControlAdapter | CanvasInpaintMask | null = null; | CanvasLayerState
| CanvasControlAdapterState
| CanvasRegionalGuidanceState
| CanvasInpaintMaskState
| null = null;
let entityAdapter: CanvasLayerAdapter | CanvasControlAdapter | CanvasRegion | CanvasMaskAdapter | null = null;
if (identifier.type === 'layer') { if (identifier.type === 'layer') {
entityState = state.layers.entities.find((i) => i.id === identifier.id) ?? null; entityState = state.layers.entities.find((i) => i.id === identifier.id) ?? null;
@ -348,7 +339,12 @@ export class CanvasManager {
} }
if (entityState && entityAdapter && entityState.type === entityAdapter.type) { if (entityState && entityAdapter && entityState.type === entityAdapter.type) {
return { state: entityState, adapter: entityAdapter } as EntityStateAndAdapter; return {
id: entityState.id,
type: entityState.type,
state: entityState,
adapter: entityAdapter,
} as EntityStateAndAdapter; // TODO(psyche): make TS happy w/o this cast
} }
return null; return null;
@ -400,6 +396,8 @@ export class CanvasManager {
return this.layers.get(id) ?? null; return this.layers.get(id) ?? null;
} else if (type === 'inpaint_mask') { } else if (type === 'inpaint_mask') {
return this.inpaintMask; return this.inpaintMask;
} else if (type === 'regional_guidance') {
return this.regions.get(id) ?? null;
} }
return null; return null;
@ -413,14 +411,14 @@ export class CanvasManager {
if (this.getIsTransforming()) { if (this.getIsTransforming()) {
return; return;
} }
const layer = this.getSelectedEntity(); const entity = this.getSelectedEntity();
if (!entity) {
this.log.warn('No entity selected to transform');
return;
}
// TODO(psyche): Support other entity types // TODO(psyche): Support other entity types
assert( entity.adapter.transformer.startTransform();
layer && (layer.adapter instanceof CanvasLayer || layer.adapter instanceof CanvasInpaintMask), this.transformingEntity.publish({ id: entity.id, type: entity.type });
'No selected layer'
);
layer.adapter.transformer.startTransform();
this.transformingEntity.publish({ id: layer.state.id, type: layer.state.type });
} }
async applyTransform() { async applyTransform() {
@ -460,7 +458,7 @@ export class CanvasManager {
for (const entityState of state.layers.entities) { for (const entityState of state.layers.entities) {
let adapter = this.layers.get(entityState.id); let adapter = this.layers.get(entityState.id);
if (!adapter) { if (!adapter) {
adapter = new CanvasLayer(entityState, this); adapter = new CanvasLayerAdapter(entityState, this);
this.layers.set(adapter.id, adapter); this.layers.set(adapter.id, adapter);
this.stage.add(adapter.konva.layer); this.stage.add(adapter.konva.layer);
} }
@ -491,7 +489,28 @@ export class CanvasManager {
state.selectedEntityIdentifier?.id !== this._prevState.selectedEntityIdentifier?.id state.selectedEntityIdentifier?.id !== this._prevState.selectedEntityIdentifier?.id
) { ) {
this.log.debug('Rendering regions'); this.log.debug('Rendering regions');
await this.renderRegions();
// Destroy the konva nodes for nonexistent entities
for (const canvasRegion of this.regions.values()) {
if (!state.regions.entities.find((rg) => rg.id === canvasRegion.id)) {
canvasRegion.destroy();
this.regions.delete(canvasRegion.id);
}
}
for (const entityState of state.regions.entities) {
let adapter = this.regions.get(entityState.id);
if (!adapter) {
adapter = new CanvasMaskAdapter(entityState, this);
this.regions.set(adapter.id, adapter);
this.stage.add(adapter.konva.layer);
}
await adapter.update({
state: entityState,
toolState: state.tool,
isSelected: state.selectedEntityIdentifier?.id === entityState.id,
});
}
} }
if ( if (
@ -697,14 +716,14 @@ export class CanvasManager {
| CanvasImageRenderer | CanvasImageRenderer
| CanvasTransformer | CanvasTransformer
| CanvasObjectRenderer | CanvasObjectRenderer
| CanvasLayer | CanvasLayerAdapter
| CanvasInpaintMask | CanvasMaskAdapter
| CanvasStagingArea | CanvasStagingArea
): GetLoggingContext => { ): GetLoggingContext => {
if ( if (
instance instanceof CanvasLayer || instance instanceof CanvasLayerAdapter ||
instance instanceof CanvasStagingArea || instance instanceof CanvasStagingArea ||
instance instanceof CanvasInpaintMask instance instanceof CanvasMaskAdapter
) { ) {
return (extra?: JSONObject): JSONObject => { return (extra?: JSONObject): JSONObject => {
return { return {

36
invokeai/frontend/web/src/features/controlLayers/konva/CanvasInpaintMask.ts → invokeai/frontend/web/src/features/controlLayers/konva/CanvasMaskAdapter.ts
View File

@ -1,23 +1,24 @@
import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer'; import { CanvasObjectRenderer } from 'features/controlLayers/konva/CanvasObjectRenderer';
import { CanvasTransformer } from 'features/controlLayers/konva/CanvasTransformer'; import { CanvasTransformer } from 'features/controlLayers/konva/CanvasTransformer';
import type { CanvasInpaintMaskState, CanvasV2State, GetLoggingContext } from 'features/controlLayers/store/types'; import type {
CanvasInpaintMaskState,
CanvasRegionalGuidanceState,
CanvasV2State,
GetLoggingContext,
} from 'features/controlLayers/store/types';
import Konva from 'konva'; import Konva from 'konva';
import { get } from 'lodash-es'; import { get } from 'lodash-es';
import type { Logger } from 'roarr'; import type { Logger } from 'roarr';
export class CanvasInpaintMask { export class CanvasMaskAdapter {
static TYPE = 'inpaint_mask' as const; id: string;
static NAME_PREFIX = 'inpaint-mask'; type: CanvasInpaintMaskState['type'] | CanvasRegionalGuidanceState['type'];
static KONVA_LAYER_NAME = `${CanvasInpaintMask.NAME_PREFIX}_layer`;
id = CanvasInpaintMask.TYPE;
type = CanvasInpaintMask.TYPE;
manager: CanvasManager; manager: CanvasManager;
log: Logger; log: Logger;
getLoggingContext: GetLoggingContext; getLoggingContext: GetLoggingContext;
state: CanvasInpaintMaskState; state: CanvasInpaintMaskState | CanvasRegionalGuidanceState;
maskOpacity: number; maskOpacity: number;
transformer: CanvasTransformer; transformer: CanvasTransformer;
@ -29,15 +30,18 @@ export class CanvasInpaintMask {
layer: Konva.Layer; layer: Konva.Layer;
}; };
constructor(state: CanvasInpaintMaskState, manager: CanvasManager) { constructor(state: CanvasMaskAdapter['state'], manager: CanvasMaskAdapter['manager']) {
this.id = state.id;
this.type = state.type;
this.manager = manager; this.manager = manager;
this.getLoggingContext = this.manager.buildGetLoggingContext(this); this.getLoggingContext = this.manager.buildGetLoggingContext(this);
this.log = this.manager.buildLogger(this.getLoggingContext); this.log = this.manager.buildLogger(this.getLoggingContext);
this.log.debug({ state }, 'Creating inpaint mask'); this.log.debug({ state }, 'Creating mask');
this.konva = { this.konva = {
layer: new Konva.Layer({ layer: new Konva.Layer({
name: CanvasInpaintMask.KONVA_LAYER_NAME, id: this.id,
name: `${this.type}:layer`,
listening: false, listening: false,
imageSmoothingEnabled: false, imageSmoothingEnabled: false,
}), }),
@ -51,14 +55,18 @@ export class CanvasInpaintMask {
} }
destroy = (): void => { destroy = (): void => {
this.log.debug('Destroying inpaint mask'); this.log.debug('Destroying mask');
// We need to call the destroy method on all children so they can do their own cleanup. // We need to call the destroy method on all children so they can do their own cleanup.
this.transformer.destroy(); this.transformer.destroy();
this.renderer.destroy(); this.renderer.destroy();
this.konva.layer.destroy(); this.konva.layer.destroy();
}; };
update = async (arg?: { state: CanvasInpaintMaskState; toolState: CanvasV2State['tool']; isSelected: boolean }) => { update = async (arg?: {
state: CanvasMaskAdapter['state'];
toolState: CanvasV2State['tool'];
isSelected: boolean;
}) => {
const state = get(arg, 'state', this.state); const state = get(arg, 'state', this.state);
const maskOpacity = this.manager.stateApi.getMaskOpacity(); const maskOpacity = this.manager.stateApi.getMaskOpacity();

39
invokeai/frontend/web/src/features/controlLayers/konva/CanvasObjectRenderer.ts
View File

@ -4,9 +4,9 @@ import { deepClone } from 'common/util/deepClone';
import { CanvasBrushLineRenderer } from 'features/controlLayers/konva/CanvasBrushLine'; import { CanvasBrushLineRenderer } from 'features/controlLayers/konva/CanvasBrushLine';
import { CanvasEraserLineRenderer } from 'features/controlLayers/konva/CanvasEraserLine'; import { CanvasEraserLineRenderer } from 'features/controlLayers/konva/CanvasEraserLine';
import { CanvasImageRenderer } from 'features/controlLayers/konva/CanvasImage'; import { CanvasImageRenderer } from 'features/controlLayers/konva/CanvasImage';
import type { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import type { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
import { CanvasRectRenderer } from 'features/controlLayers/konva/CanvasRect'; import { CanvasRectRenderer } from 'features/controlLayers/konva/CanvasRect';
import { getPrefixedId, konvaNodeToBlob, previewBlob } from 'features/controlLayers/konva/util'; import { getPrefixedId, konvaNodeToBlob, previewBlob } from 'features/controlLayers/konva/util';
import { import {
@ -36,11 +36,11 @@ type AnyObjectState = CanvasBrushLineState | CanvasEraserLineState | CanvasImage
*/ */
export class CanvasObjectRenderer { export class CanvasObjectRenderer {
static TYPE = 'object_renderer'; static TYPE = 'object_renderer';
static KONVA_OBJECT_GROUP_NAME = 'object-group'; static KONVA_OBJECT_GROUP_NAME = `${CanvasObjectRenderer.TYPE}:object_group`;
static KONVA_COMPOSITING_RECT_NAME = 'compositing-rect'; static KONVA_COMPOSITING_RECT_NAME = `${CanvasObjectRenderer.TYPE}:compositing_rect`;
id: string; id: string;
parent: CanvasLayer | CanvasInpaintMask; parent: CanvasLayerAdapter | CanvasMaskAdapter;
manager: CanvasManager; manager: CanvasManager;
log: Logger; log: Logger;
getLoggingContext: (extra?: JSONObject) => JSONObject; getLoggingContext: (extra?: JSONObject) => JSONObject;
@ -87,7 +87,7 @@ export class CanvasObjectRenderer {
compositingRect: Konva.Rect | null; compositingRect: Konva.Rect | null;
}; };
constructor(parent: CanvasLayer | CanvasInpaintMask) { constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) {
this.id = getPrefixedId(CanvasObjectRenderer.TYPE); this.id = getPrefixedId(CanvasObjectRenderer.TYPE);
this.parent = parent; this.parent = parent;
this.manager = parent.manager; this.manager = parent.manager;
@ -102,7 +102,7 @@ export class CanvasObjectRenderer {
this.parent.konva.layer.add(this.konva.objectGroup); this.parent.konva.layer.add(this.konva.objectGroup);
if (this.parent.type === 'inpaint_mask') { if (this.parent.type === 'inpaint_mask' || this.parent.type === 'regional_guidance') {
this.konva.compositingRect = new Konva.Rect({ this.konva.compositingRect = new Konva.Rect({
name: CanvasObjectRenderer.KONVA_COMPOSITING_RECT_NAME, name: CanvasObjectRenderer.KONVA_COMPOSITING_RECT_NAME,
listening: false, listening: false,
@ -122,13 +122,13 @@ export class CanvasObjectRenderer {
// The compositing rect must cover the whole stage at all times. When the stage is scaled, moved or resized, we // The compositing rect must cover the whole stage at all times. When the stage is scaled, moved or resized, we
// need to update the compositing rect to match the stage. // need to update the compositing rect to match the stage.
this.subscriptions.add( this.subscriptions.add(
this.manager.stateApi.$stageAttrs.listen((attrs) => { this.manager.stateApi.$stageAttrs.listen(({ x, y, width, height, scale }) => {
if (this.konva.compositingRect) { if (this.konva.compositingRect) {
this.konva.compositingRect.setAttrs({ this.konva.compositingRect.setAttrs({
x: -attrs.position.x / attrs.scale, x: -x / scale,
y: -attrs.position.y / attrs.scale, y: -y / scale,
width: attrs.dimensions.width / attrs.scale, width: width / scale,
height: attrs.dimensions.height / attrs.scale, height: height / scale,
}); });
} }
}) })
@ -168,9 +168,14 @@ export class CanvasObjectRenderer {
assert(this.konva.compositingRect, 'Missing compositing rect'); assert(this.konva.compositingRect, 'Missing compositing rect');
const rgbColor = rgbColorToString(fill); const rgbColor = rgbColorToString(fill);
const { x, y, width, height, scale } = this.manager.stateApi.$stageAttrs.get();
this.konva.compositingRect.setAttrs({ this.konva.compositingRect.setAttrs({
fill: rgbColor, fill: rgbColor,
opacity, opacity,
x: -x / scale,
y: -y / scale,
width: width / scale,
height: height / scale,
}); });
}; };
@ -288,11 +293,11 @@ export class CanvasObjectRenderer {
this.buffer.id = getPrefixedId(this.buffer.type); this.buffer.id = getPrefixedId(this.buffer.type);
if (this.buffer.type === 'brush_line') { if (this.buffer.type === 'brush_line') {
this.manager.stateApi.addBrushLine({ id: this.parent.id, brushLine: this.buffer }, this.parent.state.type); this.manager.stateApi.addBrushLine({ id: this.parent.id, brushLine: this.buffer }, this.parent.type);
} else if (this.buffer.type === 'eraser_line') { } else if (this.buffer.type === 'eraser_line') {
this.manager.stateApi.addEraserLine({ id: this.parent.id, eraserLine: this.buffer }, this.parent.state.type); this.manager.stateApi.addEraserLine({ id: this.parent.id, eraserLine: this.buffer }, this.parent.type);
} else if (this.buffer.type === 'rect') { } else if (this.buffer.type === 'rect') {
this.manager.stateApi.addRect({ id: this.parent.id, rect: this.buffer }, this.parent.state.type); this.manager.stateApi.addRect({ id: this.parent.id, rect: this.buffer }, this.parent.type);
} else { } else {
this.log.warn({ buffer: this.buffer }, 'Invalid buffer object type'); this.log.warn({ buffer: this.buffer }, 'Invalid buffer object type');
} }
@ -340,13 +345,13 @@ export class CanvasObjectRenderer {
const rect = interactionRectClone.getClientRect(); const rect = interactionRectClone.getClientRect();
const blob = await konvaNodeToBlob(objectGroupClone, rect); const blob = await konvaNodeToBlob(objectGroupClone, rect);
if (this.manager._isDebugging) { if (this.manager._isDebugging) {
previewBlob(blob, 'Rasterized layer'); previewBlob(blob, 'Rasterized entity');
} }
const imageDTO = await uploadImage(blob, `${this.id}_rasterized.png`, 'other', true); const imageDTO = await uploadImage(blob, `${this.id}_rasterized.png`, 'other', true);
const imageObject = imageDTOToImageObject(imageDTO); const imageObject = imageDTOToImageObject(imageDTO);
await this.renderObject(imageObject, true); await this.renderObject(imageObject, true);
this.manager.stateApi.rasterizeEntity( this.manager.stateApi.rasterizeEntity(
{ id: this.id, imageObject, position: { x: Math.round(rect.x), y: Math.round(rect.y) } }, { id: this.parent.id, imageObject, position: { x: Math.round(rect.x), y: Math.round(rect.y) } },
this.parent.type this.parent.type
); );
}; };

3
invokeai/frontend/web/src/features/controlLayers/konva/CanvasStateApi.ts
View File

@ -31,6 +31,7 @@ import {
layerRectAdded, layerRectAdded,
layerReset, layerReset,
layerTranslated, layerTranslated,
regionMaskRasterized,
rgBrushLineAdded, rgBrushLineAdded,
rgEraserLineAdded, rgEraserLineAdded,
rgImageCacheChanged, rgImageCacheChanged,
@ -122,6 +123,8 @@ export class CanvasStateApi {
this._store.dispatch(layerRasterized(arg)); this._store.dispatch(layerRasterized(arg));
} else if (entityType === 'inpaint_mask') { } else if (entityType === 'inpaint_mask') {
this._store.dispatch(inpaintMaskRasterized(arg)); this._store.dispatch(inpaintMaskRasterized(arg));
} else if (entityType === 'regional_guidance') {
this._store.dispatch(regionMaskRasterized(arg));
} else { } else {
assert(false, 'Rasterizing not supported for this entity type'); assert(false, 'Rasterizing not supported for this entity type');
} }

41
invokeai/frontend/web/src/features/controlLayers/konva/CanvasTransformer.ts
View File

@ -1,6 +1,6 @@
import type { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; import type { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
import type { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer';
import { CanvasManager } from 'features/controlLayers/konva/CanvasManager'; import { CanvasManager } from 'features/controlLayers/konva/CanvasManager';
import type { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
import { getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util'; import { getEmptyRect, getPrefixedId } from 'features/controlLayers/konva/util';
import type { Coordinate, GetLoggingContext, Rect } from 'features/controlLayers/store/types'; import type { Coordinate, GetLoggingContext, Rect } from 'features/controlLayers/store/types';
import Konva from 'konva'; import Konva from 'konva';
@ -17,9 +17,10 @@ import type { Logger } from 'roarr';
*/ */
export class CanvasTransformer { export class CanvasTransformer {
static TYPE = 'entity_transformer'; static TYPE = 'entity_transformer';
static TRANSFORMER_NAME = `${CanvasTransformer.TYPE}:transformer`; static KONVA_TRANSFORMER_NAME = `${CanvasTransformer.TYPE}:transformer`;
static PROXY_RECT_NAME = `${CanvasTransformer.TYPE}:proxy_rect`; static KONVA_PROXY_RECT_NAME = `${CanvasTransformer.TYPE}:proxy_rect`;
static BBOX_OUTLINE_NAME = `${CanvasTransformer.TYPE}:bbox_outline`; static KONVA_OUTLINE_RECT_NAME = `${CanvasTransformer.TYPE}:outline_rect`;
static STROKE_COLOR = 'hsl(200 76% 50% / 1)'; // invokeBlue.500 static STROKE_COLOR = 'hsl(200 76% 50% / 1)'; // invokeBlue.500
static ANCHOR_FILL_COLOR = CanvasTransformer.STROKE_COLOR; static ANCHOR_FILL_COLOR = CanvasTransformer.STROKE_COLOR;
static ANCHOR_STROKE_COLOR = 'hsl(200 76% 77% / 1)'; // invokeBlue.200 static ANCHOR_STROKE_COLOR = 'hsl(200 76% 77% / 1)'; // invokeBlue.200
@ -32,7 +33,7 @@ export class CanvasTransformer {
static ANCHOR_HIT_PADDING = 10; static ANCHOR_HIT_PADDING = 10;
id: string; id: string;
parent: CanvasLayer | CanvasInpaintMask; parent: CanvasLayerAdapter | CanvasMaskAdapter;
manager: CanvasManager; manager: CanvasManager;
log: Logger; log: Logger;
getLoggingContext: GetLoggingContext; getLoggingContext: GetLoggingContext;
@ -87,10 +88,10 @@ export class CanvasTransformer {
konva: { konva: {
transformer: Konva.Transformer; transformer: Konva.Transformer;
proxyRect: Konva.Rect; proxyRect: Konva.Rect;
bboxOutline: Konva.Rect; outlineRect: Konva.Rect;
}; };
constructor(parent: CanvasLayer | CanvasInpaintMask) { constructor(parent: CanvasLayerAdapter | CanvasMaskAdapter) {
this.id = getPrefixedId(CanvasTransformer.TYPE); this.id = getPrefixedId(CanvasTransformer.TYPE);
this.parent = parent; this.parent = parent;
this.manager = parent.manager; this.manager = parent.manager;
@ -99,16 +100,16 @@ export class CanvasTransformer {
this.log = this.manager.buildLogger(this.getLoggingContext); this.log = this.manager.buildLogger(this.getLoggingContext);
this.konva = { this.konva = {
bboxOutline: new Konva.Rect({ outlineRect: new Konva.Rect({
listening: false, listening: false,
draggable: false, draggable: false,
name: CanvasTransformer.BBOX_OUTLINE_NAME, name: CanvasTransformer.KONVA_OUTLINE_RECT_NAME,
stroke: CanvasTransformer.STROKE_COLOR, stroke: CanvasTransformer.STROKE_COLOR,
perfectDrawEnabled: false, perfectDrawEnabled: false,
strokeHitEnabled: false, strokeHitEnabled: false,
}), }),
transformer: new Konva.Transformer({ transformer: new Konva.Transformer({
name: CanvasTransformer.TRANSFORMER_NAME, name: CanvasTransformer.KONVA_TRANSFORMER_NAME,
// Visibility and listening are managed via activate() and deactivate() // Visibility and listening are managed via activate() and deactivate()
visible: false, visible: false,
listening: false, listening: false,
@ -227,7 +228,7 @@ export class CanvasTransformer {
}, },
}), }),
proxyRect: new Konva.Rect({ proxyRect: new Konva.Rect({
name: CanvasTransformer.PROXY_RECT_NAME, name: CanvasTransformer.KONVA_PROXY_RECT_NAME,
listening: false, listening: false,
draggable: true, draggable: true,
}), }),
@ -330,7 +331,7 @@ export class CanvasTransformer {
// The bbox should be updated to reflect the new position of the interaction rect, taking into account its padding // The bbox should be updated to reflect the new position of the interaction rect, taking into account its padding
// and border // and border
this.konva.bboxOutline.setAttrs({ this.konva.outlineRect.setAttrs({
x: this.konva.proxyRect.x() - this.manager.getScaledBboxPadding(), x: this.konva.proxyRect.x() - this.manager.getScaledBboxPadding(),
y: this.konva.proxyRect.y() - this.manager.getScaledBboxPadding(), y: this.konva.proxyRect.y() - this.manager.getScaledBboxPadding(),
}); });
@ -392,7 +393,7 @@ export class CanvasTransformer {
}) })
); );
this.parent.konva.layer.add(this.konva.bboxOutline); this.parent.konva.layer.add(this.konva.outlineRect);
this.parent.konva.layer.add(this.konva.proxyRect); this.parent.konva.layer.add(this.konva.proxyRect);
this.parent.konva.layer.add(this.konva.transformer); this.parent.konva.layer.add(this.konva.transformer);
} }
@ -406,7 +407,7 @@ export class CanvasTransformer {
const onePixel = this.manager.getScaledPixel(); const onePixel = this.manager.getScaledPixel();
const bboxPadding = this.manager.getScaledBboxPadding(); const bboxPadding = this.manager.getScaledBboxPadding();
this.konva.bboxOutline.setAttrs({ this.konva.outlineRect.setAttrs({
x: position.x + bbox.x - bboxPadding, x: position.x + bbox.x - bboxPadding,
y: position.y + bbox.y - bboxPadding, y: position.y + bbox.y - bboxPadding,
width: bbox.width + bboxPadding * 2, width: bbox.width + bboxPadding * 2,
@ -473,7 +474,7 @@ export class CanvasTransformer {
const onePixel = this.manager.getScaledPixel(); const onePixel = this.manager.getScaledPixel();
const bboxPadding = this.manager.getScaledBboxPadding(); const bboxPadding = this.manager.getScaledBboxPadding();
this.konva.bboxOutline.setAttrs({ this.konva.outlineRect.setAttrs({
x: this.konva.proxyRect.x() - bboxPadding, x: this.konva.proxyRect.x() - bboxPadding,
y: this.konva.proxyRect.y() - bboxPadding, y: this.konva.proxyRect.y() - bboxPadding,
width: this.konva.proxyRect.width() * this.konva.proxyRect.scaleX() + bboxPadding * 2, width: this.konva.proxyRect.width() * this.konva.proxyRect.scaleX() + bboxPadding * 2,
@ -539,7 +540,7 @@ export class CanvasTransformer {
rotation: 0, rotation: 0,
}; };
this.parent.renderer.konva.objectGroup.setAttrs(attrs); this.parent.renderer.konva.objectGroup.setAttrs(attrs);
this.konva.bboxOutline.setAttrs(attrs); this.konva.outlineRect.setAttrs(attrs);
this.konva.proxyRect.setAttrs(attrs); this.konva.proxyRect.setAttrs(attrs);
}; };
@ -706,11 +707,11 @@ export class CanvasTransformer {
}; };
_showBboxOutline = () => { _showBboxOutline = () => {
this.konva.bboxOutline.visible(true); this.konva.outlineRect.visible(true);
}; };
_hideBboxOutline = () => { _hideBboxOutline = () => {
this.konva.bboxOutline.visible(false); this.konva.outlineRect.visible(false);
}; };
/** /**
@ -735,7 +736,7 @@ export class CanvasTransformer {
this.log.trace('Cleaning up listener'); this.log.trace('Cleaning up listener');
cleanup(); cleanup();
} }
this.konva.bboxOutline.destroy(); this.konva.outlineRect.destroy();
this.konva.transformer.destroy(); this.konva.transformer.destroy();
this.konva.proxyRect.destroy(); this.konva.proxyRect.destroy();
}; };

24
invokeai/frontend/web/src/features/controlLayers/konva/events.ts
View File

@ -487,8 +487,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
stage.scaleY(newScale); stage.scaleY(newScale);
stage.position(newPos); stage.position(newPos);
$stageAttrs.set({ $stageAttrs.set({
position: newPos, x: newPos.x,
dimensions: { width: stage.width(), height: stage.height() }, y: newPos.y,
width: stage.width(),
height: stage.height(),
scale: newScale, scale: newScale,
}); });
manager.background.render(); manager.background.render();
@ -500,8 +502,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
//#region dragmove //#region dragmove
stage.on('dragmove', () => { stage.on('dragmove', () => {
$stageAttrs.set({ $stageAttrs.set({
position: { x: Math.floor(stage.x()), y: Math.floor(stage.y()) }, x: Math.floor(stage.x()),
dimensions: { width: stage.width(), height: stage.height() }, y: Math.floor(stage.y()),
width: stage.width(),
height: stage.height(),
scale: stage.scaleX(), scale: stage.scaleX(),
}); });
manager.background.render(); manager.background.render();
@ -512,8 +516,10 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
stage.on('dragend', () => { stage.on('dragend', () => {
// Stage position should always be an integer, else we get fractional pixels which are blurry // Stage position should always be an integer, else we get fractional pixels which are blurry
$stageAttrs.set({ $stageAttrs.set({
position: { x: Math.floor(stage.x()), y: Math.floor(stage.y()) }, x: Math.floor(stage.x()),
dimensions: { width: stage.width(), height: stage.height() }, y: Math.floor(stage.y()),
width: stage.width(),
height: stage.height(),
scale: stage.scaleX(), scale: stage.scaleX(),
}); });
manager.preview.tool.render(); manager.preview.tool.render();
@ -529,7 +535,11 @@ export const setStageEventHandlers = (manager: CanvasManager): (() => void) => {
} }
if (e.key === 'Escape') { if (e.key === 'Escape') {
// Cancel shape drawing on escape // Cancel shape drawing on escape
$lastMouseDownPos.set(null); const selectedEntity = getSelectedEntity();
if (selectedEntity) {
selectedEntity.adapter.renderer.clearBuffer();
$lastMouseDownPos.set(null);
}
} else if (e.key === ' ') { } else if (e.key === ' ') {
// Select the view tool on space key down // Select the view tool on space key down
setToolBuffer(getToolState().selected); setToolBuffer(getToolState().selected);

7
invokeai/frontend/web/src/features/controlLayers/store/canvasV2Slice.ts
View File

@ -289,6 +289,7 @@ export const {
rgBrushLineAdded, rgBrushLineAdded,
rgEraserLineAdded, rgEraserLineAdded,
rgRectAdded, rgRectAdded,
regionMaskRasterized,
// Compositing // Compositing
setInfillMethod, setInfillMethod,
setInfillTileSize, setInfillTileSize,
@ -371,8 +372,10 @@ const migrate = (state: any): any => {
// Ephemeral state that does not need to be in redux // Ephemeral state that does not need to be in redux
export const $isPreviewVisible = atom(true); export const $isPreviewVisible = atom(true);
export const $stageAttrs = atom<StageAttrs>({ export const $stageAttrs = atom<StageAttrs>({
position: { x: 0, y: 0 }, x: 0,
dimensions: { width: 0, height: 0 }, y: 0,
width: 0,
height: 0,
scale: 0, scale: 0,
}); });
export const $shouldShowStagedImage = atom(true); export const $shouldShowStagedImage = atom(true);

11
invokeai/frontend/web/src/features/controlLayers/store/regionsReducers.ts
View File

@ -6,6 +6,7 @@ import type {
CanvasRectState, CanvasRectState,
CanvasV2State, CanvasV2State,
CLIPVisionModelV2, CLIPVisionModelV2,
EntityRasterizedArg,
IPMethodV2, IPMethodV2,
PositionChangedArg, PositionChangedArg,
ScaleChangedArg, ScaleChangedArg,
@ -361,4 +362,14 @@ export const regionsReducers = {
rg.bboxNeedsUpdate = true; rg.bboxNeedsUpdate = true;
state.layers.imageCache = null; state.layers.imageCache = null;
}, },
regionMaskRasterized: (state, action: PayloadAction<EntityRasterizedArg>) => {
const { id, imageObject, position } = action.payload;
const rg = selectRG(state, id);
if (!rg) {
return;
}
rg.objects = [imageObject];
rg.position = position;
rg.imageCache = null;
},
} satisfies SliceCaseReducers<CanvasV2State>; } satisfies SliceCaseReducers<CanvasV2State>;

22
invokeai/frontend/web/src/features/controlLayers/store/types.ts
View File

@ -1,7 +1,7 @@
import type { JSONObject } from 'common/types'; import type { JSONObject } from 'common/types';
import type { CanvasControlAdapter } from 'features/controlLayers/konva/CanvasControlAdapter'; import type { CanvasControlAdapter } from 'features/controlLayers/konva/CanvasControlAdapter';
import { CanvasInpaintMask } from 'features/controlLayers/konva/CanvasInpaintMask'; import { CanvasLayerAdapter } from 'features/controlLayers/konva/CanvasLayerAdapter';
import { CanvasLayer } from 'features/controlLayers/konva/CanvasLayer'; import { CanvasMaskAdapter } from 'features/controlLayers/konva/CanvasMaskAdapter';
import { CanvasRegion } from 'features/controlLayers/konva/CanvasRegion'; import { CanvasRegion } from 'features/controlLayers/konva/CanvasRegion';
import { getObjectId } from 'features/controlLayers/konva/util'; import { getObjectId } from 'features/controlLayers/konva/util';
import { zModelIdentifierField } from 'features/nodes/types/common'; import { zModelIdentifierField } from 'features/nodes/types/common';
@ -658,7 +658,7 @@ export const zCanvasRegionalGuidanceState = z.object({
position: zCoordinate, position: zCoordinate,
bbox: zRect.nullable(), bbox: zRect.nullable(),
bboxNeedsUpdate: z.boolean(), bboxNeedsUpdate: z.boolean(),
objects: z.array(zMaskObject), objects: z.array(zCanvasObjectState),
positivePrompt: zParameterPositivePrompt.nullable(), positivePrompt: zParameterPositivePrompt.nullable(),
negativePrompt: zParameterNegativePrompt.nullable(), negativePrompt: zParameterNegativePrompt.nullable(),
ipAdapters: z.array(zCanvasIPAdapterState), ipAdapters: z.array(zCanvasIPAdapterState),
@ -933,7 +933,13 @@ export type CanvasV2State = {
}; };
}; };
export type StageAttrs = { position: Coordinate; dimensions: Dimensions; scale: number }; export type StageAttrs = {
x: Coordinate['x'];
y: Coordinate['y'];
width: Dimensions['width'];
height: Dimensions['height'];
scale: number;
};
export type PositionChangedArg = { id: string; position: Coordinate }; export type PositionChangedArg = { id: string; position: Coordinate };
export type ScaleChangedArg = { id: string; scale: Coordinate; position: Coordinate }; export type ScaleChangedArg = { id: string; scale: Coordinate; position: Coordinate };
export type BboxChangedArg = { id: string; bbox: Rect | null }; export type BboxChangedArg = { id: string; bbox: Rect | null };
@ -969,9 +975,11 @@ export function isDrawableEntity(
} }
export function isDrawableEntityAdapter( export function isDrawableEntityAdapter(
adapter: CanvasLayer | CanvasRegion | CanvasControlAdapter | CanvasInpaintMask adapter: CanvasLayerAdapter | CanvasRegion | CanvasControlAdapter | CanvasMaskAdapter
): adapter is CanvasLayer | CanvasRegion | CanvasInpaintMask { ): adapter is CanvasLayerAdapter | CanvasRegion | CanvasMaskAdapter {
return adapter instanceof CanvasLayer || adapter instanceof CanvasRegion || adapter instanceof CanvasInpaintMask; return (
adapter instanceof CanvasLayerAdapter || adapter instanceof CanvasRegion || adapter instanceof CanvasMaskAdapter
);
} }
export function isDrawableEntityType( export function isDrawableEntityType(