mirror of
https://github.com/invoke-ai/InvokeAI
synced 2024-08-30 20:32:17 +00:00
Partial migration of UI to nodes API (#3195)
* feat(ui): add axios client generator and simple example * fix(ui): update client & nodes test code w/ new Edge type * chore(ui): organize generated files * chore(ui): update .eslintignore, .prettierignore * chore(ui): update openapi.json * feat(backend): fixes for nodes/generator * feat(ui): generate object args for api client * feat(ui): more nodes api prototyping * feat(ui): nodes cancel * chore(ui): regenerate api client * fix(ui): disable OG web server socket connection * fix(ui): fix scrollbar styles typing and prop just noticed the typo, and made the types stronger. * feat(ui): add socketio types * feat(ui): wip nodes - extract api client method arg types instead of manually declaring them - update example to display images - general tidy up * start building out node translations from frontend state and add notes about missing features * use reference to sampler_name * use reference to sampler_name * add optional apiUrl prop * feat(ui): start hooking up dynamic txt2img node generation, create middleware for session invocation * feat(ui): write separate nodes socket layer, txt2img generating and rendering w single node * feat(ui): img2img implementation * feat(ui): get intermediate images working but types are stubbed out * chore(ui): add support for package mode * feat(ui): add nodes mode script * feat(ui): handle random seeds * fix(ui): fix middleware types * feat(ui): add rtk action type guard * feat(ui): disable NodeAPITest This was polluting the network/socket logs. * feat(ui): fix parameters panel border color This commit should be elsewhere but I don't want to break my flow * feat(ui): make thunk types more consistent * feat(ui): add type guards for outputs * feat(ui): load images on socket connect Rudimentary * chore(ui): bump redux-toolkit * docs(ui): update readme * chore(ui): regenerate api client * chore(ui): add typescript as dev dependency I am having trouble with TS versions after vscode updated and now uses TS 5. `madge` has installed 3.9.10 and for whatever reason my vscode wants to use that. Manually specifying 4.9.5 and then setting vscode to use that as the workspace TS fixes the issue. * feat(ui): begin migrating gallery to nodes Along the way, migrate to use RTK `createEntityAdapter` for gallery images, and separate `results` and `uploads` into separate slices. Much cleaner this way. * feat(ui): clean up & comment results slice * fix(ui): separate thunk for initial gallery load so it properly gets index 0 * feat(ui): POST upload working * fix(ui): restore removed type * feat(ui): patch api generation for headers access * chore(ui): regenerate api * feat(ui): wip gallery migration * feat(ui): wip gallery migration * chore(ui): regenerate api * feat(ui): wip refactor socket events * feat(ui): disable panels based on app props * feat(ui): invert logic to be disabled * disable panels when app mounts * feat(ui): add support to disableTabs * docs(ui): organise and update docs * lang(ui): add toast strings * feat(ui): wip events, comments, and general refactoring * feat(ui): add optional token for auth * feat(ui): export StatusIndicator and ModelSelect for header use * feat(ui) working on making socket URL dynamic * feat(ui): dynamic middleware loading * feat(ui): prep for socket jwt * feat(ui): migrate cancelation also updated action names to be event-like instead of declaration-like sorry, i was scattered and this commit has a lot of unrelated stuff in it. * fix(ui): fix img2img type * chore(ui): regenerate api client * feat(ui): improve InvocationCompleteEvent types * feat(ui): increase StatusIndicator font size * fix(ui): fix middleware order for multi-node graphs * feat(ui): add exampleGraphs object w/ iterations example * feat(ui): generate iterations graph * feat(ui): update ModelSelect for nodes API * feat(ui): add hi-res functionality for txt2img generations * feat(ui): "subscribe" to particular nodes feels like a dirty hack but oh well it works * feat(ui): first steps to node editor ui * fix(ui): disable event subscription it is not fully baked just yet * feat(ui): wip node editor * feat(ui): remove extraneous field types * feat(ui): nodes before deleting stuff * feat(ui): cleanup nodes ui stuff * feat(ui): hook up nodes to redux * fix(ui): fix handle * fix(ui): add basic node edges & connection validation * feat(ui): add connection validation styling * feat(ui): increase edge width * feat(ui): it blends * feat(ui): wip model handling and graph topology validation * feat(ui): validation connections w/ graphlib * docs(ui): update nodes doc * feat(ui): wip node editor * chore(ui): rebuild api, update types * add redux-dynamic-middlewares as a dependency * feat(ui): add url host transformation * feat(ui): handle already-connected fields * feat(ui): rewrite SqliteItemStore in sqlalchemy * fix(ui): fix sqlalchemy dynamic model instantiation * feat(ui, nodes): metadata wip * feat(ui, nodes): models * feat(ui, nodes): more metadata wip * feat(ui): wip range/iterate * fix(nodes): fix sqlite typing * feat(ui): export new type for invoke component * tests(nodes): fix test instantiation of ImageField * feat(nodes): fix LoadImageInvocation * feat(nodes): add `title` ui hint * feat(nodes): make ImageField attrs optional * feat(ui): wip nodes etc * feat(nodes): roll back sqlalchemy * fix(nodes): partially address feedback * fix(backend): roll back changes to pngwriter * feat(nodes): wip address metadata feedback * feat(nodes): add seeded rng to RandomRange * feat(nodes): address feedback * feat(nodes): move GET images error handling to DiskImageStorage * feat(nodes): move GET images error handling to DiskImageStorage * fix(nodes): fix image output schema customization * feat(ui): img2img/txt2img -> linear - remove txt2img and img2img tabs - add linear tab - add initial image selection to linear parameters accordion * feat(ui): tidy graph builders * feat(ui): tidy misc * feat(ui): improve invocation union types * feat(ui): wip metadata viewer recall * feat(ui): move fonts to normal deps * feat(nodes): fix broken upload * feat(nodes): add metadata module + tests, thumbnails - `MetadataModule` is stateless and needed in places where the `InvocationContext` is not available, so have not made it a `service` - Handles loading/parsing/building metadata, and creating png info objects - added tests for MetadataModule - Lifted thumbnail stuff to util * fix(nodes): revert change to RandomRangeInvocation * feat(nodes): address feedback - make metadata a service - rip out pydantic validation, implement metadata parsing as simple functions - update tests - address other minor feedback items * fix(nodes): fix other tests * fix(nodes): add metadata service to cli * fix(nodes): fix latents/image field parsing * feat(nodes): customise LatentsField schema * feat(nodes): move metadata parsing to frontend * fix(nodes): fix metadata test --------- Co-authored-by: maryhipp <maryhipp@gmail.com> Co-authored-by: Mary Hipp <maryhipp@Marys-MacBook-Air.local>
This commit is contained in:
psychedelicious
GitHub
50
invokeai/frontend/web/src/services/events/actions.ts
Normal file
50
invokeai/frontend/web/src/services/events/actions.ts
Normal file
@ -0,0 +1,50 @@
|
||||
import { createAction } from '@reduxjs/toolkit';
|
||||
import {
|
||||
GeneratorProgressEvent,
|
||||
InvocationCompleteEvent,
|
||||
InvocationErrorEvent,
|
||||
InvocationStartedEvent,
|
||||
} from 'services/events/types';
|
||||
|
||||
// Common socket action payload data
|
||||
type BaseSocketPayload = {
|
||||
timestamp: string;
|
||||
};
|
||||
|
||||
// Create actions for each socket event
|
||||
// Middleware and redux can then respond to them as needed
|
||||
|
||||
export const socketConnected = createAction<BaseSocketPayload>(
|
||||
'socket/socketConnected'
|
||||
);
|
||||
|
||||
export const socketDisconnected = createAction<BaseSocketPayload>(
|
||||
'socket/socketDisconnected'
|
||||
);
|
||||
|
||||
export const socketSubscribed = createAction<
|
||||
BaseSocketPayload & { sessionId: string }
|
||||
>('socket/socketSubscribed');
|
||||
|
||||
export const socketUnsubscribed = createAction<
|
||||
BaseSocketPayload & { sessionId: string }
|
||||
>('socket/socketUnsubscribed');
|
||||
|
||||
export const invocationStarted = createAction<
|
||||
BaseSocketPayload & { data: InvocationStartedEvent }
|
||||
>('socket/invocationStarted');
|
||||
|
||||
export const invocationComplete = createAction<
|
||||
BaseSocketPayload & { data: InvocationCompleteEvent }
|
||||
>('socket/invocationComplete');
|
||||
|
||||
export const invocationError = createAction<
|
||||
BaseSocketPayload & { data: InvocationErrorEvent }
|
||||
>('socket/invocationError');
|
||||
|
||||
export const generatorProgress = createAction<
|
||||
BaseSocketPayload & { data: GeneratorProgressEvent }
|
||||
>('socket/generatorProgress');
|
||||
|
||||
// dispatch this when we need to fully reset the socket connection
|
||||
export const socketReset = createAction('socket/socketReset');
|
||||
221
invokeai/frontend/web/src/services/events/middleware.ts
Normal file
221
invokeai/frontend/web/src/services/events/middleware.ts
Normal file
@ -0,0 +1,221 @@
|
||||
import { Middleware, MiddlewareAPI } from '@reduxjs/toolkit';
|
||||
import { io, Socket } from 'socket.io-client';
|
||||
|
||||
import {
|
||||
ClientToServerEvents,
|
||||
ServerToClientEvents,
|
||||
} from 'services/events/types';
|
||||
import {
|
||||
generatorProgress,
|
||||
invocationComplete,
|
||||
invocationError,
|
||||
invocationStarted,
|
||||
socketConnected,
|
||||
socketDisconnected,
|
||||
socketReset,
|
||||
socketSubscribed,
|
||||
socketUnsubscribed,
|
||||
} from './actions';
|
||||
import {
|
||||
receivedResultImagesPage,
|
||||
receivedUploadImagesPage,
|
||||
} from 'services/thunks/gallery';
|
||||
import { AppDispatch, RootState } from 'app/store';
|
||||
import { getTimestamp } from 'common/util/getTimestamp';
|
||||
import {
|
||||
sessionInvoked,
|
||||
isFulfilledSessionCreatedAction,
|
||||
sessionCanceled,
|
||||
} from 'services/thunks/session';
|
||||
import { OpenAPI } from 'services/api';
|
||||
import { receivedModels } from 'services/thunks/model';
|
||||
import { receivedOpenAPISchema } from 'services/thunks/schema';
|
||||
|
||||
export const socketMiddleware = () => {
|
||||
let areListenersSet = false;
|
||||
|
||||
let socketUrl = `ws://${window.location.host}`;
|
||||
|
||||
const socketOptions: Parameters<typeof io>[0] = {
|
||||
timeout: 60000,
|
||||
path: '/ws/socket.io',
|
||||
autoConnect: false, // achtung! removing this breaks the dynamic middleware
|
||||
};
|
||||
|
||||
// if building in package mode, replace socket url with open api base url minus the http protocol
|
||||
if (['nodes', 'package'].includes(import.meta.env.MODE)) {
|
||||
if (OpenAPI.BASE) {
|
||||
//eslint-disable-next-line
|
||||
socketUrl = OpenAPI.BASE.replace(/^https?\:\/\//i, '');
|
||||
}
|
||||
|
||||
if (OpenAPI.TOKEN) {
|
||||
// TODO: handle providing jwt to socket.io
|
||||
socketOptions.auth = { token: OpenAPI.TOKEN };
|
||||
}
|
||||
}
|
||||
|
||||
const socket: Socket<ServerToClientEvents, ClientToServerEvents> = io(
|
||||
socketUrl,
|
||||
socketOptions
|
||||
);
|
||||
|
||||
const middleware: Middleware =
|
||||
(store: MiddlewareAPI<AppDispatch, RootState>) => (next) => (action) => {
|
||||
const { dispatch, getState } = store;
|
||||
|
||||
// Nothing dispatches `socketReset` actions yet, so this is a noop, but including anyways
|
||||
if (socketReset.match(action)) {
|
||||
const { sessionId } = getState().system;
|
||||
|
||||
if (sessionId) {
|
||||
socket.emit('unsubscribe', { session: sessionId });
|
||||
dispatch(
|
||||
socketUnsubscribed({ sessionId, timestamp: getTimestamp() })
|
||||
);
|
||||
}
|
||||
|
||||
if (socket.connected) {
|
||||
socket.disconnect();
|
||||
dispatch(socketDisconnected({ timestamp: getTimestamp() }));
|
||||
}
|
||||
|
||||
socket.removeAllListeners();
|
||||
areListenersSet = false;
|
||||
}
|
||||
|
||||
// Set listeners for `connect` and `disconnect` events once
|
||||
// Must happen in middleware to get access to `dispatch`
|
||||
if (!areListenersSet) {
|
||||
socket.on('connect', () => {
|
||||
dispatch(socketConnected({ timestamp: getTimestamp() }));
|
||||
|
||||
const { results, uploads, models, nodes } = getState();
|
||||
|
||||
// These thunks need to be dispatch in middleware; cannot handle in a reducer
|
||||
if (!results.ids.length) {
|
||||
dispatch(receivedResultImagesPage());
|
||||
}
|
||||
|
||||
if (!uploads.ids.length) {
|
||||
dispatch(receivedUploadImagesPage());
|
||||
}
|
||||
|
||||
if (!models.ids.length) {
|
||||
dispatch(receivedModels());
|
||||
}
|
||||
|
||||
if (!nodes.schema) {
|
||||
dispatch(receivedOpenAPISchema());
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('disconnect', () => {
|
||||
dispatch(socketDisconnected({ timestamp: getTimestamp() }));
|
||||
});
|
||||
|
||||
areListenersSet = true;
|
||||
|
||||
// must manually connect
|
||||
socket.connect();
|
||||
}
|
||||
|
||||
// Everything else only happens once we have created a session
|
||||
if (isFulfilledSessionCreatedAction(action)) {
|
||||
const oldSessionId = getState().system.sessionId;
|
||||
|
||||
// temp disable event subscription
|
||||
const shouldHandleEvent = (id: string): boolean => true;
|
||||
|
||||
// const subscribedNodeIds = getState().system.subscribedNodeIds;
|
||||
// const shouldHandleEvent = (id: string): boolean => {
|
||||
// if (subscribedNodeIds.length === 1 && subscribedNodeIds[0] === '*') {
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// return subscribedNodeIds.includes(id);
|
||||
// };
|
||||
|
||||
if (oldSessionId) {
|
||||
// Unsubscribe when invocations complete
|
||||
socket.emit('unsubscribe', {
|
||||
session: oldSessionId,
|
||||
});
|
||||
|
||||
dispatch(
|
||||
socketUnsubscribed({
|
||||
sessionId: oldSessionId,
|
||||
timestamp: getTimestamp(),
|
||||
})
|
||||
);
|
||||
|
||||
const listenersToRemove: (keyof ServerToClientEvents)[] = [
|
||||
'invocation_started',
|
||||
'generator_progress',
|
||||
'invocation_error',
|
||||
'invocation_complete',
|
||||
];
|
||||
|
||||
// Remove listeners for these events; we need to set them up fresh whenever we subscribe
|
||||
listenersToRemove.forEach((event: keyof ServerToClientEvents) => {
|
||||
socket.removeAllListeners(event);
|
||||
});
|
||||
}
|
||||
|
||||
const sessionId = action.payload.id;
|
||||
|
||||
// After a session is created, we immediately subscribe to events and then invoke the session
|
||||
socket.emit('subscribe', { session: sessionId });
|
||||
|
||||
// Always dispatch the event actions for other consumers who want to know when we subscribed
|
||||
dispatch(
|
||||
socketSubscribed({
|
||||
sessionId,
|
||||
timestamp: getTimestamp(),
|
||||
})
|
||||
);
|
||||
|
||||
// Set up listeners for the present subscription
|
||||
socket.on('invocation_started', (data) => {
|
||||
if (shouldHandleEvent(data.node.id)) {
|
||||
dispatch(invocationStarted({ data, timestamp: getTimestamp() }));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('generator_progress', (data) => {
|
||||
if (shouldHandleEvent(data.node.id)) {
|
||||
dispatch(generatorProgress({ data, timestamp: getTimestamp() }));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('invocation_error', (data) => {
|
||||
if (shouldHandleEvent(data.node.id)) {
|
||||
dispatch(invocationError({ data, timestamp: getTimestamp() }));
|
||||
}
|
||||
});
|
||||
|
||||
socket.on('invocation_complete', (data) => {
|
||||
if (shouldHandleEvent(data.node.id)) {
|
||||
const sessionId = data.graph_execution_state_id;
|
||||
|
||||
const { cancelType, isCancelScheduled } = getState().system;
|
||||
|
||||
// Handle scheduled cancelation
|
||||
if (cancelType === 'scheduled' && isCancelScheduled) {
|
||||
dispatch(sessionCanceled({ sessionId }));
|
||||
}
|
||||
|
||||
dispatch(invocationComplete({ data, timestamp: getTimestamp() }));
|
||||
}
|
||||
});
|
||||
|
||||
// Finally we actually invoke the session, starting processing
|
||||
dispatch(sessionInvoked({ sessionId }));
|
||||
}
|
||||
|
||||
// Always pass the action on so other middleware and reducers can handle it
|
||||
next(action);
|
||||
};
|
||||
|
||||
return middleware;
|
||||
};
|
||||
109
invokeai/frontend/web/src/services/events/types.ts
Normal file
109
invokeai/frontend/web/src/services/events/types.ts
Normal file
@ -0,0 +1,109 @@
|
||||
import { Graph, GraphExecutionState } from '../api';
|
||||
|
||||
/**
|
||||
* A progress image, we get one for each step in the generation
|
||||
*/
|
||||
export type ProgressImage = {
|
||||
dataURL: string;
|
||||
width: number;
|
||||
height: number;
|
||||
};
|
||||
|
||||
export type AnyInvocationType = NonNullable<
|
||||
NonNullable<Graph['nodes']>[string]['type']
|
||||
>;
|
||||
|
||||
export type AnyInvocation = NonNullable<Graph['nodes']>[string];
|
||||
|
||||
// export type AnyInvocation = {
|
||||
// id: string;
|
||||
// type: AnyInvocationType | string;
|
||||
// [key: string]: any;
|
||||
// };
|
||||
|
||||
export type AnyResult = GraphExecutionState['results'][string];
|
||||
|
||||
/**
|
||||
* A `generator_progress` socket.io event.
|
||||
*
|
||||
* @example socket.on('generator_progress', (data: GeneratorProgressEvent) => { ... }
|
||||
*/
|
||||
export type GeneratorProgressEvent = {
|
||||
graph_execution_state_id: string;
|
||||
node: AnyInvocation;
|
||||
source_node_id: string;
|
||||
progress_image?: ProgressImage;
|
||||
step: number;
|
||||
total_steps: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* A `invocation_complete` socket.io event.
|
||||
*
|
||||
* `result` is a discriminated union with a `type` property as the discriminant.
|
||||
*
|
||||
* @example socket.on('invocation_complete', (data: InvocationCompleteEvent) => { ... }
|
||||
*/
|
||||
export type InvocationCompleteEvent = {
|
||||
graph_execution_state_id: string;
|
||||
node: AnyInvocation;
|
||||
source_node_id: string;
|
||||
result: AnyResult;
|
||||
};
|
||||
|
||||
/**
|
||||
* A `invocation_error` socket.io event.
|
||||
*
|
||||
* @example socket.on('invocation_error', (data: InvocationErrorEvent) => { ... }
|
||||
*/
|
||||
export type InvocationErrorEvent = {
|
||||
graph_execution_state_id: string;
|
||||
node: AnyInvocation;
|
||||
source_node_id: string;
|
||||
error: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A `invocation_started` socket.io event.
|
||||
*
|
||||
* @example socket.on('invocation_started', (data: InvocationStartedEvent) => { ... }
|
||||
*/
|
||||
export type InvocationStartedEvent = {
|
||||
graph_execution_state_id: string;
|
||||
node: AnyInvocation;
|
||||
source_node_id: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* A `graph_execution_state_complete` socket.io event.
|
||||
*
|
||||
* @example socket.on('graph_execution_state_complete', (data: GraphExecutionStateCompleteEvent) => { ... }
|
||||
*/
|
||||
export type GraphExecutionStateCompleteEvent = {
|
||||
graph_execution_state_id: string;
|
||||
};
|
||||
|
||||
export type ClientEmitSubscribe = {
|
||||
session: string;
|
||||
};
|
||||
|
||||
export type ClientEmitUnsubscribe = {
|
||||
session: string;
|
||||
};
|
||||
|
||||
export type ServerToClientEvents = {
|
||||
generator_progress: (payload: GeneratorProgressEvent) => void;
|
||||
invocation_complete: (payload: InvocationCompleteEvent) => void;
|
||||
invocation_error: (payload: InvocationErrorEvent) => void;
|
||||
invocation_started: (payload: InvocationStartedEvent) => void;
|
||||
graph_execution_state_complete: (
|
||||
payload: GraphExecutionStateCompleteEvent
|
||||
) => void;
|
||||
};
|
||||
|
||||
export type ClientToServerEvents = {
|
||||
connect: () => void;
|
||||
disconnect: () => void;
|
||||
subscribe: (payload: ClientEmitSubscribe) => void;
|
||||
unsubscribe: (payload: ClientEmitUnsubscribe) => void;
|
||||
};
|
||||
Reference in New Issue
Block a user