Merge remote-tracking branch 'origin/main' into maryhipp/indexdb-per-project

IndexedDB has a much larger storage limit than LocalStorage, and is widely supported.

Implemented as a custom storage driver for `redux-remember` via `idb-keyval`. `idb-keyval` is a simple wrapper for IndexedDB that allows it to be used easily as a key-value store.

The logic to clear persisted storage has been updated throughout the app.

invokeai/frontend/web/package.json

@@ -75,6 +75,7 @@
     "framer-motion": "^10.16.4",
     "i18next": "^23.6.0",
     "i18next-http-backend": "^2.3.1",
+    "idb-keyval": "^6.2.1",
     "konva": "^9.2.3",
     "lodash-es": "^4.17.21",
     "nanostores": "^0.9.4",

invokeai/frontend/web/src/app/components/App.tsx

@@ -1,11 +1,13 @@
 import { Flex, Grid } from '@chakra-ui/react';
 import { useStore } from '@nanostores/react';
+import { useSocketIO } from 'app/hooks/useSocketIO';
 import { useLogger } from 'app/logging/useLogger';
 import { appStarted } from 'app/store/middleware/listenerMiddleware/listeners/appStarted';
 import { $headerComponent } from 'app/store/nanostores/headerComponent';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import { PartialAppConfig } from 'app/types/invokeai';
 import ImageUploader from 'common/components/ImageUploader';
+import { useClearStorage } from 'common/hooks/useClearStorage';
 import ChangeBoardModal from 'features/changeBoardModal/components/ChangeBoardModal';
 import DeleteImageModal from 'features/deleteImageModal/components/DeleteImageModal';
 import SiteHeader from 'features/system/components/SiteHeader';
@@ -20,7 +22,6 @@ import AppErrorBoundaryFallback from './AppErrorBoundaryFallback';
 import GlobalHotkeys from './GlobalHotkeys';
 import PreselectedImage from './PreselectedImage';
 import Toaster from './Toaster';
-import { useSocketIO } from 'app/hooks/useSocketIO';
 
 const DEFAULT_CONFIG = {};
 
@@ -36,15 +37,16 @@ const App = ({ config = DEFAULT_CONFIG, selectedImage }: Props) => {
   const language = useAppSelector(languageSelector);
   const logger = useLogger('system');
   const dispatch = useAppDispatch();
+  const clearStorage = useClearStorage();
 
   // singleton!
   useSocketIO();
 
   const handleReset = useCallback(() => {
-    localStorage.clear();
+    clearStorage();
     location.reload();
     return false;
-  }, []);
+  }, [clearStorage]);
 
   useEffect(() => {
     i18n.changeLanguage(language);

invokeai/frontend/web/src/app/components/InvokeAIUI.tsx

@@ -7,14 +7,16 @@ import { $headerComponent } from 'app/store/nanostores/headerComponent';
 import { $isDebugging } from 'app/store/nanostores/isDebugging';
 import { $projectId } from 'app/store/nanostores/projectId';
 import { $queueId, DEFAULT_QUEUE_ID } from 'app/store/nanostores/queueId';
-import { store } from 'app/store/store';
+import { createStore } from 'app/store/store';
 import { PartialAppConfig } from 'app/types/invokeai';
+import 'i18n';
 import React, {
   PropsWithChildren,
   ReactNode,
   lazy,
   memo,
   useEffect,
+  useMemo,
 } from 'react';
 import { Provider } from 'react-redux';
 import { addMiddleware, resetMiddlewares } from 'redux-dynamic-middlewares';
@@ -22,6 +24,7 @@ import { ManagerOptions, SocketOptions } from 'socket.io-client';
 import Loading from 'common/components/Loading/Loading';
 import AppDndContext from 'features/dnd/components/AppDndContext';
 import 'i18n';
+import { $store } from 'app/store/nanostores/store';
 
 const App = lazy(() => import('./App'));
 const ThemeLocaleProvider = lazy(() => import('./ThemeLocaleProvider'));
@@ -119,6 +122,14 @@ const InvokeAIUI = ({
     };
   }, [headerComponent]);
 
+  const store = useMemo(() => {
+    return createStore(projectId);
+  }, [projectId]);
+
+  useEffect(() => {
+    $store.set(store);
+  }, [store]);
+
   useEffect(() => {
     if (socketOptions) {
       $socketOptions.set(socketOptions);

invokeai/frontend/web/src/app/components/ThemeLocaleProvider.tsx

@@ -9,9 +9,9 @@ import { TOAST_OPTIONS, theme as invokeAITheme } from 'theme/theme';
 
 import '@fontsource-variable/inter';
 import { MantineProvider } from '@mantine/core';
+import { useMantineTheme } from 'mantine-theme/theme';
 import 'overlayscrollbars/overlayscrollbars.css';
 import 'theme/css/overlayscrollbars.css';
-import { useMantineTheme } from 'mantine-theme/theme';
 
 type ThemeLocaleProviderProps = {
   children: ReactNode;

invokeai/frontend/web/src/app/store/constants.ts

@@ -1,8 +1 @@
-export const LOCALSTORAGE_KEYS = [
-  'chakra-ui-color-mode',
-  'i18nextLng',
-  'ROARR_FILTER',
-  'ROARR_LOG',
-];
-
-export const LOCALSTORAGE_PREFIX = '@@invokeai-';
+export const STORAGE_PREFIX = '@@invokeai-';

invokeai/frontend/web/src/app/store/store.ts

@@ -23,16 +23,16 @@ import systemReducer from 'features/system/store/systemSlice';
 import hotkeysReducer from 'features/ui/store/hotkeysSlice';
 import uiReducer from 'features/ui/store/uiSlice';
 import dynamicMiddlewares from 'redux-dynamic-middlewares';
-import { rememberEnhancer, rememberReducer } from 'redux-remember';
+import { Driver, rememberEnhancer, rememberReducer } from 'redux-remember';
 import { api } from 'services/api';
-import { LOCALSTORAGE_PREFIX } from './constants';
+import { STORAGE_PREFIX } from './constants';
 import { serialize } from './enhancers/reduxRemember/serialize';
 import { unserialize } from './enhancers/reduxRemember/unserialize';
 import { actionSanitizer } from './middleware/devtools/actionSanitizer';
 import { actionsDenylist } from './middleware/devtools/actionsDenylist';
 import { stateSanitizer } from './middleware/devtools/stateSanitizer';
 import { listenerMiddleware } from './middleware/listenerMiddleware';
-import { $store } from './nanostores/store';
+import { createStore as createIDBKeyValStore, get, set } from 'idb-keyval';
 
 const allReducers = {
   canvas: canvasReducer,
@@ -74,57 +74,71 @@ const rememberedKeys: (keyof typeof allReducers)[] = [
   'modelmanager',
 ];
 
-export const store = configureStore({
-  reducer: rememberedRootReducer,
-  enhancers: (existingEnhancers) => {
-    return existingEnhancers
-      .concat(
-        rememberEnhancer(window.localStorage, rememberedKeys, {
-          persistDebounce: 300,
-          serialize,
-          unserialize,
-          prefix: LOCALSTORAGE_PREFIX,
-        })
-      )
-      .concat(autoBatchEnhancer());
-  },
-  middleware: (getDefaultMiddleware) =>
-    getDefaultMiddleware({
-      serializableCheck: false,
-      immutableCheck: false,
-    })
-      .concat(api.middleware)
-      .concat(dynamicMiddlewares)
-      .prepend(listenerMiddleware.middleware),
-  devTools: {
-    actionSanitizer,
-    stateSanitizer,
-    trace: true,
-    predicate: (state, action) => {
-      // TODO: hook up to the log level param in system slice
-      // manually type state, cannot type the arg
-      // const typedState = state as ReturnType<typeof rootReducer>;
+// Create a custom idb-keyval store (just needed to customize the name)
+export const idbKeyValStore = createIDBKeyValStore('invoke', 'invoke-store');
 
-      // TODO: doing this breaks the rtk query devtools, commenting out for now
-      // if (action.type.startsWith('api/')) {
-      //   // don't log api actions, with manual cache updates they are extremely noisy
-      //   return false;
-      // }
+// Create redux-remember driver, wrapping idb-keyval
+const idbKeyValDriver: Driver = {
+  getItem: (key) => get(key, idbKeyValStore),
+  setItem: (key, value) => set(key, value, idbKeyValStore),
+};
 
-      if (actionsDenylist.includes(action.type)) {
-        // don't log other noisy actions
-        return false;
-      }
-
-      return true;
+export const createStore = (projectId?: string) =>
+  configureStore({
+    reducer: rememberedRootReducer,
+    enhancers: (existingEnhancers) => {
+      return existingEnhancers
+        .concat(
+          rememberEnhancer(idbKeyValDriver, rememberedKeys, {
+            persistDebounce: 300,
+            serialize,
+            unserialize,
+            prefix: projectId
+              ? `${STORAGE_PREFIX}${projectId}-`
+              : STORAGE_PREFIX,
+          })
+        )
+        .concat(autoBatchEnhancer());
     },
-  },
-});
+    middleware: (getDefaultMiddleware) =>
+      getDefaultMiddleware({
+        serializableCheck: false,
+        immutableCheck: false,
+      })
+        .concat(api.middleware)
+        .concat(dynamicMiddlewares)
+        .prepend(listenerMiddleware.middleware),
+    devTools: {
+      actionSanitizer,
+      stateSanitizer,
+      trace: true,
+      predicate: (state, action) => {
+        // TODO: hook up to the log level param in system slice
+        // manually type state, cannot type the arg
+        // const typedState = state as ReturnType<typeof rootReducer>;
 
-export type AppGetState = typeof store.getState;
-export type RootState = ReturnType<typeof store.getState>;
+        // TODO: doing this breaks the rtk query devtools, commenting out for now
+        // if (action.type.startsWith('api/')) {
+        //   // don't log api actions, with manual cache updates they are extremely noisy
+        //   return false;
+        // }
+
+        if (actionsDenylist.includes(action.type)) {
+          // don't log other noisy actions
+          return false;
+        }
+
+        return true;
+      },
+    },
+  });
+
+export type AppGetState = ReturnType<
+  ReturnType<typeof createStore>['getState']
+>;
+export type RootState = ReturnType<ReturnType<typeof createStore>['getState']>;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export type AppThunkDispatch = ThunkDispatch<RootState, any, AnyAction>;
-export type AppDispatch = typeof store.dispatch;
+export type AppDispatch = ReturnType<typeof createStore>['dispatch'];
+
 export const stateSelector = (state: RootState) => state;
-$store.set(store);

invokeai/frontend/web/src/common/hooks/useClearStorage.ts

@@ -0,0 +1,12 @@
+import { idbKeyValStore } from 'app/store/store';
+import { clear } from 'idb-keyval';
+import { useCallback } from 'react';
+
+export const useClearStorage = () => {
+  const clearStorage = useCallback(() => {
+    clear(idbKeyValStore);
+    localStorage.clear();
+  }, []);
+
+  return clearStorage;
+};

invokeai/frontend/web/src/features/system/components/SettingsModal/SettingsModal.tsx

@@ -14,11 +14,11 @@ import {
 } from '@chakra-ui/react';
 import { createSelector } from '@reduxjs/toolkit';
 import { VALID_LOG_LEVELS } from 'app/logging/logger';
-import { LOCALSTORAGE_KEYS, LOCALSTORAGE_PREFIX } from 'app/store/constants';
 import { stateSelector } from 'app/store/store';
 import { useAppDispatch, useAppSelector } from 'app/store/storeHooks';
 import IAIButton from 'common/components/IAIButton';
 import IAIMantineSelect from 'common/components/IAIMantineSelect';
+import { useClearStorage } from 'common/hooks/useClearStorage';
 import {
   consoleLogLevelChanged,
   setEnableImageDebugging,
@@ -164,20 +164,14 @@ const SettingsModal = ({ children, config }: SettingsModalProps) => {
     shouldEnableInformationalPopovers,
   } = useAppSelector(selector);
 
+  const clearStorage = useClearStorage();
+
   const handleClickResetWebUI = useCallback(() => {
-    // Only remove our keys
-    Object.keys(window.localStorage).forEach((key) => {
-      if (
-        LOCALSTORAGE_KEYS.includes(key) ||
-        key.startsWith(LOCALSTORAGE_PREFIX)
-      ) {
-        localStorage.removeItem(key);
-      }
-    });
+    clearStorage();
     onSettingsModalClose();
     onRefreshModalOpen();
     setInterval(() => setCountdown((prev) => prev - 1), 1000);
-  }, [onSettingsModalClose, onRefreshModalOpen]);
+  }, [clearStorage, onSettingsModalClose, onRefreshModalOpen]);
 
   useEffect(() => {
     if (countdown <= 0) {

invokeai/frontend/web/src/main.tsx

@@ -2,6 +2,9 @@ import ReactDOM from 'react-dom/client';
 
 import InvokeAIUI from './app/components/InvokeAIUI';
 
+const urlParams = new URLSearchParams(window.location.search);
+const projectId = urlParams.get('projectId') || '';
+
 ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
-  <InvokeAIUI />
+  <InvokeAIUI token="INSERT_TOKEN_HERE" projectId={projectId} />
 );

invokeai/frontend/web/yarn.lock

@@ -4158,6 +4158,11 @@ i18next@^23.6.0:
   dependencies:
     "@babel/runtime" "^7.22.5"
 
+idb-keyval@^6.2.1:
+  version "6.2.1"
+  resolved "https://registry.yarnpkg.com/idb-keyval/-/idb-keyval-6.2.1.tgz#94516d625346d16f56f3b33855da11bfded2db33"
+  integrity sha512-8Sb3veuYCyrZL+VBt9LJfZjLUPWVvqn8tG28VqYNFCo43KHcKuq+b4EiXGeuaLAQWL2YmyDgMp2aSpH9JHsEQg==
+
 ieee754@^1.1.13:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"