Merge branch 'release-candidate-2' into release-candidate-2

- This includes #949 "Bug fixes for new Threshold and Perlin Options"

backend/invoke_ai_web_server.py

@@ -53,17 +53,14 @@ class InvokeAIWebServer:
         cors_allowed_origins = [
             'http://127.0.0.1:5173',
             'http://localhost:5173',
+            'http://localhost:9090'
         ]
         additional_allowed_origins = (
             opt.cors if opt.cors else []
         )  # additional CORS allowed origins
-        if self.host == '127.0.0.1':
+        cors_allowed_origins.append(f'http://{self.host}:{self.port}')
-            cors_allowed_origins.extend(
+        if self.host == '127.0.0.1' or self.host == '0.0.0.0':
-                [
+            cors_allowed_origins.append(f'http://localhost:{self.port}')
-                    f'http://{self.host}:{self.port}',
-                    f'http://localhost:{self.port}',
-                ]
-            )
         cors_allowed_origins = (
             cors_allowed_origins + additional_allowed_origins
         )

frontend/package.json

@@ -23,6 +23,7 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-dropzone": "^14.2.2",
+    "react-hotkeys-hook": "^3.4.7",
     "react-icons": "^4.4.0",
     "react-redux": "^8.0.2",
     "redux-persist": "^6.0.0",

frontend/src/features/gallery/CurrentImageButtons.tsx

@@ -19,6 +19,8 @@ import { MdDelete, MdFace, MdHd, MdImage, MdInfo } from 'react-icons/md';
 import InvokePopover from './InvokePopover';
 import UpscaleOptions from '../options/AdvancedOptions/Upscale/UpscaleOptions';
 import FaceRestoreOptions from '../options/AdvancedOptions/FaceRestore/FaceRestoreOptions';
+import { useHotkeys } from 'react-hotkeys-hook';
+import { useToast } from '@chakra-ui/react';
 
 const systemSelector = createSelector(
   (state: RootState) => state.system,
@@ -54,6 +56,8 @@ const CurrentImageButtons = ({
 }: CurrentImageButtonsProps) => {
   const dispatch = useAppDispatch();
 
+  const toast = useToast();
+
   const intermediateImage = useAppSelector(
     (state: RootState) => state.gallery.intermediateImage
   );
@@ -71,19 +75,163 @@ const CurrentImageButtons = ({
 
   const handleClickUseAsInitialImage = () =>
     dispatch(setInitialImagePath(image.url));
+  useHotkeys(
+    'shift+i',
+    () => {
+      if (image) {
+        handleClickUseAsInitialImage();
+        toast({
+          title: 'Sent To Image To Image',
+          status: 'success',
+          duration: 2500,
+          isClosable: true,
+        });
+      } else {
+        toast({
+          title: 'No Image Loaded',
+          description: 'No image found to send to image to image module.',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [image]
+  );
 
   const handleClickUseAllParameters = () =>
     dispatch(setAllParameters(image.metadata));
+  useHotkeys(
+    'a',
+    () => {
+      if (['txt2img', 'img2img'].includes(image?.metadata?.image?.type)) {
+        handleClickUseAllParameters();
+        toast({
+          title: 'Parameters Set',
+          status: 'success',
+          duration: 2500,
+          isClosable: true,
+        });
+      } else {
+        toast({
+          title: 'Parameters Not Set',
+          description: 'No metadata found for this image.',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [image]
+  );
 
   // Non-null assertion: this button is disabled if there is no seed.
   // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
   const handleClickUseSeed = () => dispatch(setSeed(image.metadata.image.seed));
+  useHotkeys(
+    's',
+    () => {
+      if (image?.metadata?.image?.seed) {
+        handleClickUseSeed();
+        toast({
+          title: 'Seed Set',
+          status: 'success',
+          duration: 2500,
+          isClosable: true,
+        });
+      } else {
+        toast({
+          title: 'Seed Not Set',
+          description: 'Could not find seed for this image.',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [image]
+  );
+
   const handleClickUpscale = () => dispatch(runESRGAN(image));
+  useHotkeys(
+    'u',
+    () => {
+      if (
+        isESRGANAvailable &&
+        Boolean(!intermediateImage) &&
+        isConnected &&
+        !isProcessing &&
+        upscalingLevel
+      ) {
+        handleClickUpscale();
+      } else {
+        toast({
+          title: 'Upscaling Failed',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [
+      image,
+      isESRGANAvailable,
+      intermediateImage,
+      isConnected,
+      isProcessing,
+      upscalingLevel,
+    ]
+  );
 
   const handleClickFixFaces = () => dispatch(runGFPGAN(image));
+  useHotkeys(
+    'r',
+    () => {
+      if (
+        isGFPGANAvailable &&
+        Boolean(!intermediateImage) &&
+        isConnected &&
+        !isProcessing &&
+        gfpganStrength
+      ) {
+        handleClickFixFaces();
+      } else {
+        toast({
+          title: 'Face Restoration Failed',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [
+      image,
+      isGFPGANAvailable,
+      intermediateImage,
+      isConnected,
+      isProcessing,
+      gfpganStrength,
+    ]
+  );
 
   const handleClickShowImageDetails = () =>
     setShouldShowImageDetails(!shouldShowImageDetails);
+  useHotkeys(
+    'i',
+    () => {
+      if (image) {
+        handleClickShowImageDetails();
+      } else {
+        toast({
+          title: 'Failed to load metadata',
+          status: 'error',
+          duration: 2500,
+          isClosable: true,
+        });
+      }
+    },
+    [image, shouldShowImageDetails]
+  );
 
   return (
     <div className="current-image-options">

frontend/src/features/gallery/CurrentImageDisplay.scss

@@ -67,6 +67,44 @@
   }
 }
 
+.current-image-next-prev-buttons {
+  position: absolute;
+  top: 0;
+  left: 0;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  width: calc(100% - 2rem);
+  padding: 0.5rem;
+  margin-left: 1rem;
+  z-index: 1;
+  height: calc($app-metadata-height - 1rem);
+  pointer-events: none;
+}
+
+.next-prev-button-trigger-area {
+  width: 7rem;
+  height: 100%;
+  display: flex;
+  align-items: center;
+  pointer-events: auto;
+
+  &.prev-button-trigger-area {
+    justify-content: flex-start;
+  }
+
+  &.next-button-trigger-area {
+    justify-content: flex-end;
+  }
+}
+
+.next-prev-button {
+  font-size: 5rem;
+  fill: var(--text-color-secondary);
+  filter: drop-shadow(0 0 1rem var(--text-color-secondary));
+  opacity: 70%;
+}
+
 .current-image-metadata-viewer {
   border-radius: 0.5rem;
   position: absolute;

frontend/src/features/gallery/CurrentImageDisplay.tsx

@@ -1,15 +1,21 @@
-import { Image } from '@chakra-ui/react';
+import { IconButton, Image } from '@chakra-ui/react';
-import { useAppSelector } from '../../app/store';
+import { useAppDispatch, useAppSelector } from '../../app/store';
 import { RootState } from '../../app/store';
 import { useState } from 'react';
 import ImageMetadataViewer from './ImageMetadataViewer';
 import CurrentImageButtons from './CurrentImageButtons';
 import { MdPhoto } from 'react-icons/md';
+import { FaAngleLeft, FaAngleRight } from 'react-icons/fa';
+import { selectNextImage, selectPrevImage } from './gallerySlice';
 
 /**
  * Displays the current image if there is one, plus associated actions.
  */
 const CurrentImageDisplay = () => {
+  const dispatch = useAppDispatch();
+  const [shouldShowNextPrevButtons, setShouldShowNextPrevButtons] =
+    useState<boolean>(false);
+
   const { currentImage, intermediateImage } = useAppSelector(
     (state: RootState) => state.gallery
   );
@@ -19,6 +25,22 @@ const CurrentImageDisplay = () => {
 
   const imageToDisplay = intermediateImage || currentImage;
 
+  const handleCurrentImagePreviewMouseOver = () => {
+    setShouldShowNextPrevButtons(true);
+  };
+
+  const handleCurrentImagePreviewMouseOut = () => {
+    setShouldShowNextPrevButtons(false);
+  };
+
+  const handleClickPrevButton = () => {
+    dispatch(selectPrevImage());
+  };
+
+  const handleClickNextButton = () => {
+    dispatch(selectNextImage());
+  };
+
   return imageToDisplay ? (
     <div className="current-image-display">
       <div className="current-image-tools">
@@ -40,6 +62,38 @@ const CurrentImageDisplay = () => {
             <ImageMetadataViewer image={imageToDisplay} />
           </div>
         )}
+        {!shouldShowImageDetails && (
+          <div className="current-image-next-prev-buttons">
+            <div
+              className="next-prev-button-trigger-area prev-button-trigger-area"
+              onMouseOver={handleCurrentImagePreviewMouseOver}
+              onMouseOut={handleCurrentImagePreviewMouseOut}
+            >
+              {shouldShowNextPrevButtons && (
+                <IconButton
+                  aria-label="Previous image"
+                  icon={<FaAngleLeft className="next-prev-button" />}
+                  variant="unstyled"
+                  onClick={handleClickPrevButton}
+                />
+              )}
+            </div>
+            <div
+              className="next-prev-button-trigger-area next-button-trigger-area"
+              onMouseOver={handleCurrentImagePreviewMouseOver}
+              onMouseOut={handleCurrentImagePreviewMouseOut}
+            >
+              {shouldShowNextPrevButtons && (
+                <IconButton
+                  aria-label="Next image"
+                  icon={<FaAngleRight className="next-prev-button" />}
+                  variant="unstyled"
+                  onClick={handleClickNextButton}
+                />
+              )}
+            </div>
+          </div>
+        )}
       </div>
     </div>
   ) : (

frontend/src/features/gallery/DeleteImageModal.tsx

@@ -27,6 +27,7 @@ import { deleteImage } from '../../app/socketio/actions';
 import { RootState } from '../../app/store';
 import { setShouldConfirmOnDelete, SystemState } from '../system/systemSlice';
 import * as InvokeAI from '../../app/invokeai';
+import { useHotkeys } from 'react-hotkeys-hook';
 
 interface DeleteImageModalProps {
   /**
@@ -67,6 +68,14 @@ const DeleteImageModal = forwardRef(
       onClose();
     };
 
+    useHotkeys(
+      'del',
+      () => {
+        shouldConfirmOnDelete ? onOpen() : handleDelete();
+      },
+      [image, shouldConfirmOnDelete]
+    );
+
     const handleChangeShouldConfirmOnDelete = (
       e: ChangeEvent<HTMLInputElement>
     ) => dispatch(setShouldConfirmOnDelete(!e.target.checked));

frontend/src/features/gallery/ImageGallery.tsx

@@ -1,8 +1,10 @@
 import { Button } from '@chakra-ui/react';
+import { useHotkeys } from 'react-hotkeys-hook';
 import { MdPhotoLibrary } from 'react-icons/md';
 import { requestImages } from '../../app/socketio/actions';
 import { RootState, useAppDispatch } from '../../app/store';
 import { useAppSelector } from '../../app/store';
+import { selectNextImage, selectPrevImage } from './gallerySlice';
 import HoverableImage from './HoverableImage';
 
 /**
@@ -25,6 +27,22 @@ const ImageGallery = () => {
     dispatch(requestImages());
   };
 
+  useHotkeys(
+    'left',
+    () => {
+      dispatch(selectPrevImage());
+    },
+    []
+  );
+
+  useHotkeys(
+    'right',
+    () => {
+      dispatch(selectNextImage());
+    },
+    []
+  );
+
   return (
     <div className="image-gallery-container">
       {images.length ? (

frontend/src/features/gallery/gallerySlice.ts

@@ -1,6 +1,6 @@
 import { createSlice } from '@reduxjs/toolkit';
 import type { PayloadAction } from '@reduxjs/toolkit';
-import { clamp } from 'lodash';
+import _, { clamp } from 'lodash';
 import * as InvokeAI from '../../app/invokeai';
 
 export interface GalleryState {
@@ -85,6 +85,32 @@ export const gallerySlice = createSlice({
     clearIntermediateImage: (state) => {
       state.intermediateImage = undefined;
     },
+    selectNextImage: (state) => {
+      const { images, currentImage } = state;
+      if (currentImage) {
+        const currentImageIndex = images.findIndex(
+          (i) => i.uuid === currentImage.uuid
+        );
+        if (_.inRange(currentImageIndex, 0, images.length)) {
+          const newCurrentImage = images[currentImageIndex + 1];
+          state.currentImage = newCurrentImage;
+          state.currentImageUuid = newCurrentImage.uuid;
+        }
+      }
+    },
+    selectPrevImage: (state) => {
+      const { images, currentImage } = state;
+      if (currentImage) {
+        const currentImageIndex = images.findIndex(
+          (i) => i.uuid === currentImage.uuid
+        );
+        if (_.inRange(currentImageIndex, 1, images.length + 1)) {
+          const newCurrentImage = images[currentImageIndex - 1];
+          state.currentImage = newCurrentImage;
+          state.currentImageUuid = newCurrentImage.uuid;
+        }
+      }
+    },
     addGalleryImages: (
       state,
       action: PayloadAction<{
@@ -122,6 +148,8 @@ export const {
   setCurrentImage,
   addGalleryImages,
   setIntermediateImage,
+  selectNextImage,
+  selectPrevImage,
 } = gallerySlice.actions;
 
 export default gallerySlice.reducer;

frontend/src/features/options/ProcessButtons/CancelButton.tsx

@@ -4,12 +4,23 @@ import { cancelProcessing } from '../../../app/socketio/actions';
 import { useAppDispatch, useAppSelector } from '../../../app/store';
 import IAIIconButton from '../../../common/components/IAIIconButton';
 import { systemSelector } from '../../../common/hooks/useCheckParameters';
+import { useHotkeys } from 'react-hotkeys-hook';
 
 export default function CancelButton() {
   const dispatch = useAppDispatch();
   const { isProcessing, isConnected } = useAppSelector(systemSelector);
   const handleClickCancel = () => dispatch(cancelProcessing());
 
+  useHotkeys(
+    'shift+x',
+    () => {
+      if (isConnected || isProcessing) {
+        handleClickCancel();
+      }
+    },
+    [isConnected, isProcessing]
+  );
+
   return (
     <IAIIconButton
       icon={<MdCancel />}

frontend/src/features/options/PromptInput/PromptInput.tsx

@@ -1,5 +1,5 @@
 import { FormControl, Textarea } from '@chakra-ui/react';
-import { ChangeEvent, KeyboardEvent } from 'react';
+import { ChangeEvent, KeyboardEvent, useRef } from 'react';
 import { RootState, useAppDispatch, useAppSelector } from '../../../app/store';
 import { generateImage } from '../../../app/socketio/actions';
 
@@ -9,6 +9,7 @@ import { isEqual } from 'lodash';
 import useCheckParameters, {
   systemSelector,
 } from '../../../common/hooks/useCheckParameters';
+import { useHotkeys } from 'react-hotkeys-hook';
 
 export const optionsSelector = createSelector(
   (state: RootState) => state.options,
@@ -28,6 +29,7 @@ export const optionsSelector = createSelector(
  * Prompt input text area.
  */
 const PromptInput = () => {
+  const promptRef = useRef<HTMLTextAreaElement>(null);
   const { prompt } = useAppSelector(optionsSelector);
   const { isProcessing } = useAppSelector(systemSelector);
   const dispatch = useAppDispatch();
@@ -37,6 +39,24 @@ const PromptInput = () => {
     dispatch(setPrompt(e.target.value));
   };
 
+  useHotkeys(
+    'ctrl+enter',
+    () => {
+      if (isReady) {
+        dispatch(generateImage());
+      }
+    },
+    [isReady]
+  );
+
+  useHotkeys(
+    'alt+a',
+    () => {
+      promptRef.current?.focus();
+    },
+    []
+  );
+
   const handleKeyDown = (e: KeyboardEvent<HTMLTextAreaElement>) => {
     if (e.key === 'Enter' && e.shiftKey === false && isReady) {
       e.preventDefault();
@@ -60,6 +80,7 @@ const PromptInput = () => {
           onKeyDown={handleKeyDown}
           resize="vertical"
           height={30}
+          ref={promptRef}
         />
       </FormControl>
     </div>

frontend/src/features/system/HotkeysModal/HotkeysModal.scss

@@ -0,0 +1,53 @@
+@use '../../../styles/Mixins/' as *;
+
+.hotkeys-modal {
+  display: grid;
+  padding: 1rem;
+  background-color: var(--settings-modal-bg) !important;
+  row-gap: 1rem;
+  font-family: Inter;
+
+  h1 {
+    font-size: 1.2rem;
+    font-weight: bold;
+  }
+}
+
+.hotkeys-modal-items {
+  display: grid;
+  row-gap: 0.5rem;
+  max-height: 32rem;
+  overflow-y: scroll;
+  @include HideScrollbar;
+}
+
+.hotkey-modal-item {
+  display: grid;
+  grid-template-columns: auto max-content;
+  justify-content: space-between;
+  align-items: center;
+  background-color: var(--background-color);
+  padding: 0.5rem 1rem;
+  border-radius: 0.3rem;
+
+  .hotkey-info {
+    display: grid;
+
+    .hotkey-title {
+      font-weight: bold;
+    }
+
+    .hotkey-description {
+      font-size: 0.9rem;
+      color: var(--text-color-secondary);
+    }
+  }
+
+  .hotkey-key {
+    font-size: 0.8rem;
+    font-weight: bold;
+    border: 2px solid var(--settings-modal-bg);
+    padding: 0.2rem 0.5rem;
+    border-radius: 0.3rem;
+  }
+}

frontend/src/features/system/HotkeysModal/HotkeysModal.tsx

@@ -0,0 +1,98 @@
+import {
+  Modal,
+  ModalCloseButton,
+  ModalContent,
+  ModalOverlay,
+  useDisclosure,
+} from '@chakra-ui/react';
+import React, { cloneElement, ReactElement } from 'react';
+import HotkeysModalItem from './HotkeysModalItem';
+
+type HotkeysModalProps = {
+  /* The button to open the Settings Modal */
+  children: ReactElement;
+};
+
+export default function HotkeysModal({ children }: HotkeysModalProps) {
+  const {
+    isOpen: isHotkeyModalOpen,
+    onOpen: onHotkeysModalOpen,
+    onClose: onHotkeysModalClose,
+  } = useDisclosure();
+
+  const hotkeys = [
+    { title: 'Invoke', desc: 'Generate an image', hotkey: 'Ctrl+Enter' },
+    { title: 'Cancel', desc: 'Cancel image generation', hotkey: 'Shift+X' },
+    {
+      title: 'Set Seed',
+      desc: 'Use the seed of the current image',
+      hotkey: 'S',
+    },
+    {
+      title: 'Set Parameters',
+      desc: 'Use all parameters of the current image',
+      hotkey: 'A',
+    },
+    { title: 'Restore Faces', desc: 'Restore the current image', hotkey: 'R' },
+    { title: 'Upscale', desc: 'Upscale the current image', hotkey: 'U' },
+    {
+      title: 'Show Info',
+      desc: 'Show metadata info of the current image',
+      hotkey: 'I',
+    },
+    {
+      title: 'Send To Image To Image',
+      desc: 'Send the current image to Image to Image module',
+      hotkey: 'Shift+I',
+    },
+    { title: 'Delete Image', desc: 'Delete the current image', hotkey: 'Del' },
+    {
+      title: 'Focus Prompt',
+      desc: 'Focus the prompt input area',
+      hotkey: 'Alt+A',
+    },
+    {
+      title: 'Previous Image',
+      desc: 'Display the previous image in the gallery',
+      hotkey: 'Arrow left',
+    },
+    {
+      title: 'Next Image',
+      desc: 'Display the next image in the gallery',
+      hotkey: 'Arrow right',
+    },
+  ];
+
+  const renderHotkeyModalItems = () => {
+    const hotkeyModalItemsToRender: ReactElement[] = [];
+
+    hotkeys.forEach((hotkey, i) => {
+      hotkeyModalItemsToRender.push(
+        <HotkeysModalItem
+          key={i}
+          title={hotkey.title}
+          description={hotkey.desc}
+          hotkey={hotkey.hotkey}
+        />
+      );
+    });
+
+    return hotkeyModalItemsToRender;
+  };
+
+  return (
+    <>
+      {cloneElement(children, {
+        onClick: onHotkeysModalOpen,
+      })}
+      <Modal isOpen={isHotkeyModalOpen} onClose={onHotkeysModalClose}>
+        <ModalOverlay />
+        <ModalContent className="hotkeys-modal">
+          <ModalCloseButton />
+          <h1>Keyboard Shorcuts</h1>
+          <div className="hotkeys-modal-items">{renderHotkeyModalItems()}</div>
+        </ModalContent>
+      </Modal>
+    </>
+  );
+}

frontend/src/features/system/HotkeysModal/HotkeysModalItem.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+
+interface HotkeysModalProps {
+  hotkey: string;
+  title: string;
+  description?: string;
+}
+
+export default function HotkeysModalItem(props: HotkeysModalProps) {
+  const { title, hotkey, description } = props;
+  return (
+    <div className="hotkey-modal-item">
+      <div className="hotkey-info">
+        <p className="hotkey-title">{title}</p>
+        {description && <p className="hotkey-description">{description}</p>}
+      </div>
+      <div className="hotkey-key">{hotkey}</div>
+    </div>
+  );
+}

frontend/src/features/system/SettingsModal/SettingsModal.scss

@@ -2,6 +2,7 @@
 
 .settings-modal {
   background-color: var(--settings-modal-bg) !important;
+  font-family: Inter;
 
   .settings-modal-content {
     display: grid;

frontend/src/features/system/SiteHeader.scss

@@ -21,7 +21,7 @@
 
 .site-header-right-side {
   display: grid;
-  grid-template-columns: repeat(5, max-content);
+  grid-template-columns: repeat(6, max-content);
   align-items: center;
   column-gap: 0.5rem;
 }

frontend/src/features/system/SiteHeader.tsx

@@ -1,9 +1,11 @@
 import { IconButton, Link, useColorMode } from '@chakra-ui/react';
 
 import { FaSun, FaMoon, FaGithub } from 'react-icons/fa';
-import { MdHelp, MdSettings } from 'react-icons/md';
+import { MdHelp, MdKeyboard, MdSettings } from 'react-icons/md';
 
 import InvokeAILogo from '../../assets/images/logo.png';
+import HotkeysModal from './HotkeysModal/HotkeysModal';
+
 import SettingsModal from './SettingsModal/SettingsModal';
 import StatusIndicator from './StatusIndicator';
 
@@ -40,6 +42,16 @@ const SiteHeader = () => {
           />
         </SettingsModal>
 
+        <HotkeysModal>
+          <IconButton
+            aria-label="Hotkeys"
+            variant="link"
+            fontSize={24}
+            size={'sm'}
+            icon={<MdKeyboard />}
+          />
+        </HotkeysModal>
+
         <IconButton
           aria-label="Link to Github Issues"
           variant="link"

frontend/src/styles/index.scss

@@ -11,6 +11,7 @@
 @use '../features/system/SiteHeader.scss';
 @use '../features/system/StatusIndicator.scss';
 @use '../features/system/SettingsModal/SettingsModal.scss';
+@use '../features/system/HotkeysModal/HotkeysModal.scss';
 @use '../features/system/Console.scss';
 
 // options

frontend/yarn.lock

@@ -1582,6 +1582,11 @@ balanced-match@^1.0.0:
   resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
   integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
 
+base64id@2.0.0, base64id@~2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6"
+  integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==
+
 binary-extensions@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.2.0.tgz#75f502eeaf9ffde42fc98829645be4ea76bd9e2d"
@@ -2359,6 +2364,11 @@ hoist-non-react-statics@^3.3.0, hoist-non-react-statics@^3.3.1, hoist-non-react-
   dependencies:
     react-is "^16.7.0"
 
+hotkeys-js@3.9.4:
+  version "3.9.4"
+  resolved "https://registry.yarnpkg.com/hotkeys-js/-/hotkeys-js-3.9.4.tgz#ce1aa4c3a132b6a63a9dd5644fc92b8a9b9cbfb9"
+  integrity sha512-2zuLt85Ta+gIyvs4N88pCYskNrxf1TFv3LR9t5mdAZIX8BcgQQ48F2opUptvHa6m8zsy5v/a0i9mWzTrlNWU0Q==
+
 ignore@^5.2.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.0.tgz#6d3bac8fa7fe0d45d9f9be7bac2fc279577e345a"
@@ -2618,7 +2628,7 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
-object-assign@^4.1.1:
+object-assign@^4, object-assign@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
   integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==
@@ -2818,6 +2828,13 @@ react-focus-lock@^2.9.1:
     use-callback-ref "^1.3.0"
     use-sidecar "^1.1.2"
 
+react-hotkeys-hook@^3.4.7:
+  version "3.4.7"
+  resolved "https://registry.yarnpkg.com/react-hotkeys-hook/-/react-hotkeys-hook-3.4.7.tgz#e16a0a85f59feed9f48d12cfaf166d7df4c96b7a"
+  integrity sha512-+bbPmhPAl6ns9VkXkNNyxlmCAIyDAcWbB76O4I0ntr3uWCRuIQf/aRLartUahe9chVMPj+OEzzfk3CQSjclUEQ==
+  dependencies:
+    hotkeys-js "3.9.4"
+
 react-icons@^4.4.0:
   version "4.4.0"
   resolved "https://registry.yarnpkg.com/react-icons/-/react-icons-4.4.0.tgz#a13a8a20c254854e1ec9aecef28a95cdf24ef703"
@@ -3044,6 +3061,18 @@ socket.io-parser@~4.2.0:
     "@socket.io/component-emitter" "~3.1.0"
     debug "~4.3.1"
 
+socket.io@^4.5.2:
+  version "4.5.2"
+  resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.5.2.tgz#1eb25fd380ab3d63470aa8279f8e48d922d443ac"
+  integrity sha512-6fCnk4ARMPZN448+SQcnn1u8OHUC72puJcNtSgg2xS34Cu7br1gQ09YKkO1PFfDn/wyUE9ZgMAwosJed003+NQ==
+  dependencies:
+    accepts "~1.3.4"
+    base64id "~2.0.0"
+    debug "~4.3.2"
+    engine.io "~6.2.0"
+    socket.io-adapter "~2.4.0"
+    socket.io-parser "~4.2.0"
+
 "source-map-js@>=0.6.2 <2.0.0", source-map-js@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"

ldm/dream/generator/base.py

@@ -21,6 +21,8 @@ class Generator():
         self.seed                = None
         self.latent_channels     = model.channels
         self.downsampling_factor = downsampling   # BUG: should come from model or config
+        self.perlin              = 0.0
+        self.threshold           = 0
         self.variation_amount    = 0
         self.with_variations     = []
 

ldm/dream/generator/inpaint.py

@@ -27,7 +27,7 @@ class Inpaint(Img2Img):
         # klms samplers not supported yet, so ignore previous sampler
         if isinstance(sampler,KSampler):
             print(
-                f">> sampler '{sampler.__class__.__name__}' is not yet supported for inpainting, using DDIMSampler instead."
+                f">> Using recommended DDIM sampler for inpainting."
             )
             sampler = DDIMSampler(self.model, device=self.model.device)
         

ldm/dream/generator/txt2img2img.py

@@ -64,7 +64,7 @@ class Txt2Img2Img(Generator):
             )
             
             print(
-                  f"\n>> Interpolating from {init_width}x{init_height} to {width}x{height}"
+                  f"\n>> Interpolating from {init_width}x{init_height} to {width}x{height} using DDIM sampling"
                  )
             
             # resizing
@@ -75,17 +75,19 @@ class Txt2Img2Img(Generator):
             )
 
             t_enc = int(strength * steps)
+            ddim_sampler = DDIMSampler(self.model, device=self.model.device)
+            ddim_sampler.make_schedule(
+                    ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
+            )
 
-            x = self.get_noise(width,height,False)
+            z_enc = ddim_sampler.stochastic_encode(
-
-            z_enc = sampler.stochastic_encode(
                 samples,
                 torch.tensor([t_enc]).to(self.model.device),
-                noise=x
+                noise=self.get_noise(width,height,False)
             )
 
             # decode it
-            samples = sampler.decode(
+            samples = ddim_sampler.decode(
                 z_enc,
                 c,
                 t_enc,

ldm/dream/restoration/realesrgan.py

@@ -1,6 +1,7 @@
 import torch
 import warnings
 import numpy as np
+from ldm.dream.devices import choose_precision, choose_torch_device
 
 from PIL import Image
 
@@ -8,17 +9,12 @@ from PIL import Image
 class ESRGAN():
     def __init__(self, bg_tile_size=400) -> None:
         self.bg_tile_size = bg_tile_size
+        device = torch.device(choose_torch_device())
+        precision = choose_precision(device)
+        use_half_precision = precision == 'float16'
 
-        if not torch.cuda.is_available():  # CPU or MPS on M1
+    def load_esrgan_bg_upsampler(self, precision):
-            use_half_precision = False
+        use_half_precision = precision == 'float16'
-        else:
-            use_half_precision = True
-
-    def load_esrgan_bg_upsampler(self):
-        if not torch.cuda.is_available():  # CPU or MPS on M1
-            use_half_precision = False
-        else:
-            use_half_precision = True
 
         from realesrgan.archs.srvgg_arch import SRVGGNetCompact
         from realesrgan import RealESRGANer
@@ -39,13 +35,13 @@ class ESRGAN():
 
         return bg_upsampler
 
-    def process(self, image, strength: float, seed: str = None, upsampler_scale: int = 2):
+    def process(self, image, strength: float, seed: str = None, upsampler_scale: int = 2, precision: str = 'float16'):
         with warnings.catch_warnings():
             warnings.filterwarnings('ignore', category=DeprecationWarning)
             warnings.filterwarnings('ignore', category=UserWarning)
 
             try:
-                upsampler = self.load_esrgan_bg_upsampler()
+                upsampler = self.load_esrgan_bg_upsampler(precision)
             except Exception:
                 import traceback
                 import sys

ldm/generate.py

@@ -174,7 +174,8 @@ class Generate:
             config                = None,
             gfpgan=None,
             codeformer=None,
-            esrgan=None
+            esrgan=None,
+            free_gpu_mem=False,
     ):
         models              = OmegaConf.load(conf)
         mconfig             = models[model]
@@ -201,6 +202,7 @@ class Generate:
         self.gfpgan = gfpgan
         self.codeformer = codeformer
         self.esrgan = esrgan
+        self.free_gpu_mem = free_gpu_mem
 
         # Note that in previous versions, there was an option to pass the
         # device to Generate(). However the device was then ignored, so
@@ -417,7 +419,8 @@ class Generate:
                 generator = self._make_txt2img()
 
             generator.set_variation(
-                self.seed, variation_amount, with_variations)
+                self.seed, variation_amount, with_variations
+            )
             results = generator.generate(
                 prompt,
                 iterations=iterations,
@@ -596,7 +599,8 @@ class Generate:
                 opt,
                 args,
                 image_callback = callback,
-                prefix         = prefix
+                prefix         = prefix,
+                precision      = self.precision,
             )
                 
         elif tool is None:
@@ -626,18 +630,14 @@ class Generate:
             height,
         )
 
+        if image.width < self.width and image.height < self.height:
+            print(f'>> WARNING: img2img and inpainting may produce unexpected results with initial images smaller than {self.width}x{self.height} in both dimensions')
+
         # if image has a transparent area and no mask was provided, then try to generate mask
-        if self._has_transparency(image) and not mask:
+        if self._has_transparency(image):
-            print(
+            self._transparency_check_and_warning(image, mask)
-                '>> Initial image has transparent areas. Will inpaint in these regions.')
-            if self._check_for_erasure(image):
-                print(
-                    '>> WARNING: Colors underneath the transparent region seem to have been erased.\n',
-                    '>>          Inpainting will be suboptimal. Please preserve the colors when making\n',
-                    '>>          a transparency mask, or provide mask explicitly using --init_mask (-M).'
-                )
             # this returns a torch tensor
-            init_mask = self._create_init_mask(image,width,height,fit=fit)
+            init_mask = self._create_init_mask(image, width, height, fit=fit)
             
         if (image.width * image.height) > (self.width * self.height):
             print(">> This input is larger than your defaults. If you run out of memory, please use a smaller image.")
@@ -771,7 +771,7 @@ class Generate:
                         if len(upscale) < 2:
                             upscale.append(0.75)
                         image = self.esrgan.process(
-                            image, upscale[1], seed, int(upscale[0]))
+                            image, upscale[1], seed, int(upscale[0]), precision=self.precision)
                     else:
                         print(">> ESRGAN is disabled. Image not upscaled.")
             except Exception as e:
@@ -953,6 +953,17 @@ class Generate:
                         colored += 1
         return colored == 0
 
+    def _transparency_check_and_warning(image, mask):
+        if not mask:
+            print(
+                '>> Initial image has transparent areas. Will inpaint in these regions.')
+            if self._check_for_erasure(image):
+                print(
+                    '>> WARNING: Colors underneath the transparent region seem to have been erased.\n',
+                    '>>          Inpainting will be suboptimal. Please preserve the colors when making\n',
+                    '>>          a transparency mask, or provide mask explicitly using --init_mask (-M).'
+                )
+
     def _squeeze_image(self, image):
         x, y, resize_needed = self._resolution_check(image.width, image.height)
         if resize_needed:

ldm/models/diffusion/ksampler.py

@@ -51,8 +51,9 @@ class KSampler(Sampler):
             schedule,
             steps=model.num_timesteps,
         )
-        self.ds    = None
+        self.sigmas = None
-        self.s_in  = None
+        self.ds     = None
+        self.s_in   = None
 
         def forward(self, x, sigma, uncond, cond, cond_scale):
             x_in = torch.cat([x] * 2)
@@ -140,7 +141,7 @@ class KSampler(Sampler):
             'uncond': unconditional_conditioning,
             'cond_scale': unconditional_guidance_scale,
         }
-        print(f'>> Sampling with k_{self.schedule}')
+        print(f'>> Sampling with k_{self.schedule} starting at step {len(self.sigmas)-S-1} of {len(self.sigmas)-1} ({S} new sampling steps)')
         return (
             K.sampling.__dict__[f'sample_{self.schedule}'](
                 model_wrap_cfg, x, sigmas, extra_args=extra_args,
@@ -149,6 +150,8 @@ class KSampler(Sampler):
             None,
         )
 
+    # this code will support inpainting if and when ksampler API modified or
+    # a workaround is found.
     @torch.no_grad()
     def p_sample(
             self,

ldm/models/diffusion/sampler.py

@@ -39,6 +39,7 @@ class Sampler(object):
             ddim_eta=0.0,
             verbose=False,
     ):
+        self.total_steps = ddim_num_steps
         self.ddim_timesteps = make_ddim_timesteps(
             ddim_discr_method=ddim_discretize,
             num_ddim_timesteps=ddim_num_steps,
@@ -211,6 +212,7 @@ class Sampler(object):
             if ddim_use_original_steps
             else np.flip(timesteps)
         )
+
         total_steps=steps
 
         iterator = tqdm(
@@ -305,7 +307,7 @@ class Sampler(object):
 
         time_range = np.flip(timesteps)
         total_steps = timesteps.shape[0]
-        print(f'>> Running {self.__class__.__name__} Sampling with {total_steps} timesteps')
+        print(f'>> Running {self.__class__.__name__} sampling starting at step {self.total_steps - t_start} of {self.total_steps} ({total_steps} new sampling steps)')
 
         iterator = tqdm(time_range, desc='Decoding image', total=total_steps)
         x_dec    = x_latent

scripts/dream.py

@@ -75,7 +75,8 @@ def main():
             precision      = opt.precision,
             gfpgan=gfpgan,
             codeformer=codeformer,
-            esrgan=esrgan
+            esrgan=esrgan,
+            free_gpu_mem=opt.free_gpu_mem,
             )
     except (FileNotFoundError, IOError, KeyError) as e:
         print(f'{e}. Aborting.')
@@ -104,8 +105,6 @@ def main():
 
     # preload the model
     gen.load_model()
-    #set additional option
-    gen.free_gpu_mem = opt.free_gpu_mem
 
     # web server loops forever
     if opt.web or opt.gui: