add option to show intermediate latent space

backend/invoke_ai_web_server.py

@@ -614,11 +614,15 @@ class InvokeAIWebServer:
                 progress.set_current_status("Generating")
                 progress.set_current_status_has_steps(True)
 
+                wants_progress_image = generation_parameters['progress_images'] and step % 5 == 0
+                wants_progress_latents = generation_parameters['progress_latents']
+
                 if (
-                    generation_parameters['progress_images'] and step % 5 == 0 \
-                        and step < generation_parameters['steps'] - 1
+                    wants_progress_image | wants_progress_latents
+                    and step < generation_parameters['steps'] - 1
                 ):
-                    image = self.generate.sample_to_image(sample)
+                    image = self.generate.sample_to_image(sample) if wants_progress_image \
+                            else self.generate.sample_to_lowres_estimated_image(sample)
                     metadata = self.parameters_to_generated_image_metadata(
                         generation_parameters
                     )

frontend/src/common/util/parameterTranslation.ts

@@ -62,7 +62,7 @@ export const frontendToBackendParameters = (
     shouldRandomizeSeed,
   } = optionsState;
 
-  const { shouldDisplayInProgressType } = systemState;
+  const { shouldDisplayInProgress, shouldDisplayInProgressLatents } = systemState;
 
   const generationParameters: { [k: string]: any } = {
     prompt,
@@ -76,8 +76,8 @@ export const frontendToBackendParameters = (
     width,
     sampler_name: sampler,
     seed,
-    progress_images: shouldDisplayInProgressType === 'full-res',
-    progress_latents: shouldDisplayInProgressType === 'latents'
+    progress_images: shouldDisplayInProgress,
+    progress_latents: shouldDisplayInProgressLatents,
   };
 
   generationParameters.seed = shouldRandomizeSeed

frontend/src/features/system/SettingsModal/SettingsModal.tsx

@@ -22,7 +22,8 @@ import {
   setSaveIntermediatesInterval,
   setShouldConfirmOnDelete,
   setShouldDisplayGuides,
-  setShouldDisplayInProgressType,
+  setShouldDisplayInProgress,
+  setShouldDisplayInProgressLatents,
   SystemState,
 } from '../systemSlice';
 import ModelList from './ModelList';
@@ -33,13 +34,15 @@ const systemSelector = createSelector(
   (state: RootState) => state.system,
   (system: SystemState) => {
     const {
-      shouldDisplayInProgressType,
+      shouldDisplayInProgress,
+      shouldDisplayInProgressLatents,
       shouldConfirmOnDelete,
       shouldDisplayGuides,
       model_list,
     } = system;
     return {
-      shouldDisplayInProgressType,
+      shouldDisplayInProgress,
+      shouldDisplayInProgressLatents,
       shouldConfirmOnDelete,
       shouldDisplayGuides,
       models: _.map(model_list, (_model, key) => key),
@@ -83,7 +86,8 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
   } = useDisclosure();
 
   const {
-    shouldDisplayInProgressType,
+    shouldDisplayInProgress,
+    shouldDisplayInProgressLatents,
     shouldConfirmOnDelete,
     shouldDisplayGuides,
   } = useAppSelector(systemSelector);
@@ -125,6 +129,12 @@ const SettingsModal = ({ children }: SettingsModalProps) => {
                 dispatcher={setShouldDisplayInProgressType}
               />
 
+              <SettingsModalItem
+                settingTitle="Display In-Progress Latents (quick; lo-res)"
+                isChecked={shouldDisplayInProgressLatents}
+                dispatcher={setShouldDisplayInProgressLatents}
+              />
+
               <SettingsModalItem
                 settingTitle="Confirm on Delete"
                 isChecked={shouldConfirmOnDelete}

frontend/src/features/system/systemSlice.ts

@@ -25,7 +25,8 @@ export type InProgressImageType = 'none' | 'full-res' | 'latents';
 export interface SystemState
   extends InvokeAI.SystemStatus,
     InvokeAI.SystemConfig {
-  shouldDisplayInProgressType: string;
+  shouldDisplayInProgress: boolean;
+  shouldDisplayInProgressLatents: boolean;
   log: Array<LogEntry>;
   shouldShowLogViewer: boolean;
   isGFPGANAvailable: boolean;
@@ -51,7 +52,8 @@ const initialSystemState: SystemState = {
   isProcessing: false,
   log: [],
   shouldShowLogViewer: false,
-  shouldDisplayInProgressType: "none",
+  shouldDisplayInProgress: false,
+  shouldDisplayInProgressLatents: false,
   shouldDisplayGuides: true,
   isGFPGANAvailable: true,
   isESRGANAvailable: true,
@@ -83,6 +85,9 @@ export const systemSlice = createSlice({
     setShouldDisplayInProgressType: (state, action: PayloadAction<string>) => {
       state.shouldDisplayInProgressType = action.payload;
     },
+    setShouldDisplayInProgressLatents: (state, action: PayloadAction<boolean>) => {
+      state.shouldDisplayInProgressLatents = action.payload;
+    },
     setIsProcessing: (state, action: PayloadAction<boolean>) => {
       state.isProcessing = action.payload;
     },
@@ -192,7 +197,8 @@ export const systemSlice = createSlice({
 });
 
 export const {
-  setShouldDisplayInProgressType,
+  setShouldDisplayInProgress,
+  setShouldDisplayInProgressLatents,
   setIsProcessing,
   addLogEntry,
   setShouldShowLogViewer,

ldm/generate.py

@@ -913,6 +913,9 @@ class Generate:
     def sample_to_image(self, samples):
         return self._make_base().sample_to_image(samples)
 
+    def sample_to_lowres_estimated_image(self, samples):
+        return self._make_base().sample_to_lowres_estimated_image(samples)
+
     # very repetitive code - can this be simplified? The KSampler names are
     # consistent, at least
     def _set_sampler(self):

ldm/invoke/generator/base.py

@@ -119,19 +119,19 @@ class Generator():
         # write an approximate RGB image from latent samples for a single step to PNG
 
     def sample_to_lowres_estimated_image(self,samples):
-        # origingally adapted from code by @erucipe and @keturn here:
+        # adapted from code by @erucipe and @keturn here:
         # https://discuss.huggingface.co/t/decoding-latents-to-rgb-without-upscaling/23204/7
 
-        # these updated numbers for v1.5 are from @torridgristle
-        v1_5_latent_rgb_factors = torch.tensor([
-            #    R        G        B
-            [ 0.3444,  0.1385,  0.0670], # L1
-            [ 0.1247,  0.4027,  0.1494], # L2
-            [-0.3192,  0.2513,  0.2103], # L3
-            [-0.1307, -0.1874, -0.7445]  # L4
+        # these numbers were determined empirically by @keturn
+        v1_4_latent_rgb_factors = torch.tensor([
+                    # R        G        B
+                    [ 0.298, 0.207, 0.208],  # L1
+                    [ 0.187, 0.286, 0.173],  # L2
+                    [-0.158, 0.189, 0.264],  # L3
+                    [-0.184, -0.271, -0.473],  # L4
         ], dtype=samples.dtype, device=samples.device)
 
-        latent_image = samples[0].permute(1, 2, 0) @ v1_5_latent_rgb_factors
+        latent_image = samples[0].permute(1, 2, 0) @ v1_4_latent_rgb_factors
         latents_ubyte = (((latent_image + 1) / 2)
                          .clamp(0, 1)  # change scale from -1..1 to 0..1
                          .mul(0xFF)  # to 0..255

ldm/invoke/server.py

@@ -34,6 +34,7 @@ def build_opt(post_data, seed, gfpgan_model_exists):
     setattr(opt, 'facetool_strength', float(post_data['facetool_strength']) if gfpgan_model_exists else 0)
     setattr(opt, 'upscale', [int(post_data['upscale_level']), float(post_data['upscale_strength'])] if post_data['upscale_level'] != '' else None)
     setattr(opt, 'progress_images', 'progress_images' in post_data)
+    setattr(opt, 'progress_latents', 'progress_latents' in post_data)
     setattr(opt, 'seed', None if int(post_data['seed']) == -1 else int(post_data['seed']))
     setattr(opt, 'threshold', float(post_data['threshold']))
     setattr(opt, 'perlin', float(post_data['perlin']))
@@ -227,8 +228,13 @@ class DreamServer(BaseHTTPRequestHandler):
             # since rendering images is moderately expensive, only render every 5th image
             # and don't bother with the last one, since it'll render anyway
             nonlocal step_index
-            if opt.progress_images and step % 5 == 0 and step < opt.steps - 1:
-                image = self.model.sample_to_image(sample)
+
+            wants_progress_latents = opt.progress_latents
+            wants_progress_image = opt.progress_image and step % 5 == 0
+
+            if (wants_progress_image | wants_progress_latents) and step < opt.steps - 1:
+                image = self.model.sample_to_image(sample) if wants_progress_image \
+                        else self.model.sample_to_lowres_estimated_image(sample)
                 step_index_padded = str(step_index).rjust(len(str(opt.steps)), '0')
                 name = f'{prefix}.{opt.seed}.{step_index_padded}.png'
                 metadata = f'{opt.prompt} -S{opt.seed} [intermediate]'

server/models.py

@@ -39,6 +39,7 @@ class DreamBase():
   model: str = None # The model to use (currently unused)
   embeddings = None # The embeddings to use (currently unused)
   progress_images: bool = False
+  progress_latents: bool = False
 
   # GFPGAN
   enable_gfpgan: bool
@@ -94,6 +95,7 @@ class DreamBase():
       self.seamless = 'seamless' in j
       self.hires_fix = 'hires_fix' in j
       self.progress_images = 'progress_images' in j
+      self.progress_latents = 'progress_latents' in j
 
     # GFPGAN
     self.enable_gfpgan = 'enable_gfpgan' in j and bool(j.get('enable_gfpgan'))