feat: Add Lora to Canvas (#3643)

- Add Loras to Canvas
- Revert inference_mode to no_grad coz inference tensors fail with
latent to latent.

invokeai/app/invocations/compel.py

@@ -56,7 +56,7 @@ class CompelInvocation(BaseInvocation):
             },
         }
 
-    @torch.inference_mode()
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> CompelOutput:
         tokenizer_info = context.services.model_manager.get_model(
             **self.clip.tokenizer.dict(),

invokeai/app/invocations/latent.py

@@ -285,7 +285,7 @@ class TextToLatentsInvocation(BaseInvocation):
                 # MultiControlNetModel has been refactored out, just need list[ControlNetData]
         return control_data
 
-    @torch.inference_mode()
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         noise = context.services.latents.get(self.noise.latents_name)
 
@@ -369,7 +369,7 @@ class LatentsToLatentsInvocation(TextToLatentsInvocation):
             },
         }
 
-    @torch.inference_mode()
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         noise = context.services.latents.get(self.noise.latents_name)
         latent = context.services.latents.get(self.latents.latents_name)
@@ -461,7 +461,7 @@ class LatentsToImageInvocation(BaseInvocation):
             },
         }
 
-    @torch.inference_mode()
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> ImageOutput:
         latents = context.services.latents.get(self.latents.latents_name)
 
@@ -599,7 +599,7 @@ class ImageToLatentsInvocation(BaseInvocation):
             },
         }
 
-    @torch.inference_mode()
+    @torch.no_grad()
     def invoke(self, context: InvocationContext) -> LatentsOutput:
         # image = context.services.images.get(
         #     self.image.image_type, self.image.image_name

invokeai/backend/model_management/lora.py

@@ -526,7 +526,7 @@ class ModelPatcher:
     ):
         original_weights = dict()
         try:
-            with torch.inference_mode():
+            with torch.no_grad():
                 for lora, lora_weight in loras:
                     #assert lora.device.type == "cpu"
                     for layer_key, layer in lora.layers.items():
@@ -552,7 +552,7 @@ class ModelPatcher:
             yield # wait for context manager exit
 
         finally:
-            with torch.inference_mode():
+            with torch.no_grad():
                 for module_key, weight in original_weights.items():
                     model.get_submodule(module_key).weight.copy_(weight)
 

invokeai/frontend/web/src/features/nodes/util/graphBuilders/buildCanvasInpaintGraph.ts

@@ -8,6 +8,7 @@ import {
   RangeOfSizeInvocation,
 } from 'services/api/types';
 import { modelIdToMainModelField } from '../modelIdToMainModelField';
+import { addLoRAsToGraph } from './addLoRAsToGraph';
 import { addVAEToGraph } from './addVAEToGraph';
 import {
   INPAINT,
@@ -194,6 +195,8 @@ export const buildCanvasInpaintGraph = (
     ],
   };
 
+  addLoRAsToGraph(graph, state, INPAINT);
+
   // Add VAE
   addVAEToGraph(graph, state);
 

invokeai/frontend/web/src/features/ui/components/tabs/UnifiedCanvas/UnifiedCanvasParameters.tsx

@@ -1,14 +1,15 @@
-import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons';
+import ParamDynamicPromptsCollapse from 'features/dynamicPrompts/components/ParamDynamicPromptsCollapse';
-import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse';
+import ParamLoraCollapse from 'features/lora/components/ParamLoraCollapse';
-import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse';
 import ParamInfillAndScalingCollapse from 'features/parameters/components/Parameters/Canvas/InfillAndScaling/ParamInfillAndScalingCollapse';
 import ParamSeamCorrectionCollapse from 'features/parameters/components/Parameters/Canvas/SeamCorrection/ParamSeamCorrectionCollapse';
-import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters';
-import { memo } from 'react';
-import ParamPositiveConditioning from 'features/parameters/components/Parameters/Core/ParamPositiveConditioning';
-import ParamNegativeConditioning from 'features/parameters/components/Parameters/Core/ParamNegativeConditioning';
 import ParamControlNetCollapse from 'features/parameters/components/Parameters/ControlNet/ParamControlNetCollapse';
-import ParamDynamicPromptsCollapse from 'features/dynamicPrompts/components/ParamDynamicPromptsCollapse';
+import ParamNegativeConditioning from 'features/parameters/components/Parameters/Core/ParamNegativeConditioning';
+import ParamPositiveConditioning from 'features/parameters/components/Parameters/Core/ParamPositiveConditioning';
+import ParamSymmetryCollapse from 'features/parameters/components/Parameters/Symmetry/ParamSymmetryCollapse';
+import ParamVariationCollapse from 'features/parameters/components/Parameters/Variations/ParamVariationCollapse';
+import ProcessButtons from 'features/parameters/components/ProcessButtons/ProcessButtons';
+import { memo } from 'react';
+import UnifiedCanvasCoreParameters from './UnifiedCanvasCoreParameters';
 
 const UnifiedCanvasParameters = () => {
   return (
@@ -17,6 +18,7 @@ const UnifiedCanvasParameters = () => {
       <ParamNegativeConditioning />
       <ProcessButtons />
       <UnifiedCanvasCoreParameters />
+      <ParamLoraCollapse />
       <ParamDynamicPromptsCollapse />
       <ParamControlNetCollapse />
       <ParamVariationCollapse />