diff --git a/ldm/models/diffusion/shared_invokeai_diffusion.py b/ldm/models/diffusion/shared_invokeai_diffusion.py
index 10ad328575..1ecbd1c488 100644
--- a/ldm/models/diffusion/shared_invokeai_diffusion.py
+++ b/ldm/models/diffusion/shared_invokeai_diffusion.py
@@ -69,7 +69,8 @@ class InvokeAIDiffuserComponent:
         try:
             yield None
         finally:
-            self.remove_cross_attention_control(old_attn_processor)
+            if old_attn_processor is not None:
+                self.remove_cross_attention_control(old_attn_processor)
             # TODO resuscitate attention map saving
             #self.remove_attention_map_saving()