diff --git a/invokeai/backend/model_management/model_cache.py b/invokeai/backend/model_management/model_cache.py
index 1af02637d3..02448c59a6 100644
--- a/invokeai/backend/model_management/model_cache.py
+++ b/invokeai/backend/model_management/model_cache.py
@@ -178,6 +178,7 @@ class ModelCache(object):
             if hasattr(model,'to') and (key in self.loaded_models
                                         and self.locked_models[key] == 0):
                 model.to(self.storage_device)
+                self.loaded_models.remove(key)
             yield model
 
     def _offload_unlocked_models(self):