diff --git a/invokeai/backend/model_management/model_manager.py b/invokeai/backend/model_management/model_manager.py
index 4fbc80703a..87fb938338 100644
--- a/invokeai/backend/model_management/model_manager.py
+++ b/invokeai/backend/model_management/model_manager.py
@@ -357,6 +357,13 @@ class ModelManager(object):
             vae_id = mconfig.vae.repo_id
             vae = (SDModelType.vae, vae_id)
 
+        # optimization - don't load whole model if the user
+        # is asking for just a piece of it
+        if model_type == SDModelType.diffusers and submodel and not subfolder:
+            model_type = submodel
+            subfolder = submodel.name
+            submodel = None
+
         model_context = self.cache.get_model(
             location,
             model_type = model_type,