Bump Diffusers Dependency (#5243)

## What type of PR is this? (check all applicable)

- [ ] Refactor
- [ ] Feature
- [ ] Bug Fix
- [ X ] Optimization
- [ ] Documentation Update
- [ ] Community Node Submission


## Have you discussed this change with the InvokeAI team?
- [ ] Yes
- [ X ] No, because: dependency bump

      
## Have you updated all relevant documentation?
- [ ] Yes
- [ x ] No


## Description
Updating diffusers to .24 - fixes a few issues. Needs to be tested to
ensure things like our IP Adapter implementation don't break

invokeai/backend/stable_diffusion/diffusers_pipeline.py

@@ -242,17 +242,6 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
         control_model: ControlNetModel = None,
     ):
         super().__init__(
-            vae,
-            text_encoder,
-            tokenizer,
-            unet,
-            scheduler,
-            safety_checker,
-            feature_extractor,
-            requires_safety_checker,
-        )
-
-        self.register_modules(
             vae=vae,
             text_encoder=text_encoder,
             tokenizer=tokenizer,
@@ -260,9 +249,9 @@ class StableDiffusionGeneratorPipeline(StableDiffusionPipeline):
             scheduler=scheduler,
             safety_checker=safety_checker,
             feature_extractor=feature_extractor,
-            # FIXME: can't currently register control module
-            # control_model=control_model,
+            requires_safety_checker=requires_safety_checker,
         )
+
         self.invokeai_diffuser = InvokeAIDiffuserComponent(self.unet, self._unet_forward)
         self.control_model = control_model
         self.use_ip_adapter = False

pyproject.toml

@@ -41,7 +41,7 @@ dependencies = [
   "controlnet-aux>=0.0.6",
   "timm==0.6.13",               # needed to override timm latest in controlnet_aux, see  https://github.com/isl-org/ZoeDepth/issues/26
   "datasets",
-  "diffusers[torch]~=0.23.0",
+  "diffusers[torch]~=0.24.0",
   "dnspython~=2.4.0",
   "dynamicprompts",
   "easing-functions",
@@ -49,7 +49,7 @@ dependencies = [
   "facexlib",
   "fastapi~=0.105.0",
   "fastapi-events~=0.9.1",
-  "huggingface-hub~=0.16.4",
+  "huggingface-hub~=0.19.4",
   "imohash",
   "invisible-watermark~=0.2.0", # needed to install SDXL base and refiner using their repo_ids
   "matplotlib",                 # needed for plotting of Penner easing functions