TOIL(pip): Refactor pip requirements across the board (#531)

* Refactor pip requirements across the board Signed-off-by: Ben Alkov <ben.alkov@gmail.com> * fix name, version in setup.py Signed-off-by: Ben Alkov <ben.alkov@gmail.com> * Update notebooks for new requirements file changes Signed-off-by: Ben Alkov <ben.alkov@gmail.com>
2024-08-30 20:32:17 +00:00 · 2022-09-14 07:01:58 -04:00 · 2022-09-14 07:01:58 -04:00 · d15c75ecae
commit d15c75ecae
parent ecb84ecc10
11 changed files with 191 additions and 314 deletions
--- a/notebooks/Stable-Diffusion-local-Windows.ipynb
+++ b/notebooks/Stable-Diffusion-local-Windows.ipynb
@ -40,57 +40,9 @@
   "outputs": [],
   "source": [
    "%%cmd\n",
-    "git clone https://github.com/lstein/stable-diffusion.git"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%cd stable-diffusion"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "%%writefile requirements.txt\n",
-    "albumentations==0.4.3\n",
-    "einops==0.3.0\n",
-    "huggingface-hub==0.8.1\n",
-    "imageio-ffmpeg==0.4.2\n",
-    "imageio==2.9.0\n",
-    "kornia==0.6.0\n",
-    "# pip will resolve the version which matches torch\n",
-    "numpy\n",
-    "omegaconf==2.1.1\n",
-    "opencv-python==4.6.0.66\n",
-    "pillow==9.2.0\n",
-    "pip>=22\n",
-    "pudb==2019.2\n",
-    "pytorch-lightning==1.4.2\n",
-    "streamlit==1.12.0\n",
-    "# \"CompVis/taming-transformers\" doesn't work\n",
-    "# ldm\\models\\autoencoder.py\", line 6, in <module>\n",
-    "#    from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer\n",
-    "# ModuleNotFoundError\n",
-    "taming-transformers-rom1504==0.0.6\n",
-    "test-tube>=0.7.5\n",
-    "torch-fidelity==0.3.0\n",
-    "torchmetrics==0.6.0\n",
-    "transformers==4.19.2\n",
-    "git+https://github.com/openai/CLIP.git@main#egg=clip\n",
-    "git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion\n",
-    "# No CUDA in PyPi builds\n",
-    "--extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org\n",
-    "torch==1.11.0\n",
-    "# Same as numpy - let pip do its thing\n",
-    "torchvision\n",
-    "-e .\n"
+    "git clone https://github.com/lstein/stable-diffusion.git\n",
+    "cd /content/stable-diffusion/\n",
+    "git checkout --quiet development"
   ]
  },
  {
@ -100,14 +52,14 @@
   "outputs": [],
   "source": [
    "%%cmd\n",
-    "pew new --python 3.10 -r requirements.txt --dont-activate ldm"
+    "pew new --python 3.10 -r requirements-lin-win-colab-CUDA.txt --dont-activate stable-diffusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
-    "# Switch the notebook kernel to the new 'ldm' environment!\n",
+    "# Switch the notebook kernel to the new 'stable-diffusion' environment!\n",
    "\n",
    "## VSCode: restart VSCode and come back to this cell\n",
    "\n",
@ -115,7 +67,7 @@
    "1. Type \"Select Interpreter\" and select \"Jupyter: Select Interpreter to Start Jupyter Server\"\n",
    "1. VSCode will say that it needs to install packages. Click the \"Install\" button.\n",
    "1. Once the install is finished, do 1 & 2 again\n",
-    "1. Pick 'ldm'\n",
+    "1. Pick 'stable-diffusion'\n",
    "1. Run the following cell"
   ]
  },
@ -136,7 +88,7 @@
    "## Jupyter/JupyterLab\n",
    "\n",
    "1. Run the cell below\n",
-    "1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'ldm' from the drop-down.\n"
+    "1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'stable-diffusion' from the drop-down.\n"
   ]
  },
  {
@ -154,9 +106,9 @@
   "source": [
    "# DO NOT RUN THIS CELL IF YOU ARE USING VSCODE!!\n",
    "%%cmd\n",
-    "pew workon ldm\n",
+    "pew workon stable-diffusion\n",
    "pip3 install ipykernel\n",
-    "python -m ipykernel install --name=ldm"
+    "python -m ipykernel install --name=stable-diffusion"
   ]
  },
  {
@ -231,7 +183,7 @@
    "Now:\n",
    "\n",
    "1. `cd` to wherever the 'stable-diffusion' directory is\n",
-    "1. Run `pew workon ldm`\n",
+    "1. Run `pew workon stable-diffusion`\n",
    "1. Run `winpty python scripts\\dream.py`"
   ]
  }
--- a/notebooks/Stable_Diffusion_AI_Notebook.ipynb
+++ b/notebooks/Stable_Diffusion_AI_Notebook.ipynb
@ -1,27 +1,12 @@
 {
-  "nbformat": 4,
-  "nbformat_minor": 0,
-  "metadata": {
-    "colab": {
-      "provenance": [],
-      "collapsed_sections": [],
-      "private_outputs": true
-    },
-    "kernelspec": {
-      "name": "python3",
-      "display_name": "Python 3"
-    },
-    "language_info": {
-      "name": "python"
-    },
-    "accelerator": "GPU",
-    "gpuClass": "standard"
-  },
  "cells": [
    {
      "cell_type": "markdown",
+      "metadata": {
+        "id": "ycYWcsEKc6w7"
+      },
      "source": [
-        "# Stable Diffusion AI Notebook (Release 1.13)\n",
+        "# Stable Diffusion AI Notebook (Release 1.14)\n",
        "\n",
        "<img src=\"https://user-images.githubusercontent.com/60411196/186547976-d9de378a-9de8-4201-9c25-c057a9c59bad.jpeg\" alt=\"stable-diffusion-ai\" width=\"170px\"/> <br>\n",
        "#### Instructions:\n",
@ -35,33 +20,30 @@
        "<font color=\"red\">Requirements:</font> For this notebook to work you need to have [Stable-Diffusion-v-1-4](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) stored in your Google Drive, it will be needed in cell #7\n",
        "##### For more details visit Github repository: [lstein/stable-diffusion](https://github.com/lstein/stable-diffusion)\n",
        "---\n"
-      ],
-      "metadata": {
-        "id": "ycYWcsEKc6w7"
-      }
+      ]
    },
    {
      "cell_type": "markdown",
-      "source": [
-        "## ◢ Installation"
-      ],
      "metadata": {
        "id": "dr32VLxlnouf"
-      }
+      },
+      "source": [
+        "## ◢ Installation"
+      ]
    },
    {
      "cell_type": "code",
-      "source": [
-        "#@title 1. Check current GPU assigned\n",
-        "!nvidia-smi -L\n",
-        "!nvidia-smi"
-      ],
+      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "a2Z5Qu_o8VtQ"
      },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "#@title 1. Check current GPU assigned\n",
+        "!nvidia-smi -L\n",
+        "!nvidia-smi"
+      ]
    },
    {
      "cell_type": "code",
@ -75,90 +57,91 @@
        "#@title 2. Download stable-diffusion Repository\n",
        "from os.path import exists\n",
        "\n",
-        "if exists(\"/content/stable-diffusion/\")==True:\n",
-        "  %cd /content/stable-diffusion/\n",
-        "  print(\"Already downloaded repo\")\n",
-        "else:\n",
-        "  !git clone --quiet https://github.com/lstein/stable-diffusion.git # Original repo\n",
-        "  %cd /content/stable-diffusion/\n",
-        "  !git checkout --quiet tags/release-1.13"
+        "!git clone --quiet https://github.com/lstein/stable-diffusion.git # Original repo\n",
+        "%cd /content/stable-diffusion/\n",
+        "!git checkout --quiet tags/release-1.14.1"
      ]
    },
    {
      "cell_type": "code",
-      "source": [
-        "#@title 3. Install dependencies\n",
-        "import gc\n",
-        "\n",
-        "if exists(\"/content/stable-diffusion/requirements-colab.txt\")==True:\n",
-        "  %cd /content/stable-diffusion/\n",
-        "  print(\"Already downloaded requirements file\")\n",
-        "else:\n",
-        "  !wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements-colab.txt\n",
-        "!pip install colab-xterm\n",
-        "!pip install -r requirements-colab.txt\n",
-        "gc.collect()"
-      ],
+      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "QbXcGXYEFSNB"
      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "code",
+      "outputs": [],
      "source": [
-        "#@title 4. Load small ML models required\n",
-        "%cd /content/stable-diffusion/\n",
-        "!python scripts/preload_models.py\n",
+        "#@title 3. Install dependencies\n",
+        "import gc\n",
+        "\n",
+        "!wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements.txt\n",
+        "!wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements-lin-win-colab-CUDA.txt\n",
+        "!pip install colab-xterm\n",
+        "!pip install -r requirements-lin-win-colab-CUDA.txt\n",
+        "!pip install clean-fid torchtext\n",
        "gc.collect()"
-      ],
-      "metadata": {
-        "cellView": "form",
-        "id": "ChIDWxLVHGGJ"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
    },
    {
      "cell_type": "code",
-      "source": [
-        "#@title 5. Restart Runtime\n",
-        "exit()"
-      ],
+      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "8rSMhgnAttQa"
      },
-      "execution_count": null,
-      "outputs": []
-    },
-    {
-      "cell_type": "markdown",
+      "outputs": [],
      "source": [
-        "## ◢ Configuration"
-      ],
-      "metadata": {
-        "id": "795x1tMoo8b1"
-      }
+        "#@title 4. Restart Runtime\n",
+        "exit()"
+      ]
    },
    {
      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "ChIDWxLVHGGJ"
+      },
+      "outputs": [],
      "source": [
-        "#@title 6. Mount google Drive\n",
-        "from google.colab import drive\n",
-        "drive.mount('/content/drive')"
-      ],
+        "#@title 5. Load small ML models required\n",
+        "import gc\n",
+        "%cd /content/stable-diffusion/\n",
+        "!python scripts/preload_models.py\n",
+        "gc.collect()"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "795x1tMoo8b1"
+      },
+      "source": [
+        "## ◢ Configuration"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "YEWPV-sF1RDM"
      },
-      "execution_count": null,
-      "outputs": []
+      "outputs": [],
+      "source": [
+        "#@title 6. Mount google Drive\n",
+        "from google.colab import drive\n",
+        "drive.mount('/content/drive')"
+      ]
    },
    {
      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "zRTJeZ461WGu"
+      },
+      "outputs": [],
      "source": [
        "#@title 7. Drive Path to model\n",
        "#@markdown Path should start with /content/drive/path-to-your-file <br>\n",
@ -167,20 +150,20 @@
        "from os.path import exists\n",
        "\n",
        "model_path = \"\" #@param {type:\"string\"}\n",
-        "if exists(model_path)==True:\n",
+        "if exists(model_path):\n",
        "    print(\"✅ Valid directory\")\n",
        "else: \n",
        "    print(\"❌ File doesn't exist\")"
-      ],
-      "metadata": {
-        "cellView": "form",
-        "id": "zRTJeZ461WGu"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
    },
    {
      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "UY-NNz4I8_aG"
+      },
+      "outputs": [],
      "source": [
        "#@title 8. Symlink to model\n",
        "\n",
@ -188,39 +171,39 @@
        "import os \n",
        "\n",
        "# Folder creation if it doesn't exist\n",
-        "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1\")==True:\n",
+        "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1\"):\n",
        "     print(\"❗ Dir stable-diffusion-v1 already exists\")\n",
        "else:\n",
        "    %mkdir /content/stable-diffusion/models/ldm/stable-diffusion-v1\n",
        "    print(\"✅ Dir stable-diffusion-v1 created\")\n",
        "\n",
        "# Symbolic link if it doesn't exist\n",
-        "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt\")==True:\n",
+        "if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt\"):\n",
        "    print(\"❗ Symlink already created\")\n",
        "else: \n",
        "    src = model_path\n",
        "    dst = '/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt'\n",
        "    os.symlink(src, dst) \n",
        "    print(\"✅ Symbolic link created successfully\")"
-      ],
-      "metadata": {
-        "id": "UY-NNz4I8_aG",
-        "cellView": "form"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
    },
    {
      "cell_type": "markdown",
-      "source": [
-        "## ◢ Execution"
-      ],
      "metadata": {
        "id": "Mc28N0_NrCQH"
-      }
+      },
+      "source": [
+        "## ◢ Execution"
+      ]
    },
    {
      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "ir4hCrMIuUpl"
+      },
+      "outputs": [],
      "source": [
        "#@title 9. Run Terminal and Execute Dream bot\n",
        "#@markdown <font color=\"blue\">Steps:</font> <br>\n",
@ -229,24 +212,21 @@
        "#@markdown 3. Example text: `Astronaut floating in a distant galaxy` <br>\n",
        "#@markdown 4. To quit Dream bot use: `q` command.<br>\n",
        "\n",
-        "import gc\n",
-        "%cd /content/stable-diffusion/\n",
        "%load_ext colabxterm\n",
        "%xterm\n",
        "gc.collect()"
-      ],
-      "metadata": {
-        "id": "ir4hCrMIuUpl",
-        "cellView": "form"
-      },
-      "execution_count": null,
-      "outputs": []
+      ]
    },
    {
      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "cellView": "form",
+        "id": "qnLohSHmKoGk"
+      },
+      "outputs": [],
      "source": [
        "#@title 10. Show the last 15 generated images\n",
-        "import gc\n",
        "import glob\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib.image as mpimg\n",
@ -269,13 +249,25 @@
        "    plt.imshow(image)\n",
        "    gc.collect()\n",
        "\n"
+      ]
+    }
  ],
  "metadata": {
-        "cellView": "form",
-        "id": "qnLohSHmKoGk"
+    "accelerator": "GPU",
+    "colab": {
+      "collapsed_sections": [],
+      "private_outputs": true,
+      "provenance": []
    },
-      "execution_count": null,
-      "outputs": []
+    "gpuClass": "standard",
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
    }
-  ]
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
 }
--- a/requirements-colab.txt
+++ b/requirements-colab.txt
@ -1,26 +0,0 @@
-albumentations==0.4.3
-clean-fid==0.1.29
-einops==0.3.0
-huggingface-hub==0.8.1
-imageio-ffmpeg==0.4.2
-imageio==2.9.0
-kornia==0.6.0
-numpy==1.21.6
-omegaconf==2.1.1
-opencv-python==4.6.0.66
-pillow==9.2.0
-pip>=22
-pudb==2019.2
-pytorch-lightning==1.4.2
-streamlit==1.12.0
-taming-transformers-rom1504==0.0.6
-test-tube>=0.7.5
-torch-fidelity==0.3.0
-torchmetrics==0.6.0
-torchtext==0.6.0
-transformers==4.19.2
-torch==1.12.1+cu113
-torchvision==0.13.1+cu113
-git+https://github.com/openai/CLIP.git@main#egg=clip
-git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion
-e .
--- a/requirements-lin-AMD.txt
+++ b/requirements-lin-AMD.txt
@ -0,0 +1,7 @@
+-r requirements.txt
+
+# Get hardware-appropriate torch/torchvision 
+--extra-index-url https://download.pytorch.org/whl/rocm5.1.1 --trusted-host https://download.pytorch.org
+torch
+torchvision
+-e .
--- a/requirements-lin-win-colab-CUDA.txt
+++ b/requirements-lin-win-colab-CUDA.txt
@ -0,0 +1,7 @@
+-r requirements.txt
+
+# Get hardware-appropriate torch/torchvision 
+--extra-index-url https://download.pytorch.org/whl/cu116 --trusted-host https://download.pytorch.org
+torch
+torchvision
+-e .
--- a/requirements-lin.txt
+++ b/requirements-lin.txt
@ -1,33 +0,0 @@
-albumentations==0.4.3
-einops==0.3.0
-huggingface-hub==0.8.1
-imageio-ffmpeg==0.4.2
-imageio==2.9.0
-kornia==0.6.0
-# pip will resolve the version which matches torch
-numpy
-omegaconf==2.1.1
-opencv-python==4.6.0.66
-pillow==9.2.0
-pip>=22
-pudb==2019.2
-pytorch-lightning==1.4.2
-streamlit==1.12.0
-# "CompVis/taming-transformers" doesn't work
-# ldm\models\autoencoder.py", line 6, in <module>
-#    from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer
-# ModuleNotFoundError
-taming-transformers-rom1504==0.0.6
-test-tube>=0.7.5
-torch-fidelity==0.3.0
-torchmetrics==0.6.0
-transformers==4.19.2
-git+https://github.com/openai/CLIP.git@main#egg=clip
-git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion
-git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan
-# No CUDA in PyPi builds
--extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org
-torch==1.11.0
-# Same as numpy - let pip do its thing
-torchvision
-e .
--- a/requirements-mac-MPS-CPU.txt
+++ b/requirements-mac-MPS-CPU.txt
@ -0,0 +1,8 @@
+-r requirements.txt
+
+--pre
+--extra-index-url https://download.pytorch.org/whl/nightly/cpu --trusted-host https://download.pytorch.org
+
+torch
+torchvision
+-e .
--- a/requirements-mac.txt
+++ b/requirements-mac.txt
@ -1,24 +0,0 @@
-albumentations==0.4.3
-einops==0.3.0
-huggingface-hub==0.8.1
-imageio==2.9.0
-imageio-ffmpeg==0.4.2
-kornia==0.6.0
-numpy==1.23.1
--pre torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/nightly/cpu
-omegaconf==2.1.1
-opencv-python==4.6.0.66
-pillow==9.2.0
-pudb==2019.2
-torch==1.12.1
-torchvision==0.13.0
-pytorch-lightning==1.4.2
-streamlit==1.12.0
-test-tube>=0.7.5
-torch-fidelity==0.3.0
-torchmetrics==0.6.0
-transformers==4.19.2
-e git+https://github.com/openai/CLIP.git@main#egg=clip
-e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers
-e git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
-e git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan
--- a/requirements-win.txt
+++ b/requirements-win.txt
@ -1,33 +0,0 @@
-albumentations==0.4.3
-einops==0.3.0
-huggingface-hub==0.8.1
-imageio-ffmpeg==0.4.2
-imageio==2.9.0
-kornia==0.6.0
-# pip will resolve the version which matches torch
-numpy
-omegaconf==2.1.1
-opencv-python==4.6.0.66
-pillow==9.2.0
-pip>=22
-pudb==2019.2
-pytorch-lightning==1.4.2
-streamlit==1.12.0
-# "CompVis/taming-transformers" doesn't work
-# ldm\models\autoencoder.py", line 6, in <module>
-#    from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer
-# ModuleNotFoundError
-taming-transformers-rom1504==0.0.6
-test-tube>=0.7.5
-torch-fidelity==0.3.0
-torchmetrics==0.6.0
-transformers==4.19.2
-git+https://github.com/openai/CLIP.git@main#egg=clip
-git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion
-git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan
-# No CUDA in PyPi builds
--extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org
-torch==1.11.0
-# Same as numpy - let pip do its thing
-torchvision
-e .
--- a/requirements.txt
+++ b/requirements.txt
@ -0,0 +1,27 @@
+--prefer-binary
+
+albumentations
+einops
+huggingface-hub
+imageio-ffmpeg
+imageio
+kornia
+# pip will resolve the version which matches torch
+numpy
+omegaconf
+opencv-python
+pillow
+pip>=22
+pudb
+pytorch-lightning
+streamlit
+# "CompVis/taming-transformers" IS NOT INSTALLABLE
+# This is a drop-in replacement
+taming-transformers-rom1504
+test-tube
+torch-fidelity
+torchmetrics
+transformers
+git+https://github.com/openai/CLIP.git@main#egg=clip
+git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k-diffusion
+git+https://github.com/lstein/GFPGAN@fix-dark-cast-images#egg=gfpgan
--- a/setup.py
+++ b/setup.py
@ -1,8 +1,8 @@
 from setuptools import setup, find_packages

 setup(
-    name='latent-diffusion',
-    version='0.0.1',
+    name='stable-diffusion',
+    version='1.15.0-dev',
    description='',
    packages=find_packages(),
    install_requires=[