mirror of
https://github.com/invoke-ai/InvokeAI
synced 2024-08-30 20:32:17 +00:00
moved the notebook files into their own directory
This commit is contained in:
Lincoln Stein
265
notebooks/Stable-Diffusion-local-Windows.ipynb
Normal file
265
notebooks/Stable-Diffusion-local-Windows.ipynb
Normal file
@ -0,0 +1,265 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Easy-peasy Windows install"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that you will need NVIDIA drivers, Python 3.10, and Git installed\n",
|
||||
"beforehand - simplified\n",
|
||||
"[step-by-step instructions](https://github.com/lstein/stable-diffusion/wiki/Easy-peasy-Windows-install)\n",
|
||||
"are available in the wiki (you'll only need steps 1, 2, & 3 )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Run each cell in turn. In VSCode, either hit SHIFT-ENTER, or click on the little ▶️ to the left of the cell. In Jupyter/JupyterLab, you **must** hit SHIFT-ENTER"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install pew"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%cmd\n",
|
||||
"git clone https://github.com/lstein/stable-diffusion.git"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%cd stable-diffusion"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%writefile requirements.txt\n",
|
||||
"albumentations==0.4.3\n",
|
||||
"einops==0.3.0\n",
|
||||
"huggingface-hub==0.8.1\n",
|
||||
"imageio-ffmpeg==0.4.2\n",
|
||||
"imageio==2.9.0\n",
|
||||
"kornia==0.6.0\n",
|
||||
"# pip will resolve the version which matches torch\n",
|
||||
"numpy\n",
|
||||
"omegaconf==2.1.1\n",
|
||||
"opencv-python==4.6.0.66\n",
|
||||
"pillow==9.2.0\n",
|
||||
"pip>=22\n",
|
||||
"pudb==2019.2\n",
|
||||
"pytorch-lightning==1.4.2\n",
|
||||
"streamlit==1.12.0\n",
|
||||
"# \"CompVis/taming-transformers\" doesn't work\n",
|
||||
"# ldm\\models\\autoencoder.py\", line 6, in <module>\n",
|
||||
"# from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer\n",
|
||||
"# ModuleNotFoundError\n",
|
||||
"taming-transformers-rom1504==0.0.6\n",
|
||||
"test-tube>=0.7.5\n",
|
||||
"torch-fidelity==0.3.0\n",
|
||||
"torchmetrics==0.6.0\n",
|
||||
"transformers==4.19.2\n",
|
||||
"git+https://github.com/openai/CLIP.git@main#egg=clip\n",
|
||||
"git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion\n",
|
||||
"# No CUDA in PyPi builds\n",
|
||||
"--extra-index-url https://download.pytorch.org/whl/cu113 --trusted-host https://download.pytorch.org\n",
|
||||
"torch==1.11.0\n",
|
||||
"# Same as numpy - let pip do its thing\n",
|
||||
"torchvision\n",
|
||||
"-e .\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%cmd\n",
|
||||
"pew new --python 3.10 -r requirements.txt --dont-activate ldm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Switch the notebook kernel to the new 'ldm' environment!\n",
|
||||
"\n",
|
||||
"## VSCode: restart VSCode and come back to this cell\n",
|
||||
"\n",
|
||||
"1. Ctrl+Shift+P\n",
|
||||
"1. Type \"Select Interpreter\" and select \"Jupyter: Select Interpreter to Start Jupyter Server\"\n",
|
||||
"1. VSCode will say that it needs to install packages. Click the \"Install\" button.\n",
|
||||
"1. Once the install is finished, do 1 & 2 again\n",
|
||||
"1. Pick 'ldm'\n",
|
||||
"1. Run the following cell"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%cd stable-diffusion"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"\n",
|
||||
"## Jupyter/JupyterLab\n",
|
||||
"\n",
|
||||
"1. Run the cell below\n",
|
||||
"1. Click on the toolbar where it says \"(ipyknel)\" ↗️. You should get a pop-up asking you to \"Select Kernel\". Pick 'ldm' from the drop-down.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### DO NOT RUN THE FOLLOWING CELL IF YOU ARE USING VSCODE!!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# DO NOT RUN THIS CELL IF YOU ARE USING VSCODE!!\n",
|
||||
"%%cmd\n",
|
||||
"pew workon ldm\n",
|
||||
"pip3 install ipykernel\n",
|
||||
"python -m ipykernel install --name=ldm"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### When running the next cell, Jupyter/JupyterLab users might get a warning saying \"IProgress not found\". This can be ignored."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%run \"scripts/preload_models.py\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%%cmd\n",
|
||||
"mkdir \"models/ldm/stable-diffusion-v1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Now copy the SD model you downloaded from Hugging Face into the above new directory, and (if necessary) rename it to 'model.ckpt'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Now go create some magic!\n",
|
||||
"\n",
|
||||
"VSCode\n",
|
||||
"\n",
|
||||
"- The actual input box for the 'dream' prompt will appear at the very top of the VSCode window. Type in your commands and hit 'ENTER'.\n",
|
||||
"- To quit, hit the 'Interrupt' button in the toolbar up there ⬆️ a couple of times, then hit ENTER (you'll probably see a terrifying traceback from Python - just ignore it).\n",
|
||||
"\n",
|
||||
"Jupyter/JupyterLab\n",
|
||||
"\n",
|
||||
"- The input box for the 'dream' prompt will appear below. Type in your commands and hit 'ENTER'.\n",
|
||||
"- To quit, hit the interrupt button (⏹️) in the toolbar up there ⬆️ a couple of times, then hit ENTER (you'll probably see a terrifying traceback from Python - just ignore it)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%run \"scripts/dream.py\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### Once this seems to be working well, you can try opening a terminal\n",
|
||||
"\n",
|
||||
"- VSCode: type ('CTRL+`')\n",
|
||||
"- Jupyter/JupyterLab: File|New Terminal\n",
|
||||
"- Or jump out of the notebook entirely, and open Powershell/Command Prompt\n",
|
||||
"\n",
|
||||
"Now:\n",
|
||||
"\n",
|
||||
"1. `cd` to wherever the 'stable-diffusion' directory is\n",
|
||||
"1. Run `pew workon ldm`\n",
|
||||
"1. Run `winpty python scripts\\dream.py`"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.10.6 ('ldm')",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "a05e4574567b7bc2c98f7f9aa579f9ea5b8739b54844ab610ac85881c4be2659"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
281
notebooks/Stable_Diffusion_AI_Notebook.ipynb
Normal file
281
notebooks/Stable_Diffusion_AI_Notebook.ipynb
Normal file
@ -0,0 +1,281 @@
|
||||
{
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"provenance": [],
|
||||
"collapsed_sections": [],
|
||||
"private_outputs": true
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
},
|
||||
"accelerator": "GPU",
|
||||
"gpuClass": "standard"
|
||||
},
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"# Stable Diffusion AI Notebook (Release 1.13)\n",
|
||||
"\n",
|
||||
"<img src=\"https://user-images.githubusercontent.com/60411196/186547976-d9de378a-9de8-4201-9c25-c057a9c59bad.jpeg\" alt=\"stable-diffusion-ai\" width=\"170px\"/> <br>\n",
|
||||
"#### Instructions:\n",
|
||||
"1. Execute each cell in order to mount a Dream bot and create images from text. <br>\n",
|
||||
"2. Once cells 1-8 were run correctly you'll be executing a terminal in cell #9, you'll need to enter `python scripts/dream.py` command to run Dream bot.<br> \n",
|
||||
"3. After launching dream bot, you'll see: <br> `Dream > ` in terminal. <br> Insert a command, eg. `Dream > Astronaut floating in a distant galaxy`, or type `-h` for help.\n",
|
||||
"3. After completion you'll see your generated images in path `stable-diffusion/outputs/img-samples/`, you can also show last generated images in cell #10.\n",
|
||||
"4. To quit Dream bot use `q` command. <br> \n",
|
||||
"---\n",
|
||||
"<font color=\"red\">Note:</font> It takes some time to load, but after installing all dependencies you can use the bot all time you want while colab instance is up. <br>\n",
|
||||
"<font color=\"red\">Requirements:</font> For this notebook to work you need to have [Stable-Diffusion-v-1-4](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original) stored in your Google Drive, it will be needed in cell #7\n",
|
||||
"##### For more details visit Github repository: [lstein/stable-diffusion](https://github.com/lstein/stable-diffusion)\n",
|
||||
"---\n"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ycYWcsEKc6w7"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## ◢ Installation"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "dr32VLxlnouf"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 1. Check current GPU assigned\n",
|
||||
"!nvidia-smi -L\n",
|
||||
"!nvidia-smi"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "a2Z5Qu_o8VtQ"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "vbI9ZsQHzjqF"
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"#@title 2. Download stable-diffusion Repository\n",
|
||||
"from os.path import exists\n",
|
||||
"\n",
|
||||
"if exists(\"/content/stable-diffusion/\")==True:\n",
|
||||
" %cd /content/stable-diffusion/\n",
|
||||
" print(\"Already downloaded repo\")\n",
|
||||
"else:\n",
|
||||
" !git clone --quiet https://github.com/lstein/stable-diffusion.git # Original repo\n",
|
||||
" %cd /content/stable-diffusion/\n",
|
||||
" !git checkout --quiet tags/release-1.13"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 3. Install dependencies\n",
|
||||
"import gc\n",
|
||||
"\n",
|
||||
"if exists(\"/content/stable-diffusion/requirements-colab.txt\")==True:\n",
|
||||
" %cd /content/stable-diffusion/\n",
|
||||
" print(\"Already downloaded requirements file\")\n",
|
||||
"else:\n",
|
||||
" !wget https://raw.githubusercontent.com/lstein/stable-diffusion/development/requirements-colab.txt\n",
|
||||
"!pip install colab-xterm\n",
|
||||
"!pip install -r requirements-colab.txt\n",
|
||||
"gc.collect()"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "QbXcGXYEFSNB"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 4. Load small ML models required\n",
|
||||
"%cd /content/stable-diffusion/\n",
|
||||
"!python scripts/preload_models.py\n",
|
||||
"gc.collect()"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "ChIDWxLVHGGJ"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 5. Restart Runtime\n",
|
||||
"exit()"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "8rSMhgnAttQa"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## ◢ Configuration"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "795x1tMoo8b1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 6. Mount google Drive\n",
|
||||
"from google.colab import drive\n",
|
||||
"drive.mount('/content/drive')"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "YEWPV-sF1RDM"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 7. Drive Path to model\n",
|
||||
"#@markdown Path should start with /content/drive/path-to-your-file <br>\n",
|
||||
"#@markdown <font color=\"red\">Note:</font> Model should be downloaded from https://huggingface.co <br>\n",
|
||||
"#@markdown Lastest release: [Stable-Diffusion-v-1-4](https://huggingface.co/CompVis/stable-diffusion-v-1-4-original)\n",
|
||||
"from os.path import exists\n",
|
||||
"\n",
|
||||
"model_path = \"\" #@param {type:\"string\"}\n",
|
||||
"if exists(model_path)==True:\n",
|
||||
" print(\"✅ Valid directory\")\n",
|
||||
"else: \n",
|
||||
" print(\"❌ File doesn't exist\")"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "zRTJeZ461WGu"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 8. Symlink to model\n",
|
||||
"\n",
|
||||
"from os.path import exists\n",
|
||||
"import os \n",
|
||||
"\n",
|
||||
"# Folder creation if it doesn't exist\n",
|
||||
"if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1\")==True:\n",
|
||||
" print(\"❗ Dir stable-diffusion-v1 already exists\")\n",
|
||||
"else:\n",
|
||||
" %mkdir /content/stable-diffusion/models/ldm/stable-diffusion-v1\n",
|
||||
" print(\"✅ Dir stable-diffusion-v1 created\")\n",
|
||||
"\n",
|
||||
"# Symbolic link if it doesn't exist\n",
|
||||
"if exists(\"/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt\")==True:\n",
|
||||
" print(\"❗ Symlink already created\")\n",
|
||||
"else: \n",
|
||||
" src = model_path\n",
|
||||
" dst = '/content/stable-diffusion/models/ldm/stable-diffusion-v1/model.ckpt'\n",
|
||||
" os.symlink(src, dst) \n",
|
||||
" print(\"✅ Symbolic link created successfully\")"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "UY-NNz4I8_aG",
|
||||
"cellView": "form"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"source": [
|
||||
"## ◢ Execution"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "Mc28N0_NrCQH"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 9. Run Terminal and Execute Dream bot\n",
|
||||
"#@markdown <font color=\"blue\">Steps:</font> <br>\n",
|
||||
"#@markdown 1. Execute command `python scripts/dream.py` to run dream bot.<br>\n",
|
||||
"#@markdown 2. After initialized you'll see `Dream>` line.<br>\n",
|
||||
"#@markdown 3. Example text: `Astronaut floating in a distant galaxy` <br>\n",
|
||||
"#@markdown 4. To quit Dream bot use: `q` command.<br>\n",
|
||||
"\n",
|
||||
"import gc\n",
|
||||
"%cd /content/stable-diffusion/\n",
|
||||
"%load_ext colabxterm\n",
|
||||
"%xterm\n",
|
||||
"gc.collect()"
|
||||
],
|
||||
"metadata": {
|
||||
"id": "ir4hCrMIuUpl",
|
||||
"cellView": "form"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"source": [
|
||||
"#@title 10. Show the last 15 generated images\n",
|
||||
"import gc\n",
|
||||
"import glob\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import matplotlib.image as mpimg\n",
|
||||
"%matplotlib inline\n",
|
||||
"\n",
|
||||
"images = []\n",
|
||||
"for img_path in sorted(glob.glob('/content/stable-diffusion/outputs/img-samples/*.png'), reverse=True):\n",
|
||||
" images.append(mpimg.imread(img_path))\n",
|
||||
"\n",
|
||||
"images = images[:15] \n",
|
||||
"\n",
|
||||
"plt.figure(figsize=(20,10))\n",
|
||||
"\n",
|
||||
"columns = 5\n",
|
||||
"for i, image in enumerate(images):\n",
|
||||
" ax = plt.subplot(len(images) / columns + 1, columns, i + 1)\n",
|
||||
" ax.axes.xaxis.set_visible(False)\n",
|
||||
" ax.axes.yaxis.set_visible(False)\n",
|
||||
" ax.axis('off')\n",
|
||||
" plt.imshow(image)\n",
|
||||
" gc.collect()\n",
|
||||
"\n"
|
||||
],
|
||||
"metadata": {
|
||||
"cellView": "form",
|
||||
"id": "qnLohSHmKoGk"
|
||||
},
|
||||
"execution_count": null,
|
||||
"outputs": []
|
||||
}
|
||||
]
|
||||
}
|
||||
271
notebooks/notebook_helpers.py
Normal file
271
notebooks/notebook_helpers.py
Normal file
@ -0,0 +1,271 @@
|
||||
from torchvision.datasets.utils import download_url
|
||||
from ldm.util import instantiate_from_config
|
||||
import torch
|
||||
import os
|
||||
# todo ?
|
||||
from google.colab import files
|
||||
from IPython.display import Image as ipyimg
|
||||
import ipywidgets as widgets
|
||||
from PIL import Image
|
||||
from numpy import asarray
|
||||
from einops import rearrange, repeat
|
||||
import torch, torchvision
|
||||
from ldm.models.diffusion.ddim import DDIMSampler
|
||||
from ldm.util import ismap
|
||||
import time
|
||||
from omegaconf import OmegaConf
|
||||
from ldm.dream.devices import choose_torch_device
|
||||
|
||||
def download_models(mode):
|
||||
|
||||
if mode == "superresolution":
|
||||
# this is the small bsr light model
|
||||
url_conf = 'https://heibox.uni-heidelberg.de/f/31a76b13ea27482981b4/?dl=1'
|
||||
url_ckpt = 'https://heibox.uni-heidelberg.de/f/578df07c8fc04ffbadf3/?dl=1'
|
||||
|
||||
path_conf = 'logs/diffusion/superresolution_bsr/configs/project.yaml'
|
||||
path_ckpt = 'logs/diffusion/superresolution_bsr/checkpoints/last.ckpt'
|
||||
|
||||
download_url(url_conf, path_conf)
|
||||
download_url(url_ckpt, path_ckpt)
|
||||
|
||||
path_conf = path_conf + '/?dl=1' # fix it
|
||||
path_ckpt = path_ckpt + '/?dl=1' # fix it
|
||||
return path_conf, path_ckpt
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def load_model_from_config(config, ckpt):
|
||||
print(f"Loading model from {ckpt}")
|
||||
pl_sd = torch.load(ckpt, map_location="cpu")
|
||||
global_step = pl_sd["global_step"]
|
||||
sd = pl_sd["state_dict"]
|
||||
model = instantiate_from_config(config.model)
|
||||
m, u = model.load_state_dict(sd, strict=False)
|
||||
model.cuda()
|
||||
model.eval()
|
||||
return {"model": model}, global_step
|
||||
|
||||
|
||||
def get_model(mode):
|
||||
path_conf, path_ckpt = download_models(mode)
|
||||
config = OmegaConf.load(path_conf)
|
||||
model, step = load_model_from_config(config, path_ckpt)
|
||||
return model
|
||||
|
||||
|
||||
def get_custom_cond(mode):
|
||||
dest = "data/example_conditioning"
|
||||
|
||||
if mode == "superresolution":
|
||||
uploaded_img = files.upload()
|
||||
filename = next(iter(uploaded_img))
|
||||
name, filetype = filename.split(".") # todo assumes just one dot in name !
|
||||
os.rename(f"{filename}", f"{dest}/{mode}/custom_{name}.{filetype}")
|
||||
|
||||
elif mode == "text_conditional":
|
||||
w = widgets.Text(value='A cake with cream!', disabled=True)
|
||||
display(w)
|
||||
|
||||
with open(f"{dest}/{mode}/custom_{w.value[:20]}.txt", 'w') as f:
|
||||
f.write(w.value)
|
||||
|
||||
elif mode == "class_conditional":
|
||||
w = widgets.IntSlider(min=0, max=1000)
|
||||
display(w)
|
||||
with open(f"{dest}/{mode}/custom.txt", 'w') as f:
|
||||
f.write(w.value)
|
||||
|
||||
else:
|
||||
raise NotImplementedError(f"cond not implemented for mode{mode}")
|
||||
|
||||
|
||||
def get_cond_options(mode):
|
||||
path = "data/example_conditioning"
|
||||
path = os.path.join(path, mode)
|
||||
onlyfiles = [f for f in sorted(os.listdir(path))]
|
||||
return path, onlyfiles
|
||||
|
||||
|
||||
def select_cond_path(mode):
|
||||
path = "data/example_conditioning" # todo
|
||||
path = os.path.join(path, mode)
|
||||
onlyfiles = [f for f in sorted(os.listdir(path))]
|
||||
|
||||
selected = widgets.RadioButtons(
|
||||
options=onlyfiles,
|
||||
description='Select conditioning:',
|
||||
disabled=False
|
||||
)
|
||||
display(selected)
|
||||
selected_path = os.path.join(path, selected.value)
|
||||
return selected_path
|
||||
|
||||
|
||||
def get_cond(mode, selected_path):
|
||||
example = dict()
|
||||
if mode == "superresolution":
|
||||
up_f = 4
|
||||
visualize_cond_img(selected_path)
|
||||
|
||||
c = Image.open(selected_path)
|
||||
c = torch.unsqueeze(torchvision.transforms.ToTensor()(c), 0)
|
||||
c_up = torchvision.transforms.functional.resize(c, size=[up_f * c.shape[2], up_f * c.shape[3]], antialias=True)
|
||||
c_up = rearrange(c_up, '1 c h w -> 1 h w c')
|
||||
c = rearrange(c, '1 c h w -> 1 h w c')
|
||||
c = 2. * c - 1.
|
||||
|
||||
device = choose_torch_device()
|
||||
c = c.to(device)
|
||||
example["LR_image"] = c
|
||||
example["image"] = c_up
|
||||
|
||||
return example
|
||||
|
||||
|
||||
def visualize_cond_img(path):
|
||||
display(ipyimg(filename=path))
|
||||
|
||||
|
||||
def run(model, selected_path, task, custom_steps, resize_enabled=False, classifier_ckpt=None, global_step=None):
|
||||
|
||||
example = get_cond(task, selected_path)
|
||||
|
||||
save_intermediate_vid = False
|
||||
n_runs = 1
|
||||
masked = False
|
||||
guider = None
|
||||
ckwargs = None
|
||||
mode = 'ddim'
|
||||
ddim_use_x0_pred = False
|
||||
temperature = 1.
|
||||
eta = 1.
|
||||
make_progrow = True
|
||||
custom_shape = None
|
||||
|
||||
height, width = example["image"].shape[1:3]
|
||||
split_input = height >= 128 and width >= 128
|
||||
|
||||
if split_input:
|
||||
ks = 128
|
||||
stride = 64
|
||||
vqf = 4 #
|
||||
model.split_input_params = {"ks": (ks, ks), "stride": (stride, stride),
|
||||
"vqf": vqf,
|
||||
"patch_distributed_vq": True,
|
||||
"tie_braker": False,
|
||||
"clip_max_weight": 0.5,
|
||||
"clip_min_weight": 0.01,
|
||||
"clip_max_tie_weight": 0.5,
|
||||
"clip_min_tie_weight": 0.01}
|
||||
else:
|
||||
if hasattr(model, "split_input_params"):
|
||||
delattr(model, "split_input_params")
|
||||
|
||||
invert_mask = False
|
||||
|
||||
x_T = None
|
||||
for n in range(n_runs):
|
||||
if custom_shape is not None:
|
||||
x_T = torch.randn(1, custom_shape[1], custom_shape[2], custom_shape[3]).to(model.device)
|
||||
x_T = repeat(x_T, '1 c h w -> b c h w', b=custom_shape[0])
|
||||
|
||||
logs = make_convolutional_sample(example, model,
|
||||
mode=mode, custom_steps=custom_steps,
|
||||
eta=eta, swap_mode=False , masked=masked,
|
||||
invert_mask=invert_mask, quantize_x0=False,
|
||||
custom_schedule=None, decode_interval=10,
|
||||
resize_enabled=resize_enabled, custom_shape=custom_shape,
|
||||
temperature=temperature, noise_dropout=0.,
|
||||
corrector=guider, corrector_kwargs=ckwargs, x_T=x_T, save_intermediate_vid=save_intermediate_vid,
|
||||
make_progrow=make_progrow,ddim_use_x0_pred=ddim_use_x0_pred
|
||||
)
|
||||
return logs
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def convsample_ddim(model, cond, steps, shape, eta=1.0, callback=None, normals_sequence=None,
|
||||
mask=None, x0=None, quantize_x0=False, img_callback=None,
|
||||
temperature=1., noise_dropout=0., score_corrector=None,
|
||||
corrector_kwargs=None, x_T=None, log_every_t=None
|
||||
):
|
||||
|
||||
ddim = DDIMSampler(model)
|
||||
bs = shape[0] # dont know where this comes from but wayne
|
||||
shape = shape[1:] # cut batch dim
|
||||
print(f"Sampling with eta = {eta}; steps: {steps}")
|
||||
samples, intermediates = ddim.sample(steps, batch_size=bs, shape=shape, conditioning=cond, callback=callback,
|
||||
normals_sequence=normals_sequence, quantize_x0=quantize_x0, eta=eta,
|
||||
mask=mask, x0=x0, temperature=temperature, verbose=False,
|
||||
score_corrector=score_corrector,
|
||||
corrector_kwargs=corrector_kwargs, x_T=x_T)
|
||||
|
||||
return samples, intermediates
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def make_convolutional_sample(batch, model, mode="vanilla", custom_steps=None, eta=1.0, swap_mode=False, masked=False,
|
||||
invert_mask=True, quantize_x0=False, custom_schedule=None, decode_interval=1000,
|
||||
resize_enabled=False, custom_shape=None, temperature=1., noise_dropout=0., corrector=None,
|
||||
corrector_kwargs=None, x_T=None, save_intermediate_vid=False, make_progrow=True,ddim_use_x0_pred=False):
|
||||
log = dict()
|
||||
|
||||
z, c, x, xrec, xc = model.get_input(batch, model.first_stage_key,
|
||||
return_first_stage_outputs=True,
|
||||
force_c_encode=not (hasattr(model, 'split_input_params')
|
||||
and model.cond_stage_key == 'coordinates_bbox'),
|
||||
return_original_cond=True)
|
||||
|
||||
log_every_t = 1 if save_intermediate_vid else None
|
||||
|
||||
if custom_shape is not None:
|
||||
z = torch.randn(custom_shape)
|
||||
print(f"Generating {custom_shape[0]} samples of shape {custom_shape[1:]}")
|
||||
|
||||
z0 = None
|
||||
|
||||
log["input"] = x
|
||||
log["reconstruction"] = xrec
|
||||
|
||||
if ismap(xc):
|
||||
log["original_conditioning"] = model.to_rgb(xc)
|
||||
if hasattr(model, 'cond_stage_key'):
|
||||
log[model.cond_stage_key] = model.to_rgb(xc)
|
||||
|
||||
else:
|
||||
log["original_conditioning"] = xc if xc is not None else torch.zeros_like(x)
|
||||
if model.cond_stage_model:
|
||||
log[model.cond_stage_key] = xc if xc is not None else torch.zeros_like(x)
|
||||
if model.cond_stage_key =='class_label':
|
||||
log[model.cond_stage_key] = xc[model.cond_stage_key]
|
||||
|
||||
with model.ema_scope("Plotting"):
|
||||
t0 = time.time()
|
||||
img_cb = None
|
||||
|
||||
sample, intermediates = convsample_ddim(model, c, steps=custom_steps, shape=z.shape,
|
||||
eta=eta,
|
||||
quantize_x0=quantize_x0, img_callback=img_cb, mask=None, x0=z0,
|
||||
temperature=temperature, noise_dropout=noise_dropout,
|
||||
score_corrector=corrector, corrector_kwargs=corrector_kwargs,
|
||||
x_T=x_T, log_every_t=log_every_t)
|
||||
t1 = time.time()
|
||||
|
||||
if ddim_use_x0_pred:
|
||||
sample = intermediates['pred_x0'][-1]
|
||||
|
||||
x_sample = model.decode_first_stage(sample)
|
||||
|
||||
try:
|
||||
x_sample_noquant = model.decode_first_stage(sample, force_not_quantize=True)
|
||||
log["sample_noquant"] = x_sample_noquant
|
||||
log["sample_diff"] = torch.abs(x_sample_noquant - x_sample)
|
||||
except:
|
||||
pass
|
||||
|
||||
log["sample"] = x_sample
|
||||
log["time"] = t1 - t0
|
||||
|
||||
return log
|
||||
Reference in New Issue
Block a user