From 28fe84177ed41b171c1fee5d4af2b71989c122ff Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 00:50:28 -0400 Subject: [PATCH 01/14] optionally scale initial image to fit box defined by width x height * This functionality is triggered by the --fit option in the CLI (default false), and by the "fit" checkbox in the WebGUI (default True) * In addition, this commit contains a number of whitespace changes to make the code more readable, as well as an attempt to unify the visual appearance of info and warning messages. --- ldm/dream/image_util.py | 43 ++++----- ldm/dream/pngwriter.py | 2 + ldm/dream/server.py | 14 +-- ldm/gfpgan/gfpgan_tools.py | 10 +-- ldm/simplet2i.py | 175 ++++++++++++++++++++---------------- scripts/dream.py | 9 +- static/dream_web/index.css | 21 ++++- static/dream_web/index.html | 140 +++++++++++++++-------------- 8 files changed, 232 insertions(+), 182 deletions(-) diff --git a/ldm/dream/image_util.py b/ldm/dream/image_util.py index a680562369..2ec7b55834 100644 --- a/ldm/dream/image_util.py +++ b/ldm/dream/image_util.py @@ -8,11 +8,10 @@ class InitImageResizer(): def resize(self,width=None,height=None) -> Image: """ - Return a copy of the image resized to width x height. - The aspect ratio is maintained, with any excess space - filled using black borders (i.e. letterboxed). If - neither width nor height are provided, then returns - a copy of the original image. If one or the other is + Return a copy of the image resized to fit within + a box width x height. The aspect ratio is + maintained. If neither width nor height are provided, + then returns a copy of the original image. If one or the other is provided, then the other will be calculated from the aspect ratio. @@ -20,39 +19,35 @@ class InitImageResizer(): that it can be passed to img2img() """ im = self.image - - if not(width or height): - return im.copy() - - ar = im.width/im.height + + ar = im.width/float(im.height) # Infer missing values from aspect ratio - if not height: # height missing + if not(width or height): # both missing + width = im.width + height = im.height + elif not height: # height missing height = int(width/ar) - if not width: # width missing + elif not width: # width missing width = int(height*ar) # rw and rh are the resizing width and height for the image # they maintain the aspect ratio, but may not completelyl fill up # the requested destination size - (rw,rh) = (width,int(width/ar)) if im.width>=im.height else (int(height*ar),width) + (rw,rh) = (width,int(width/ar)) if im.width>=im.height else (int(height*ar),height) #round everything to multiples of 64 width,height,rw,rh = map( lambda x: x-x%64, (width,height,rw,rh) - ) + ) - # resize the original image so that it fits inside the dest + # no resize necessary, but return a copy + if im.width == width and im.height == height: + return im.copy() + + # otherwise resize the original image so that it fits inside the bounding box resized_image = self.image.resize((rw,rh),resample=Image.Resampling.LANCZOS) - - # create new destination image of specified dimensions - # and paste the resized image into it centered appropriately - new_image = Image.new('RGB',(width,height)) - new_image.paste(resized_image,((width-rw)//2,(height-rh)//2)) - - print(f'>> Resized image size to {width}x{height}') - - return new_image + return resized_image def make_grid(image_list, rows=None, cols=None): image_cnt = len(image_list) diff --git a/ldm/dream/pngwriter.py b/ldm/dream/pngwriter.py index f6b1762883..8dfc09236a 100644 --- a/ldm/dream/pngwriter.py +++ b/ldm/dream/pngwriter.py @@ -61,6 +61,8 @@ class PromptFormatter: switches.append(f'-A{opt.sampler_name or t2i.sampler_name}') if opt.init_img: switches.append(f'-I{opt.init_img}') + if opt.fit: + switches.append(f'--fit') if opt.strength and opt.init_img is not None: switches.append(f'-f{opt.strength or t2i.strength}') if opt.gfpgan_strength: diff --git a/ldm/dream/server.py b/ldm/dream/server.py index 47ca48fc27..f592457e4c 100644 --- a/ldm/dream/server.py +++ b/ldm/dream/server.py @@ -70,6 +70,7 @@ class DreamServer(BaseHTTPRequestHandler): steps = int(post_data['steps']) width = int(post_data['width']) height = int(post_data['height']) + fit = 'fit' in post_data cfgscale = float(post_data['cfgscale']) sampler_name = post_data['sampler'] gfpgan_strength = float(post_data['gfpgan_strength']) if gfpgan_model_exists else 0 @@ -80,7 +81,7 @@ class DreamServer(BaseHTTPRequestHandler): seed = self.model.seed if int(post_data['seed']) == -1 else int(post_data['seed']) self.canceled.clear() - print(f"Request to generate with prompt: {prompt}") + print(f">> Request to generate with prompt: {prompt}") # In order to handle upscaled images, the PngWriter needs to maintain state # across images generated by each call to prompt2img(), so we define it in # the outer scope of image_done() @@ -177,10 +178,13 @@ class DreamServer(BaseHTTPRequestHandler): init_img = "./img2img-tmp.png", strength = strength, iterations = iterations, - cfg_scale = cfgscale, - seed = seed, - steps = steps, + cfg_scale = cfgscale, + seed = seed, + steps = steps, sampler_name = sampler_name, + width = width, + height = height, + fit = fit, gfpgan_strength=gfpgan_strength, upscale = upscale, step_callback=image_progress, @@ -192,8 +196,6 @@ class DreamServer(BaseHTTPRequestHandler): print(f"Canceled.") return - print(f"Prompt generated!") - class ThreadingDreamServer(ThreadingHTTPServer): def __init__(self, server_address): diff --git a/ldm/gfpgan/gfpgan_tools.py b/ldm/gfpgan/gfpgan_tools.py index 29786342a5..ff90a83360 100644 --- a/ldm/gfpgan/gfpgan_tools.py +++ b/ldm/gfpgan/gfpgan_tools.py @@ -14,7 +14,7 @@ model_path = os.path.join(opt.gfpgan_dir, opt.gfpgan_model_path) gfpgan_model_exists = os.path.isfile(model_path) def _run_gfpgan(image, strength, prompt, seed, upsampler_scale=4): - print(f'\n* GFPGAN - Restoring Faces: {prompt} : seed:{seed}') + print(f'>> GFPGAN - Restoring Faces: {prompt} : seed:{seed}') gfpgan = None with warnings.catch_warnings(): warnings.filterwarnings('ignore', category=DeprecationWarning) @@ -41,12 +41,12 @@ def _run_gfpgan(image, strength, prompt, seed, upsampler_scale=4): except Exception: import traceback - print('Error loading GFPGAN:', file=sys.stderr) + print('>> Error loading GFPGAN:', file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) if gfpgan is None: print( - f'GFPGAN not initialized, it must be loaded via the --gfpgan argument' + f'>> GFPGAN not initialized, it must be loaded via the --gfpgan argument' ) return image @@ -129,7 +129,7 @@ def _load_gfpgan_bg_upsampler(bg_upsampler, upsampler_scale, bg_tile=400): def real_esrgan_upscale(image, strength, upsampler_scale, prompt, seed): print( - f'\n* Real-ESRGAN Upscaling: {prompt} : seed:{seed} : scale:{upsampler_scale}x' + f'>> Real-ESRGAN Upscaling: {prompt} : seed:{seed} : scale:{upsampler_scale}x' ) with warnings.catch_warnings(): @@ -143,7 +143,7 @@ def real_esrgan_upscale(image, strength, upsampler_scale, prompt, seed): except Exception: import traceback - print('Error loading Real-ESRGAN:', file=sys.stderr) + print('>> Error loading Real-ESRGAN:', file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) output, img_mode = upsampler.enhance( diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 88cbb6ff78..46297eff70 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -194,29 +194,29 @@ class T2I: return self.prompt2png(prompt, outdir, **kwargs) def prompt2image( - self, - # these are common - prompt, - iterations=None, - steps=None, - seed=None, - cfg_scale=None, - ddim_eta=None, - skip_normalize=False, - image_callback=None, - step_callback=None, - width=None, - height=None, - # these are specific to img2img - init_img=None, - strength=None, - gfpgan_strength=0, - save_original=False, - upscale=None, - variants=None, - sampler_name=None, - log_tokenization=False, - **args, + self, + # these are common + prompt, + iterations = None, + steps = None, + seed = None, + cfg_scale = None, + ddim_eta = None, + skip_normalize = False, + image_callback = None, + step_callback = None, + width = None, + height = None, + # these are specific to img2img + init_img = None, + fit = False, + strength = None, + gfpgan_strength= 0, + save_original = False, + upscale = None, + sampler_name = None, + log_tokenization= False, + **args, ): # eat up additional cruft """ ldm.prompt2image() is the common entry point for txt2img() and img2img() @@ -232,7 +232,6 @@ class T2I: strength // strength for noising/unnoising init_img. 0.0 preserves image exactly, 1.0 replaces it completely gfpgan_strength // strength for GFPGAN. 0.0 preserves image exactly, 1.0 replaces it completely ddim_eta // image randomness (eta=0.0 means the same seed always produces the same image) - variants // if >0, the 1st generated image will be passed back to img2img to generate the requested number of variants step_callback // a function or method that will be called each step image_callback // a function or method that will be called each time an image is generated @@ -251,14 +250,14 @@ class T2I: to create the requested output directory, select a unique informative name for each image, and write the prompt into the PNG metadata. """ - steps = steps or self.steps - seed = seed or self.seed - width = width or self.width - height = height or self.height - cfg_scale = cfg_scale or self.cfg_scale - ddim_eta = ddim_eta or self.ddim_eta + steps = steps or self.steps + seed = seed or self.seed + width = width or self.width + height = height or self.height + cfg_scale = cfg_scale or self.cfg_scale + ddim_eta = ddim_eta or self.ddim_eta iterations = iterations or self.iterations - strength = strength or self.strength + strength = strength or self.strength self.log_tokenization = log_tokenization model = ( @@ -269,9 +268,7 @@ class T2I: 0.0 <= strength <= 1.0 ), 'can only work with strength in [0.0, 1.0]' - if not(width == self.width and height == self.height): - width, height, _ = self._resolution_check(width, height, log=True) - + width, height, _ = self._resolution_check(width, height, log=True) scope = autocast if self.precision == 'autocast' else nullcontext if sampler_name and (sampler_name != self.sampler_name): @@ -295,6 +292,7 @@ class T2I: init_img=init_img, width=width, height=height, + fit=fit, strength=strength, callback=step_callback, ) @@ -312,7 +310,7 @@ class T2I: ) with scope(self.device.type), self.model.ema_scope(): - for n in trange(iterations, desc='Generating'): + for n in trange(iterations, desc='>> Generating'): seed_everything(seed) image = next(images_iterator) results.append([image, seed]) @@ -365,12 +363,12 @@ class T2I: print('Are you sure your system has an adequate NVIDIA GPU?') toc = time.time() - print('Usage stats:') + print('>> Usage stats:') print( - f' {len(results)} image(s) generated in', '%4.2fs' % (toc - tic) + f'>> {len(results)} image(s) generated in', '%4.2fs' % (toc - tic) ) print( - f' Max VRAM used for this generation:', + f'>> Max VRAM used for this generation:', '%4.2fG' % (torch.cuda.max_memory_allocated() / 1e9), ) @@ -379,7 +377,7 @@ class T2I: self.session_peakmem, torch.cuda.max_memory_allocated() ) print( - f' Max VRAM used since script start: ', + f'>> Max VRAM used since script start: ', '%4.2fG' % (self.session_peakmem / 1e9), ) return results @@ -425,18 +423,19 @@ class T2I: @torch.no_grad() def _img2img( - self, - prompt, - precision_scope, - steps, - cfg_scale, - ddim_eta, - skip_normalize, - init_img, - width, - height, - strength, - callback, # Currently not implemented for img2img + self, + prompt, + precision_scope, + steps, + cfg_scale, + ddim_eta, + skip_normalize, + init_img, + width, + height, + fit, + strength, + callback, # Currently not implemented for img2img ): """ An infinite iterator of images from the prompt and the initial image @@ -445,13 +444,13 @@ class T2I: # PLMS sampler not supported yet, so ignore previous sampler if self.sampler_name != 'ddim': print( - f"sampler '{self.sampler_name}' is not yet supported. Using DDIM sampler" + f">> sampler '{self.sampler_name}' is not yet supported. Using DDIM sampler" ) sampler = DDIMSampler(self.model, device=self.device) else: sampler = self.sampler - init_image = self._load_img(init_img, width, height).to(self.device) + init_image = self._load_img(init_img, width, height,fit).to(self.device) with precision_scope(self.device.type): init_latent = self.model.get_first_stage_encoding( self.model.encode_first_stage(init_image) @@ -581,7 +580,7 @@ class T2I: print(msg) def _load_model_from_config(self, config, ckpt): - print(f'Loading model from {ckpt}') + print(f'>> Loading model from {ckpt}') pl_sd = torch.load(ckpt, map_location='cpu') # if "global_step" in pl_sd: # print(f"Global Step: {pl_sd['global_step']}") @@ -596,41 +595,63 @@ class T2I: ) else: print( - 'Using half precision math. Call with --full_precision to use more accurate but VRAM-intensive full precision.' + '>> Using half precision math. Call with --full_precision to use more accurate but VRAM-intensive full precision.' ) model.half() return model - def _load_img(self, path, width, height): - print(f'image path = {path}, cwd = {os.getcwd()}') + def _load_img(self, path, width, height, fit=False): with Image.open(path) as img: image = img.convert('RGB') print( - f'loaded input image of size {image.width}x{image.height} from {path}') - - from ldm.dream.image_util import InitImageResizer - if width == self.width and height == self.height: - new_image_width, new_image_height, resize_needed = self._resolution_check( - image.width, image.height) + f'>> loaded input image of size {image.width}x{image.height} from {path}' + ) + + # The logic here is: + # 1. If "fit" is true, then the image will be fit into the bounding box defined + # by width and height. It will do this in a way that preserves the init image's + # aspect ratio while preventing letterboxing. This means that if there is + # leftover horizontal space after rescaling the image to fit in the bounding box, + # the generated image's width will be reduced to the rescaled init image's width. + # Similarly for the vertical space. + # 2. Otherwise, if "fit" is false, then the image will be scaled, preserving its + # aspect ratio, to the nearest multiple of 64. Large images may generate an + # unexpected OOM error. + if fit: + image = self._fit_image(image,(width,height)) else: - if height == self.height: - new_image_width, new_image_height, resize_needed = self._resolution_check( - width, image.height) - if width == self.width: - new_image_width, new_image_height, resize_needed = self._resolution_check( - image.width, height) - else: - image = InitImageResizer(image).resize(width, height) - resize_needed=False - if resize_needed: - image = InitImageResizer(image).resize( - new_image_width, new_image_height) - + image = self._squeeze_image(image) image = np.array(image).astype(np.float32) / 255.0 image = image[None].transpose(0, 3, 1, 2) image = torch.from_numpy(image) return 2.0 * image - 1.0 + def _squeeze_image(self,image): + x,y,resize_needed = self._resolution_check(image.width,image.height) + if resize_needed: + return InitImageResizer(image).resize(x,y) + return image + + + def _fit_image(self,image,max_dimensions): + w,h = max_dimensions + print( + f'>> image will be resized to fit inside a box {w}x{h} in size.' + ) + if image.width > image.height: + h = None # by setting h to none, we tell InitImageResizer to fit into the width and calculate height + elif image.height > image.width: + w = None # ditto for w + else: + pass + image = InitImageResizer(image).resize(w,h) # note that InitImageResizer does the multiple of 64 truncation internally + print( + f'>> after adjusting image dimensions to be multiples of 64, init image is {image.width}x{image.height}' + ) + return image + + + # TO DO: Move this and related weighted subprompt code into its own module. def _split_weighted_subprompts(text, skip_normalize=False): """ grabs all text up to the first occurrence of ':' @@ -698,6 +719,6 @@ class T2I: f'>> Provided width and height must be multiples of 64. Auto-resizing to {w}x{h}' ) height = h - width = w + width = w resize_needed = True return width, height, resize_needed diff --git a/scripts/dream.py b/scripts/dream.py index 2911e8847a..475da29e4f 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -88,7 +88,7 @@ def main(): tic = time.time() t2i.load_model() print( - f'model loaded in', '%4.2fs' % (time.time() - tic) + f'>> model loaded in', '%4.2fs' % (time.time() - tic) ) if not infile: @@ -483,6 +483,13 @@ def create_cmd_parser(): type=str, help='Path to input image for img2img mode (supersedes width and height)', ) + parser.add_argument( + '-T', + '-fit', + '--fit', + action='store_true', + help='If specified, will resize the input image to fit within the dimensions of width x height (512x512 default)', + ) parser.add_argument( '-f', '--strength', diff --git a/static/dream_web/index.css b/static/dream_web/index.css index 00dc1c5c8d..4f3b0cfadc 100644 --- a/static/dream_web/index.css +++ b/static/dream_web/index.css @@ -8,13 +8,15 @@ margin-top: 20vh; margin-left: auto; margin-right: auto; - max-width: 800px; - + max-width: 1024px; text-align: center; } fieldset { border: none; } +div { + padding: 10px 10px 10px 10px; +} #fieldset-search { display: flex; } @@ -78,3 +80,18 @@ label { cursor: pointer; color: red; } +#txt2img { + background-color: #DCDCDC; +} +#img2img { + background-color: #F5F5F5; +} +#gfpgan { + background-color: #DCDCDC; +} +#progress-section { + background-color: #F5F5F5; +} +#about { + background-color: #DCDCDC; +} diff --git a/static/dream_web/index.html b/static/dream_web/index.html index 77c728963e..bf57afae3f 100644 --- a/static/dream_web/index.html +++ b/static/dream_web/index.html @@ -14,78 +14,84 @@
- -
- - - - - - - - -
- - - - - - - -
+
+ +
+ + + + + + + + +
+ + + + + + + + + + +
+
+ + +
- - - -
- - -
- - - - - - -
+ + +
+
+ + + + + + +
For news and support for this web service, visit our GitHub site
+
✖ From 70119602a0b4516f6029337344d0af63c8af2659 Mon Sep 17 00:00:00 2001 From: James Reynolds Date: Wed, 31 Aug 2022 22:59:20 -0600 Subject: [PATCH 02/14] Issue 270 fix (#274) * check if torch.backends has mps before calling it * Fixes issue 270 Co-authored-by: James Reynolds --- scripts/orig_scripts/img2img.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/orig_scripts/img2img.py b/scripts/orig_scripts/img2img.py index 4bbafcad01..fcd0b8cdfa 100644 --- a/scripts/orig_scripts/img2img.py +++ b/scripts/orig_scripts/img2img.py @@ -200,7 +200,7 @@ def main(): config = OmegaConf.load(f"{opt.config}") model = load_model_from_config(config, f"{opt.ckpt}") - device = choose_torch_device() + device = torch.device(choose_torch_device()) model = model.to(device) if opt.plms: From bf50ab9dd688c52a8727f02613b9843e2edf0a0c Mon Sep 17 00:00:00 2001 From: Jason Toffaletti Date: Wed, 31 Aug 2022 00:32:07 -0700 Subject: [PATCH 03/14] changes to get dream.py working on M1 - move all device init logic to T2I.__init__ - handle m1 specific edge case with autocast device type - check torch.cuda.is_available before using cuda --- environment-mac.yaml | 2 +- ldm/simplet2i.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/environment-mac.yaml b/environment-mac.yaml index be63a05540..42d2d5eaaf 100644 --- a/environment-mac.yaml +++ b/environment-mac.yaml @@ -28,7 +28,7 @@ dependencies: - kornia==0.6.0 - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers - - -e git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion + - -e git+https://github.com/Birch-san/k-diffusion.git@mps#egg=k_diffusion - -e . variables: PYTORCH_ENABLE_MPS_FALLBACK: 1 diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 230aa74c28..cdc72f2fba 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -154,7 +154,10 @@ class T2I: self.model = None # empty for now self.sampler = None self.latent_diffusion_weights = latent_diffusion_weights - self.device = device + if device == 'cuda' and not torch.cuda.is_available(): + device = choose_torch_device() + print("cuda not available, using device", device) + self.device = torch.device(device) # for VRAM usage statistics self.session_peakmem = torch.cuda.max_memory_allocated() if self.device == 'cuda' else None @@ -279,7 +282,8 @@ class T2I: self._set_sampler() tic = time.time() - torch.cuda.torch.cuda.reset_peak_memory_stats() + if torch.cuda.is_available(): + torch.cuda.torch.cuda.reset_peak_memory_stats() results = list() try: @@ -311,7 +315,10 @@ class T2I: callback=step_callback, ) - with scope(self.device.type), self.model.ema_scope(): + device_type = self.device.type # this returns 'mps' on M1 + if device_type != 'cuda' or device_type != 'cpu': + device_type = 'cpu' + with scope(device_type), self.model.ema_scope(): for n in trange(iterations, desc='Generating'): seed_everything(seed) image = next(images_iterator) @@ -523,17 +530,12 @@ class T2I: self.seed = random.randrange(0, np.iinfo(np.uint32).max) return self.seed - def _get_device(self): - device_type = choose_torch_device() - return torch.device(device_type) - def load_model(self): """Load and initialize the model from configuration variables passed at object creation time""" if self.model is None: seed_everything(self.seed) try: config = OmegaConf.load(self.config) - self.device = self._get_device() model = self._load_model_from_config(config, self.weights) if self.embedding_path is not None: model.embedding_manager.load( From 66fe11014892feb9120f4d0a3b0566ad1c77497b Mon Sep 17 00:00:00 2001 From: Jason Toffaletti Date: Wed, 31 Aug 2022 16:57:39 -0700 Subject: [PATCH 04/14] default full_prevision to True for mps device --- scripts/dream.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dream.py b/scripts/dream.py index 2911e8847a..1d6f4d2924 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -9,6 +9,7 @@ import sys import copy import warnings import time +from ldm.dream.devices import choose_torch_device import ldm.dream.readline from ldm.dream.pngwriter import PngWriter, PromptFormatter from ldm.dream.server import DreamServer, ThreadingDreamServer @@ -347,6 +348,8 @@ def create_argv_parser(): dest='full_precision', action='store_true', help='Use slower full precision math for calculations', + # MPS only functions with full precision, see https://github.com/lstein/stable-diffusion/issues/237 + default=choose_torch_device() == 'mps', ) parser.add_argument( '-g', From fa98601bfb6a26fed7b411e1a7bb727635d4b305 Mon Sep 17 00:00:00 2001 From: Jason Toffaletti Date: Wed, 31 Aug 2022 19:24:23 -0700 Subject: [PATCH 05/14] better error reporting for load_model --- ldm/simplet2i.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index cdc72f2fba..c0d2886538 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -544,12 +544,11 @@ class T2I: self.model = model.to(self.device) # model.to doesn't change the cond_stage_model.device used to move the tokenizer output, so set it here self.model.cond_stage_model.device = self.device - except AttributeError: + except AttributeError as e: import traceback - print( - 'Error loading model. Only the CUDA backend is supported', file=sys.stderr) + print(f'Error loading model. {str(e)}', file=sys.stderr) print(traceback.format_exc(), file=sys.stderr) - raise SystemExit + raise SystemExit from e self._set_sampler() From dc30adfbb43524686e208d4b00423cd75a00d919 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 01:09:56 -0400 Subject: [PATCH 06/14] closes #273, crash on M1 machines --- ldm/simplet2i.py | 2 +- scripts/dream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 230aa74c28..9251248994 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -279,7 +279,7 @@ class T2I: self._set_sampler() tic = time.time() - torch.cuda.torch.cuda.reset_peak_memory_stats() + torch.cuda.reset_peak_memory_stats() if self.device == 'cuda' else None results = list() try: diff --git a/scripts/dream.py b/scripts/dream.py index 2911e8847a..85b2ed5211 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -88,7 +88,7 @@ def main(): tic = time.time() t2i.load_model() print( - f'model loaded in', '%4.2fs' % (time.time() - tic) + f'>> model loaded in', '%4.2fs' % (time.time() - tic) ) if not infile: From 09bd9fa47eb96455d29fb575e028cfaad5b39ca7 Mon Sep 17 00:00:00 2001 From: Jason Toffaletti Date: Wed, 31 Aug 2022 22:21:14 -0700 Subject: [PATCH 07/14] move autocast device selection to a function --- ldm/dream/devices.py | 8 +++++++- ldm/simplet2i.py | 6 ++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ldm/dream/devices.py b/ldm/dream/devices.py index 240754dd36..9581abe78c 100644 --- a/ldm/dream/devices.py +++ b/ldm/dream/devices.py @@ -8,4 +8,10 @@ def choose_torch_device() -> str: return 'mps' return 'cpu' - +def choose_autocast_device(device) -> str: + '''Returns an autocast compatible device from a torch device''' + device_type = device.type # this returns 'mps' on M1 + # autocast only supports cuda or cpu + if device_type != 'cuda' or device_type != 'cpu': + return 'cpu' + return device_type diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index c0d2886538..ccecee1a46 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -27,7 +27,7 @@ from ldm.models.diffusion.ddim import DDIMSampler from ldm.models.diffusion.plms import PLMSSampler from ldm.models.diffusion.ksampler import KSampler from ldm.dream.pngwriter import PngWriter -from ldm.dream.devices import choose_torch_device +from ldm.dream.devices import choose_autocast_device, choose_torch_device """Simplified text to image API for stable diffusion/latent diffusion @@ -315,9 +315,7 @@ class T2I: callback=step_callback, ) - device_type = self.device.type # this returns 'mps' on M1 - if device_type != 'cuda' or device_type != 'cpu': - device_type = 'cpu' + device_type = choose_autocast_device(self.device) with scope(device_type), self.model.ema_scope(): for n in trange(iterations, desc='Generating'): seed_everything(seed) From 91565970c2c12e5609f7f57d8f267fc30532f609 Mon Sep 17 00:00:00 2001 From: Cora Johnson-Roberson Date: Wed, 31 Aug 2022 21:18:19 -0400 Subject: [PATCH 08/14] Move environment-mac.yaml to Python 3.9 and patch dream.py for Macs. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I'm using stable-diffusion on a 2022 Macbook M2 Air with 24 GB unified memory. I see this taking about 2.0s/it. I've moved many deps from pip to conda-forge, to take advantage of the precompiled binaries. Some notes for Mac users, since I've seen a lot of confusion about this: One doesn't need the `apple` channel to run this on a Mac-- that's only used by `tensorflow-deps`, required for running tensorflow-metal. For that, I have an example environment.yml here: https://developer.apple.com/forums/thread/711792?answerId=723276022#723276022 However, the `CONDA_ENV=osx-arm64` environment variable *is* needed to ensure that you do not run any Intel-specific packages such as `mkl`, which will fail with [cryptic errors](https://github.com/CompVis/stable-diffusion/issues/25#issuecomment-1226702274) on the ARM architecture and cause the environment to break. I've also added a comment in the env file about 3.10 not working yet. When it becomes possible to update, those commands run on an osx-arm64 machine should work to determine the new version set. Here's what a successful run of dream.py should look like: ``` $ python scripts/dream.py --full_precision  SIGABRT(6) ↵  08:42:59 * Initializing, be patient... Loading model from models/ldm/stable-diffusion-v1/model.ckpt LatentDiffusion: Running in eps-prediction mode DiffusionWrapper has 859.52 M params. making attention of type 'vanilla' with 512 in_channels Working with z of shape (1, 4, 32, 32) = 4096 dimensions. making attention of type 'vanilla' with 512 in_channels Using slower but more accurate full-precision math (--full_precision) >> Setting Sampler to k_lms model loaded in 6.12s * Initialization done! Awaiting your command (-h for help, 'q' to quit) dream> "an astronaut riding a horse" Generating: 0%| | 0/1 [00:00 /tmp/environment-mac-updated.yml + # CONDA_SUBDIR=osx-arm64 $CONDA_CMD env create -f /tmp/environment-mac-updated.yml && $CONDA_CMD list -n ldm-3.10 | awk ' {print " - " $1 "==" $2;} ' + # ``` + # + # Unfortunately, as of 2022-08-31, this fails at the pip stage. + - albumentations==1.2.1 + - coloredlogs==15.0.1 + - einops==0.4.1 + - grpcio==1.46.4 + - humanfriendly + - imageio-ffmpeg==0.4.7 + - imageio==2.21.2 + - imgaug==0.4.0 + - kornia==0.6.7 + - mpmath==1.2.1 + - nomkl + - numpy==1.23.2 + - omegaconf==2.1.1 + - onnx==1.12.0 + - onnxruntime==1.12.1 + - opencv==4.6.0 + - pudb==2022.1 + - pytorch-lightning==1.6.5 + - scipy==1.9.1 + - streamlit==1.12.2 + - sympy==1.10.1 + - tensorboard==2.9.0 + - transformers==4.21.2 - pip: - - albumentations==0.4.6 - - opencv-python==4.6.0.66 - - pudb==2019.2 - - imageio==2.9.0 - - imageio-ffmpeg==0.4.2 - - pytorch-lightning==1.4.2 - - omegaconf==2.1.1 - - test-tube>=0.7.5 - - streamlit==1.12.0 - - pillow==9.2.0 - - einops==0.3.0 - - torch-fidelity==0.3.0 - - transformers==4.19.2 - - torchmetrics==0.6.0 - - kornia==0.6.0 - - -e git+https://github.com/openai/CLIP.git@main#egg=clip + - invisible-watermark + - test-tube + - tokenizers + - torch-fidelity + - -e git+https://github.com/huggingface/diffusers.git@v0.2.4#egg=diffusers - -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers + - -e git+https://github.com/openai/CLIP.git@main#egg=clip - -e git+https://github.com/lstein/k-diffusion.git@master#egg=k-diffusion - -e . variables: diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 230aa74c28..3d3dfde3b8 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -272,14 +272,15 @@ class T2I: if not(width == self.width and height == self.height): width, height, _ = self._resolution_check(width, height, log=True) - scope = autocast if self.precision == 'autocast' else nullcontext + scope = autocast if self.precision == 'autocast' and torch.cuda.is_available() else nullcontext if sampler_name and (sampler_name != self.sampler_name): self.sampler_name = sampler_name self._set_sampler() tic = time.time() - torch.cuda.torch.cuda.reset_peak_memory_stats() + if torch.cuda.is_available(): + torch.cuda.torch.cuda.reset_peak_memory_stats() results = list() try: From c5e95adb499a1d863148fb63ef802c03f2a3f7b7 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 01:09:56 -0400 Subject: [PATCH 09/14] closes #273, crash on M1 machines --- ldm/simplet2i.py | 2 +- scripts/dream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 230aa74c28..9251248994 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -279,7 +279,7 @@ class T2I: self._set_sampler() tic = time.time() - torch.cuda.torch.cuda.reset_peak_memory_stats() + torch.cuda.reset_peak_memory_stats() if self.device == 'cuda' else None results = list() try: diff --git a/scripts/dream.py b/scripts/dream.py index 2911e8847a..85b2ed5211 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -88,7 +88,7 @@ def main(): tic = time.time() t2i.load_model() print( - f'model loaded in', '%4.2fs' % (time.time() - tic) + f'>> model loaded in', '%4.2fs' % (time.time() - tic) ) if not infile: From 2cac4697aafb7749c7cfe8cb17ee420d750468a2 Mon Sep 17 00:00:00 2001 From: Cora Johnson-Roberson Date: Thu, 1 Sep 2022 10:11:14 -0400 Subject: [PATCH 10/14] Correct some verbiage in Mac readme. --- README-Mac-MPS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README-Mac-MPS.md b/README-Mac-MPS.md index 4c078ff07d..e7f6594132 100644 --- a/README-Mac-MPS.md +++ b/README-Mac-MPS.md @@ -12,7 +12,7 @@ issue](https://github.com/CompVis/stable-diffusion/issues/25), and generally on You have to have macOS 12.3 Monterey or later. Anything earlier than that won't work. -Tested on a 2022 Macbook M2 Air with 10-core gpu 24 GB unified memory. +Tested on a 2022 Macbook M2 Air with 10-core GPU and 24 GB unified memory. How to: @@ -31,13 +31,13 @@ python scripts/preload_models.py python scripts/dream.py --full_precision # half-precision requires autocast and won't work ``` -After you follow all the instructions and run txt2img.py you might get several errors. Here's the errors I've seen and found solutions for. +After you follow all the instructions and run dream.py you might get several errors. Here's the errors I've seen and found solutions for. ### Is it slow? Be sure to specify 1 sample and 1 iteration. - python ./scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 + python ./scripts/orig_scripts/txt2img.py --prompt "ocean" --ddim_steps 5 --n_samples 1 --n_iter 1 ### Doesn't work anymore? @@ -215,4 +215,4 @@ This is due to the Intel `mkl` package getting picked up when you try to install something that depends on it-- Rosetta can translate some Intel instructions but not the specialized ones here. To avoid this, make sure to use the environment variable `CONDA_SUBDIR=osx-arm64`, which restricts the Conda environment to only -use ARM packages, and use `nomkl` as described above. \ No newline at end of file +use ARM packages, and use `nomkl` as described above. From 01e05a98dea0e0f20dbae2d005d08c617ded80d4 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 10:16:05 -0400 Subject: [PATCH 11/14] this fixes the inconsistent use of self.device, sometimes a str and sometimes an obj --- ldm/simplet2i.py | 61 ++++++++++++++++++++++++------------------------ scripts/dream.py | 8 ------- 2 files changed, 31 insertions(+), 38 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 9251248994..d27e870e27 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -133,31 +133,31 @@ class T2I: embedding_path=None, # just to keep track of this parameter when regenerating prompt latent_diffusion_weights=False, - device='cuda', ): - self.iterations = iterations - self.width = width - self.height = height - self.steps = steps - self.cfg_scale = cfg_scale - self.weights = weights - self.config = config - self.sampler_name = sampler_name - self.latent_channels = latent_channels - self.downsampling_factor = downsampling_factor - self.grid = grid - self.ddim_eta = ddim_eta - self.precision = precision - self.full_precision = full_precision - self.strength = strength - self.embedding_path = embedding_path - self.model = None # empty for now - self.sampler = None + self.iterations = iterations + self.width = width + self.height = height + self.steps = steps + self.cfg_scale = cfg_scale + self.weights = weights + self.config = config + self.sampler_name = sampler_name + self.latent_channels = latent_channels + self.downsampling_factor = downsampling_factor + self.grid = grid + self.ddim_eta = ddim_eta + self.precision = precision + self.full_precision = full_precision + self.strength = strength + self.embedding_path = embedding_path + self.model = None # empty for now + self.sampler = None + self.device = None self.latent_diffusion_weights = latent_diffusion_weights - self.device = device # for VRAM usage statistics - self.session_peakmem = torch.cuda.max_memory_allocated() if self.device == 'cuda' else None + device_type = choose_torch_device() + self.session_peakmem = torch.cuda.max_memory_allocated() if device_type == 'cuda' else None if seed is None: self.seed = self._new_seed() @@ -251,14 +251,15 @@ class T2I: to create the requested output directory, select a unique informative name for each image, and write the prompt into the PNG metadata. """ - steps = steps or self.steps - seed = seed or self.seed - width = width or self.width - height = height or self.height - cfg_scale = cfg_scale or self.cfg_scale - ddim_eta = ddim_eta or self.ddim_eta - iterations = iterations or self.iterations - strength = strength or self.strength + # TODO: convert this into a getattr() loop + steps = steps or self.steps + seed = seed or self.seed + width = width or self.width + height = height or self.height + cfg_scale = cfg_scale or self.cfg_scale + ddim_eta = ddim_eta or self.ddim_eta + iterations = iterations or self.iterations + strength = strength or self.strength self.log_tokenization = log_tokenization model = ( @@ -279,7 +280,7 @@ class T2I: self._set_sampler() tic = time.time() - torch.cuda.reset_peak_memory_stats() if self.device == 'cuda' else None + torch.cuda.reset_peak_memory_stats() if self.device.type == 'cuda' else None results = list() try: diff --git a/scripts/dream.py b/scripts/dream.py index 85b2ed5211..319b80ce21 100755 --- a/scripts/dream.py +++ b/scripts/dream.py @@ -60,7 +60,6 @@ def main(): # this is solely for recreating the prompt latent_diffusion_weights=opt.laion400m, embedding_path=opt.embedding_path, - device=opt.device, ) # make sure the output directory exists @@ -376,13 +375,6 @@ def create_argv_parser(): type=str, help='Path to a pre-trained embedding manager checkpoint - can only be set on command line', ) - parser.add_argument( - '--device', - '-d', - type=str, - default='cuda', - help='Device to run Stable Diffusion on. Defaults to cuda `torch.cuda.current_device()` if avalible', - ) parser.add_argument( '--prompt_as_dir', '-p', From 2455bb38a479c6dacdf2f459fcb673a951197648 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 10:23:45 -0400 Subject: [PATCH 12/14] Remove redundant chain of types torch->cuda and cuda->torch, so torch.cuda.torch.cuda actually works. However it looks like (and probably is) a typo. --- ldm/simplet2i.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 3d3dfde3b8..9698687157 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -280,7 +280,7 @@ class T2I: tic = time.time() if torch.cuda.is_available(): - torch.cuda.torch.cuda.reset_peak_memory_stats() + torch.cuda.reset_peak_memory_stats() results = list() try: From 9e99fcbc16b57990a39417007e5c07f7744a5077 Mon Sep 17 00:00:00 2001 From: Brent Ozar Date: Thu, 1 Sep 2022 04:52:40 -0700 Subject: [PATCH 13/14] README.md - fixing "further reading" formatting Fixing typo in header and hyperlinking a file. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b1c4898092..38cb46f681 100644 --- a/README.md +++ b/README.md @@ -740,7 +740,7 @@ and [Tesseract Cat](https://github.com/TesseractCat) Original portions of the software are Copyright (c) 2020 Lincoln D. Stein (https://github.com/lstein) -#Further Reading +# Further Reading Please see the original README for more information on this software -and underlying algorithm, located in the file README-CompViz.md. +and underlying algorithm, located in the file [README-CompViz.md](README-CompViz.md). From 833de06299b133b2045185474365b1afea538b10 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Thu, 1 Sep 2022 16:16:46 -0400 Subject: [PATCH 14/14] fix InitImageResizer not found error, closes #294 --- ldm/simplet2i.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ldm/simplet2i.py b/ldm/simplet2i.py index 379ca9cfe6..ecec4fe38f 100644 --- a/ldm/simplet2i.py +++ b/ldm/simplet2i.py @@ -22,12 +22,13 @@ import time import re import sys -from ldm.util import instantiate_from_config -from ldm.models.diffusion.ddim import DDIMSampler -from ldm.models.diffusion.plms import PLMSSampler +from ldm.util import instantiate_from_config +from ldm.models.diffusion.ddim import DDIMSampler +from ldm.models.diffusion.plms import PLMSSampler from ldm.models.diffusion.ksampler import KSampler -from ldm.dream.pngwriter import PngWriter -from ldm.dream.devices import choose_torch_device +from ldm.dream.pngwriter import PngWriter +from ldm.dream.image_util import InitImageResizer +from ldm.dream.devices import choose_torch_device """Simplified text to image API for stable diffusion/latent diffusion