mirror of
https://github.com/invoke-ai/InvokeAI
synced 2024-08-30 20:32:17 +00:00
93cba3fba5
* Removed duplicate fix_func for MPS * add support for loading VAE autoencoders To add a VAE autoencoder to an existing model: 1. Download the appropriate autoencoder and put it into models/ldm/stable-diffusion Note that you MUST use a VAE that was written for the original CompViz Stable Diffusion codebase. For v1.4, that would be the file named vae-ft-mse-840000-ema-pruned.ckpt that you can download from https://huggingface.co/stabilityai/sd-vae-ft-mse-original 2. Edit config/models.yaml to contain the following stanza, modifying `weights` and `vae` as required to match the weights and vae model file names. There is no requirement to rename the VAE file. ~~~ stable-diffusion-1.4: weights: models/ldm/stable-diffusion-v1/sd-v1-4.ckpt description: Stable Diffusion v1.4 config: configs/stable-diffusion/v1-inference.yaml vae: models/ldm/stable-diffusion-v1/vae-ft-mse-840000-ema-pruned.ckpt width: 512 height: 512 ~~~ 3. Alternatively from within the `invoke.py` CLI, you may use the command `!editmodel stable-diffusion-1.4` to bring up a simple editor that will allow you to add the path to the VAE. 4. If you are just installing InvokeAI for the first time, you can also use `!import_model models/ldm/stable-diffusion/sd-v1.4.ckpt` instead to create the configuration from scratch. 5. That's it! * ported code refactor changes from PR #1221 - pass a PIL.Image to img2img and inpaint rather than tensor - To support clipseg, inpaint needs to accept an "L" or "1" format mask. Made the appropriate change. * minor fixes to inpaint code 1. If tensors are passed to inpaint as init_image and/or init_mask, then the post-generation image fixup code will be skipped. 2. Post-generation image fixup will work with either a black and white "L" or "RGB" mask, or an "RGBA" mask. Co-authored-by: wfng92 <43742196+wfng92@users.noreply.github.com>
85 lines
3.0 KiB
Python
85 lines
3.0 KiB
Python
'''
|
|
ldm.invoke.generator.img2img descends from ldm.invoke.generator
|
|
'''
|
|
|
|
import torch
|
|
import numpy as np
|
|
import PIL
|
|
from torch import Tensor
|
|
from PIL import Image
|
|
from ldm.invoke.devices import choose_autocast
|
|
from ldm.invoke.generator.base import Generator
|
|
from ldm.models.diffusion.ddim import DDIMSampler
|
|
|
|
class Img2Img(Generator):
|
|
def __init__(self, model, precision):
|
|
super().__init__(model, precision)
|
|
self.init_latent = None # by get_noise()
|
|
|
|
def get_make_image(self,prompt,sampler,steps,cfg_scale,ddim_eta,
|
|
conditioning,init_image,strength,step_callback=None,threshold=0.0,perlin=0.0,**kwargs):
|
|
"""
|
|
Returns a function returning an image derived from the prompt and the initial image
|
|
Return value depends on the seed at the time you call it.
|
|
"""
|
|
self.perlin = perlin
|
|
|
|
sampler.make_schedule(
|
|
ddim_num_steps=steps, ddim_eta=ddim_eta, verbose=False
|
|
)
|
|
|
|
if isinstance(init_image, PIL.Image.Image):
|
|
init_image = self._image_to_tensor(init_image)
|
|
|
|
scope = choose_autocast(self.precision)
|
|
with scope(self.model.device.type):
|
|
self.init_latent = self.model.get_first_stage_encoding(
|
|
self.model.encode_first_stage(init_image)
|
|
) # move to latent space
|
|
|
|
t_enc = int(strength * steps)
|
|
uc, c = conditioning
|
|
|
|
def make_image(x_T):
|
|
# encode (scaled latent)
|
|
z_enc = sampler.stochastic_encode(
|
|
self.init_latent,
|
|
torch.tensor([t_enc]).to(self.model.device),
|
|
noise=x_T
|
|
)
|
|
# decode it
|
|
samples = sampler.decode(
|
|
z_enc,
|
|
c,
|
|
t_enc,
|
|
img_callback = step_callback,
|
|
unconditional_guidance_scale=cfg_scale,
|
|
unconditional_conditioning=uc,
|
|
init_latent = self.init_latent, # changes how noising is performed in ksampler
|
|
)
|
|
|
|
return self.sample_to_image(samples)
|
|
|
|
return make_image
|
|
|
|
def get_noise(self,width,height):
|
|
device = self.model.device
|
|
init_latent = self.init_latent
|
|
assert init_latent is not None,'call to get_noise() when init_latent not set'
|
|
if device.type == 'mps':
|
|
x = torch.randn_like(init_latent, device='cpu').to(device)
|
|
else:
|
|
x = torch.randn_like(init_latent, device=device)
|
|
if self.perlin > 0.0:
|
|
shape = init_latent.shape
|
|
x = (1-self.perlin)*x + self.perlin*self.get_perlin_noise(shape[3], shape[2])
|
|
return x
|
|
|
|
def _image_to_tensor(self, image:Image, normalize:bool=True)->Tensor:
|
|
image = np.array(image).astype(np.float32) / 255.0
|
|
image = image[None].transpose(0, 3, 1, 2)
|
|
image = torch.from_numpy(image)
|
|
if normalize:
|
|
image = 2.0 * image - 1.0
|
|
return image.to(self.model.device)
|