From d334f7f1f6430cf20ed05579aba3000cfc6ae857 Mon Sep 17 00:00:00 2001 From: Lincoln Stein Date: Tue, 28 Feb 2023 00:31:15 -0500 Subject: [PATCH] add missing files --- .gitignore | 9 +- invokeai/models/__init__.py | 10 + invokeai/models/__init__.py~ | 10 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 653 bytes .../__pycache__/autoencoder.cpython-310.pyc | Bin 0 -> 13718 bytes .../__pycache__/model_manager.cpython-310.pyc | Bin 0 -> 33421 bytes invokeai/models/autoencoder.py | 596 +++++ invokeai/models/diffusion/__init__.py | 4 + invokeai/models/diffusion/__init__.py~ | 4 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 304 bytes .../cross_attention_control.cpython-310.pyc | Bin 0 -> 20594 bytes ...cross_attention_map_saving.cpython-310.pyc | Bin 0 -> 3333 bytes .../__pycache__/ddim.cpython-310.pyc | Bin 0 -> 3044 bytes .../__pycache__/ddpm.cpython-310.pyc | Bin 0 -> 48129 bytes .../__pycache__/ksampler.cpython-310.pyc | Bin 0 -> 7538 bytes .../__pycache__/plms.cpython-310.pyc | Bin 0 -> 3913 bytes .../__pycache__/sampler.cpython-310.pyc | Bin 0 -> 9799 bytes .../shared_invokeai_diffusion.cpython-310.pyc | Bin 0 -> 15344 bytes invokeai/models/diffusion/classifier.py | 355 +++ .../diffusion/cross_attention_control.py | 642 +++++ .../diffusion/cross_attention_map_saving.py | 95 + invokeai/models/diffusion/ddim.py | 111 + invokeai/models/diffusion/ddpm.py | 2271 +++++++++++++++++ invokeai/models/diffusion/ksampler.py | 312 +++ invokeai/models/diffusion/plms.py | 146 ++ invokeai/models/diffusion/sampler.py | 450 ++++ .../diffusion/shared_invokeai_diffusion.py | 491 ++++ invokeai/models/model_manager.py | 1221 +++++++++ 28 files changed, 6721 insertions(+), 6 deletions(-) create mode 100644 invokeai/models/__init__.py create mode 100644 invokeai/models/__init__.py~ create mode 100644 invokeai/models/__pycache__/__init__.cpython-310.pyc create mode 100644 invokeai/models/__pycache__/autoencoder.cpython-310.pyc create mode 100644 invokeai/models/__pycache__/model_manager.cpython-310.pyc create mode 100644 invokeai/models/autoencoder.py create mode 100644 invokeai/models/diffusion/__init__.py create mode 100644 invokeai/models/diffusion/__init__.py~ create mode 100644 invokeai/models/diffusion/__pycache__/__init__.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/cross_attention_control.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/cross_attention_map_saving.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/ddim.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/ddpm.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/ksampler.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/plms.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/sampler.cpython-310.pyc create mode 100644 invokeai/models/diffusion/__pycache__/shared_invokeai_diffusion.cpython-310.pyc create mode 100644 invokeai/models/diffusion/classifier.py create mode 100644 invokeai/models/diffusion/cross_attention_control.py create mode 100644 invokeai/models/diffusion/cross_attention_map_saving.py create mode 100644 invokeai/models/diffusion/ddim.py create mode 100644 invokeai/models/diffusion/ddpm.py create mode 100644 invokeai/models/diffusion/ksampler.py create mode 100644 invokeai/models/diffusion/plms.py create mode 100644 invokeai/models/diffusion/sampler.py create mode 100644 invokeai/models/diffusion/shared_invokeai_diffusion.py create mode 100644 invokeai/models/model_manager.py diff --git a/.gitignore b/.gitignore index 9b33e07164..e28b2c432c 100644 --- a/.gitignore +++ b/.gitignore @@ -214,9 +214,9 @@ gfpgan/ configs/models.yaml # weights (will be created by installer) -models/ldm/stable-diffusion-v1/*.ckpt -models/clipseg -models/gfpgan +# models/ldm/stable-diffusion-v1/*.ckpt +# models/clipseg +# models/gfpgan # ignore initfile .invokeai @@ -232,6 +232,3 @@ installer/install.bat installer/install.sh installer/update.bat installer/update.sh - -# no longer stored in source directory -models diff --git a/invokeai/models/__init__.py b/invokeai/models/__init__.py new file mode 100644 index 0000000000..70abd4358e --- /dev/null +++ b/invokeai/models/__init__.py @@ -0,0 +1,10 @@ +''' +Initialization file for the invokeai.models package +''' +from .model_manager import ModelManager, SDLegacyType +from .diffusion import InvokeAIDiffuserComponent +from .diffusion.ddim import DDIMSampler +from .diffusion.ksampler import KSampler +from .diffusion.plms import PLMSSampler +from .diffusion.cross_attention_map_saving import AttentionMapSaver +from .diffusion.shared_invokeai_diffusion import PostprocessingSettings diff --git a/invokeai/models/__init__.py~ b/invokeai/models/__init__.py~ new file mode 100644 index 0000000000..6e060d7fa5 --- /dev/null +++ b/invokeai/models/__init__.py~ @@ -0,0 +1,10 @@ +''' +Initialization file for the invokeai.models package +''' +from .model_manager import ModelManager, SDLegacyType +from .diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent +from .diffusion.ddim import DDIMSampler +from .diffusion.ksampler import KSampler +from .diffusion.plms import PLMSSampler +from .diffusion.cross_attention_map_saving import AttentionMapSaver +from .diffusion.shared_invokeai_diffusion import PostprocessingSettings diff --git a/invokeai/models/__pycache__/__init__.cpython-310.pyc b/invokeai/models/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53157f298f8a4a4dd7ca0a62b4b1346bba613bab GIT binary patch literal 653 zcmZ8e%Zl4D6!jyC9VbpRT@<=Aw3}?=RX3%S4wFsjxCGplHmb0dM2%z#NfXHI=UYmD zqo3hzSN(;7fnGUyQYz!)bB^vc=Snh76A#vpho|b7?|HwS_;&>$KEZ3g0n{GyYQGM+ zF9H;Z5QVh#8g_iZM%?V>Isgh$mu#CLNFXRHP{FcnrNWGy^^^y+v~W zE?GCsGQ!zCu}n7EmT{VGrOxc0W~@2N10`%JWKFrr6sZnmN0UDg-CG3hv~Yew8ZdP` zEmya6N2=4;lcEd1eZB75uh%QK-QF3hZ=_JNp^b&j$!fJON+J}e8f5(W!Y7;CqP)nf zYipqg!WBf7nb1RLzPy+RbDltt{;O787#%tD?WMZX(ilwoaUhw( zh8!Wdj-gbK0e*V=%5eve@AJJBH0Q=r*5n&4zthT^yg%2xkDZ_IC&w@dvluVcsf|B? VdB^ETcUNW(F!1K#)bF3U{}*0p!rA}; literal 0 HcmV?d00001 diff --git a/invokeai/models/__pycache__/autoencoder.cpython-310.pyc b/invokeai/models/__pycache__/autoencoder.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a47741b0d4fb45a304320ffa59c48fc41f4c2b82 GIT binary patch literal 13718 zcmcIrd5k1id9S;wkD2M2J!aRtYq!^n?dF)ZR}hS`F~;_S@p_ECKJZ}D?&(+4)3bf~ zRn2<0(-Rxkk|Uxugea%I$qa|WkO&DWA^8J|BvK$zAQzEHiYOuhl2{=Ll1Vs>z4`s# z>*{OAfkaBT>g(?v_0I2puiH~oSp%ONKKEkvzB>)$>r9M(93bS%R(n(mTkvczim&0r}|xZTJ!vr9RJJB@sEYH6xDy)>=TZeyldSSl#oYs@z1 zmgY?3?S}A0@RT8f&|2Gd&R*J$oQ%jKCmZfYdJob$kw-e;udx?7Q(_u9)1k5Etj(j= zK4G3TN;A)6%*JBLOs1>tRvey*o0V3j7E0t49t^8-Ti&}~X~p$!D36t_#J{6eZHrJQ z{_UYcQ19UF^-8VXsx;nSS&yQ6rFDBfie-IyJ+8M~D4(ylqL^(|;;_6T+s!h%TB+Bn z?0WQ*>nEbm0||{KQy5r4vxWr|X8&vB*AdQ2?rCGi6t3`|$}HKTBd%48VxBXX+|WD6 zGA@a^vUvOT0!L5wAkYZgI7^20444HXLJVFBbTl#ftn)7N^6}K#^yW;@6!w0!Eu1a3 zTbG_i48Sgq}MfbeZct6_S+%4RAYIBe9P=~EJTNcKeQT3O5 z8ML{#mj#^f8Ef+@_jJ4u`2Mxao&^o}vVg$_=bVYRW>D_{$_~g6#Rq$a8vCJmpUA|Q z$5$Z#&}MN9x#vvvj;-EtRh!SE^va&w^F;PM`_G}J!@WGjq}TBRPQpY-YUvUs}>d$bD_KIEpI8_6^0!a+;R7`9M$rvll9V-xA8IC) zu1X2biPC4vX|$s#nVDeSBzUaP#fb}+uC69ltCe`SwOgl;i6pBE$(5&*nNB;3%Oe?1 zd=?91*IRKiBf}b4JCx<+^_3M2+r@-MEAg9c48ARsIkhBZwIpF#tw!Q@B;U=H%hg6D ziZDe?wpD3{!2I38!Jg5M87!8)zP@zkL8m4xMrCu8Y_1y@v7X-%*&Wr zNO!*6YE+g%5VFLwR};K};3o+31XBbH0NbW~E%7}7Ke+kmYP%U8ZD7Ibt)uUf?KOzg z=;)oT)9q8?t#=-+E2L6Cs^%OWt*poG{>CnJ&dPbTF^eZU0g#>ZOnwXaW=(748@#yJ z_mJmZ4D5pGpxgn*x~w;cl;;q?+zhN~mD?@!1=2!FbX5yoq#xqcca+XbqrJ}nhZ*t^ zpt)&*@9iyPi#*FTB$L7+sv#u~#cq!rYF>c!F^z6f7!N{zJ)~Q>QN7`fw4mR*ssEOx z-y+whkLvk>S9tHi@%J7#_$}Fs-TgNfPd;7;bB3ZAx5pnCQ||T>LthWUM5~Jt(PpTJDcv*pp7cU zQ2L>|2=Ls7C;ALPj4grqHX-OO3ik~=hT7nUoijI_o^!kL-n|=c&%*EQhSzhkecqNS zuZB8tHj&E zAjdRxiO8LDIG>k-Q?2rVq#Z2>yVFmh~;vlU9He`NU|^+U^0k$ zHU2^I_OKDg+%1T~;!kZ9DFuqd3Jb^ax<|VMm8S=LtZH?=BFJI2*qtgi>#$JJ2;^3= zTPUu#!ZRJPC8&WSSyJYAr|<4px(iu17Q3?#3~TAEQbrb0lft$lyO*V7 zytL|=wBjoqr(yw)0waW7v}JBkQkgxgk6FYXu{Z1u2Mhp?&pa2%CFabhnWkkteg*0m|y`bUG7VVQlmq zc{=(R^eXQ#daFOqZEHBrZN1Kt`z{|Lm>AET?hfLr+>xQ0!aAzGzy;&j9CNtFmsb5T zM@cIX0!OX!$S99+JO=^#)F9srJh5S?Opemiwo8{UgaE_U(gkDVpzV{cRzw|5njG|R zKsRx>?cT_~ayOH#HmQ`mQb$I~(c9}Wr@Wl_+S*x`t0DG?a857VXgu#euFQn9|DG|`cmm1OR*%-Uuv*bK1e28UIC$W`uUBP9Kq6MjD5a=1W z^;Ty+PIk#KUY9Lyw4ga#uZEFcb4S!yRuZ#J`%UaPKOdwFCKXSi?TC7RubNvHN~k-N z&aYo*Y;LqR`P{}A>G}4+ZA#@pfviE12=kOBXP`b*F4PcIl5*bMEoaS+oi(ZzbJN>` znxdjn76f&@uT&Q-3t$dfgwL`T2>+BR-_XNqwWi|CT6QxRXEyWXV-d{5kJ#e+c|DJP z3wOV9%9i)iqXDmbbOq0Y_c4YU)f?8flv|WEuePm)MJ4v-3{ub2`hg;uYKG0WJX_Y% z`*~8{^K{T)iq@;^c;E^^!3u$Ek(kHfLcq+BopuAN zF#-R?mFg-i7o|vCB~Rr^bg3jm0(FkNhUFPhq73YnK^s*rUHYN1OcGNh^GP6@9k9LT zmIeES&Zx3o6k~W`2V2-6qGPCGVQaD3*iK^&&!)3w2&d-@_j~}$B-ntvx#q(8^w0vo z{&^RcOaLabMSwO=zET?^bKXY_*$p_3XdySzLRK52rR?JUu3KV~Hzp zm>QL{^oOEOqaG)-WJK5@LvB32?Ib}*wrkLMGVvPi8bU0I59Ow|JOC1Q+SOG>gpl(e zWz*-rtUhCpaGS%+H^(Lb@Wh6Cm zIA>&7g?gh>laeoZl~8H8T-sc@62g0`M^%+QHOOYxB;ib@_OzPQN6}vNE`Vu!4z35t zrth>kG)wT!e$UCJd6wzD==hh`%6{L;jMs7gy}KusN@M+v$tL>61H<^8Nc0KK0-2F` z5LNl$tny%#)KWwT%itkcH4eV#Y#PBQjq8o*^2cH8@cS_1Fq9kiV&c$2Ez#VB8(cpF zSx_jSt~BaGnXPL2k~Cg&hnpcUAVWR}kmLu;h-M~Nab}{SFXN494nXT1&kQbRFFH=) zhmO;oPgi2%ox~VpJG!R>wtE?gAIWwDb`#_5X3s+>jj>#xEfE76Wy{feWULHW)rN!M zU!YTh*bHRzp2W>ou6h%S+#N6KSw(~pC-%$7+^ zttyx;io5<%%`jc}`i0joTvy8N#H{jJzGY+66aO^)NP0Fd+{y_0q0>lm<#Io_Tvqf^ znwq4V-(cU$())E{n*`)nxkd010p<8K`4HR+zMRb2o0-;1!R+o!>2i`Q##{_qqGVL14ZZ^eWMOQ;q=XL(Y`3vt z#TEjW<~fA*fWf>|n2i9o85UV!PA_Jpn zlOa%5LL+2G876UDAPzs;#aLYB=g!hoUYnzL%;>YoJZHzV)KcSlHA}l3H49D$%sILr zHLGS2hhBjfDa%5^+BUM)=WDmrzVtrz(Q#?^=E{n>E z`-+1w*uugBvJw2u4C*C;vLZnGNoIxa0bGEyrCrHXBdkcqL=e1# zm7zjhjCN=%##Krrac&irrt*5VKXMBz(A-zflXjzyFiCxtOetLf-P5YT8q%RN+Kow& z4nGOfarGT9`P+Q^cUZ|omr4ooSGDw6mQ6T2yLCu!9g)?6o{RGEXI1dbgG;Q;75wH+ z>tf!rUe4RrOL@ooq2-dAJH7uBU(gR|do4BS;lgpz7-DXnF1KOFdpZyWJA>BT#3Yp%c>@}Jfi@}}%Yhp| zZVQZwes)Vmco&}N?*VwIK!ciY6UIY@2%~wEV%|{(v?ce!(sG4)lhJcQt47Z|XX@Bb z&x6>8F%O%}kvH`GVJcea9|@o>)rx^=lCK<^v@IWPI%w0DP4w|->~a4c+b!gsStwfbLmhu|DQzW>SfO{mlYw~FTI8W+$L$#dP z5I_KxGqA1|cPV{3zWS2J);(k~4ub2lzO# zPZCg-%P$bnwvb;T;MkL?{wYf}Y&2j_45xP!tK3XLalq+C%nA(4A64urKK*U$%^|^&+igMF4Ty<6p&+e&U2_Btro-&pk z=u1!dI5TzCS*Zu^&EUvcpOxN89~Y-M;TYx8m*hi3Eh+*~n!#GIXeHDULg8^JqzWjw zjDvTyXf3J}9>k&cqPkvv#@YHIAXO;e%_7CBYL(QYsim4n!F$Ms?38hl?*7T~?RkL|qEv|qO#UNkqHFSx{5rum2)G5BEfhz5lgVLU zL#qBK6fChrl6L@SDRAao$y~1Y*oQVJKhcK)QVJd3GFSmU;`RWkRNX94cL{jU0F{{Y~?forND^R*u^ACqqJv)*;U+x?5j;x?!7iu$v!%uQY3aI!C``{2oy6= zP{>yiTupEd!L0!r=v0gk+Z8E+y`@7zdiH-YN@&BW9@JII#~iI!SEn@f*(^YF#y zV2^q>k99qp$AQCxU3E}+vAcJSMuuXDB$0kNM|xSe;fXo`|DWfBum$>OgTrIoQm`m* zN5Aq8g0~aA1K_y4lPHbSQXq@Oc{ZxG;oZa*30^~R55c{~WWsdwlF#PmJkIjyD~% zTtPysq8hON=}cmFUr~mC6g_ICG|`h5+V_uXB+f1EN^2xTYa?hNOXd-MN8D}0X(95n zX%WNn!4-x!av9hio^&`$;v>9PX?Ae8qsLL#IbeM*CtJXId55Lt;fnn$>wSx0CzTYT z;1gUx6@>;v(b-eS9DBwWai_x$N{|63y%h=A82|Uf4u%nZef%I}G&Vec(Bkbh@<8s_ zvuK0hrwNo{TO{@*!TSJGGv))#P)wj`YJzm0LFuPC^En`sv3xiT^W*d1Id-4JyBR!+ z_WulsrhSO)LF``ni9F82F%Kf{Z$g=+V)n=ZPUFLY?nBf?Mci?l0J#kGA*9&za=O-S$Tu#6zt1eu8HSK1J{x0OEjeB(ApQ01iZ*e|6Bp7;!FF!nY{GwPgY6VyVQ*>BJTMQ4lV{s+QB5ESVi4q9b30`%K$P$lLk#c#c3NwtjuW`ghnF_{InKVTs5O;=bL|j!X zsF_{Mv2;)Z5#F;0QA>MKeg-|u-vz*t)8~j634Wj84+xGCD8+XRv4aFx5h&YAvCbbN znToLAW6>WGe4b!Jgna>dd!UM=A`r%^$0K<-UIO8J0i$?%yL)(iF%bs9ZR7Fa;Mj^{ z*9MU~`Nv3T9r-83zDO`J8gly)j7EM5$fW%+9*rCK*#gw38o3KkL`kE%1RLyiVIc%}Oohz)sp#542oBe{RETdFjKgWkW~S#e46_CA z5cal>P#g|aRY=aoF9K{mN3#rhLWY=6n(X9$#WtF+QzF{w1wmx)nV zD{@G(>VjywQ9l*Re?u z<9q*O#K7d=a2$V2@Ku7Z5lr;XY2Cmb{|6wGR+kP3mCXCK54&qeA6Bsg%~56B=p~An zR7*)guR}YebgfQ`z;Fk+U9L%7GUNPJj7NTx*l>I|F?ln=zW}r;fvyIE$c~rFpXu*T z9}aSXhZs%nuK zw+{8}2BLyGMr`6vJZwkp9j5V%|3vxF=SwAu%6&JBUMaS7JL!o%F~ZtCaOoI_ZWYIp ze)tB*cn_Wzm|#Ftiwthdw!U7ZL%zfQhdZV=>rvEv2U=Cpd|2}Pcj$cwbAGgM`Cn*b z*suI9lM~&L74Jef@_Rsb>_l7gV?F~r;ll}zUjD;%UHlir{D`S8Ct=cvui-p=aZ!E; zHFxa)3KT4InJMv+PCfDx4(`f{yM=o2<8bM zB~ZeP!`6}4SLC)AP~|qxjQ&Ib=Ms@p8+Wi>$&%83S;P;nf}F*vS9KL45ciLHvL-*x z{9P({q19SgS#POh37kS-$&&0y#?vYT`imEf8}(%sIZh766$Gv?}P>VOg)_M4xzQ&774fTft{Xc?h#%mA%!V@jJmT;J_$qYS^H#wE Ubmna13FC>viMc(-6Z;!66d_y$MJ#9nB*+!1(O|k4JBzvS zx)~RQM&H{Ow#oJOh3)vxH*ah0SlA(D3&`KTaEIiJ%{yE6f-U*6=3TAXg;~j$nmb!} zFWfEp@#a0Pdl&AN{6uqCYxlx#tAu*wteF1XYE?sv5)ZEv)gXn^}L!|t=K0Q z&dAHfQf^Kw`>$$$&YP1`zmDXO{N2i)DTU4l^?KfBL6%VcGPF-m< zn`p3DM-hd)Y_Hj=Imj-mPRn*0-bMS$5_;2iU-pr>cvLCAL)a@D;aIKR?)Wtyqx9x+ zZ5?Z`wD|pW&0i{;p~Vq}S=I8D>z31D`fz(S%%AS^g<3PrJljSF<;M@N_#L-h$3&@* zK7|{(X6M2MOb)UWUcJ_?%IsF1hQgh~>6Uw;cBIo@#6|IFqwX``=ojWxSG(*2o@4r| zM~^*s^2o7wOx^2fg^~q2;kI2>^E>M4M%QgN+AeyYJJIZ%uQk1J@`7XrI*<1#r!eID>e;kUL=MIF$=MGd)&Od$lOg4X<%ZKkZC9Q^@6<&&Ynafbo97 z+3ak=y@K;uXPdJfxuWy8ogL2Y$c^FMJDfX_D>*;t*v?(ZjXSS9v(8TBCY;YXZ#efj z_o9^z1rt|qI%Q`Mt|rmaea>E#nR4cw%4?~GY3GNW1I|I**@zbJcOF1)#(B_r2+uYZ z9@hVyA9jAkdCd7RYS>(OJh}UM=SQ87I7e`Ii@VkNJI--?DW zlyeFtw#RQ39(I1*`GWI3&NH}sn{&oFi^yCiY7MzbE zcc=4~^OEy%|8|c_c%?b zh3|Wvw$s7)E|lmxmr!E2v+gLzL#|x#oomidI#-;R(StqCPdTejfcy73UvYZQ8ghHx zIp-DUd!@As@;~i-pYur>?>=0uJ6E08;;=^uFLc$lpV zW>DjRnsCfn>;RN8pT_41>FZHs{M53s zZeB|nsh-hGRnxsRQp?Y*XM5%~-pTeXq`5{4Y5oFJ&{CwuUIr=XDEe&HjsRqfcJz0$ z-S8W=W+QNIViG~y#9+4Hk-Xk!mDO6SY3nfwLD`#BKs+x)vh|Q`j&iRc^ki%m8>GhW zMCdnp6||ySyIgBDtDq<#L>GX=+MrNfOvz#cL~99@rL|IDvRB$dJG@n(g;w(ZZlKt* z9Z@6nS4fb$?CRzcx~PX>H`?|yhfiUY&1O==1%(z?+J2*n!8Sbma;>@I+Q`(pUDY9B zbt?8zcd@q8^m+T>iAwVBe7O{sd{t|Epsg)ed1?%66mA3+tX_82`HlxfQgxN;D770; z)O~!Mk+}-9(fUudYA;tc;UnsFf%vo0cLvy~elZ-cuP8-iF1N#J&%LyQORQP7vq*Sz zfTu0bZ7%9o%5hWb0CNvAA!gOd-N+GPgG&3=>IZP;O(Q8zW{s>-!bkrr8o{Ph1CU*b z-w2D+4MF&zudctM+M#v6(`nX8*J;}pIWl}3%k879i+ zSr)w;D1!wo20x!H!_5om*vZl$8!@4tAdq-%ledXG1fJ~9dD3~|FbDRju!N`tdz(z@ zbJ)NA)dn*P9+!;bUfI;TuyBv4SGoO?fwgCkJ$wDbv)sbq8v$?^+u)xDz^-CHO=i`# zvE}O*?bXhT(oc2I1?Pns>1#0a*kkrHuCE%v>g@|rCBbIRxM~z7m_`s#n7Pt${G~9v zAJDV#S@m2QQ z!RDcjmsEYd2x@5e?D-DF5AvPyANMPnFWG|J<^b{3(bP*5Yl308b=I*JuNi8pXP6+B zXHu8aK7e^SwVu8PCc-hd5eub*7ZWhVVhY9~I1&d@m}1_E5Wfs|uMG)`-AP^pZffJ2 zK!uWE-yxvb(O;O0URLZp)|(U{Ok;eZ)ogf3FS@IqdIWbz=XX4MQ_t~hC?TwDF*Avo zHE&Ft!SZ%dAg!lx!R@`Cx(00QO*{!9y^pp&$vHIE%$|uEEdrTf z<6}NeC+%3*Akbgh=n!EDdU_<~SeVCba5)+D8OVP(&09o_muhWDt0MMs2l@c$EMo`MY!pe&~hLj4p9#-3O@q6 z6jLIiXeekvgjV2=0H-+soHt|GghFzz6Oq*3%?9V1jf-wMLZ<*rudW*BiKQBTA2{dL zhtT>hfiEKOjrL;a3n=4};uS3myF1u2G+;fyEO)&?fc_Zv>q|pBm4KhdM}|9rjL*F) z4D3}hqQ<}=^$xUaHmf3^>EWB*#!e(G#5>dD4QV>rb0ah+fsb-p&7)Dx6rDy6+<`3q zPTrM-%#b>RW=(VOwfB56Q^tWu39t)8KPZ7v8!OKLgodttM-|YD8D1 zp{wzsD@ghSB`5WjGxd4~b+aZZZKc+7>Mg&}%lXBtCUO>P&f)2FQWvi_4qeUYD`(T| z>9rjCxf%WZr*S_~lkRE12DTQ};B4X8E){#+bX#lp0|g}jl}RTe~2l_$%J7L-y`nwYm>xephc-UBo72i08Fvv#Z>f7h?(;fA7*r zN@PP+G;sAD-q9VCb_v*M%n_|8q%h>6xA_h&t_{}P-`@u@9^3Wn_F6LBh$V#RIYNvy z2HIVwkPMQZpeVU7EQHr@L^ly@sPq*#+3sDEpW|fiN~+?tV^-TiO;onLlwwJzSlqD( z-_^qQa5Isp9`#Swas_xap%zJXn(@Z8E396me3hy+!mI`%wZMxsR!+FjcStCOGcIQ| z2Iq6N#N=@%hmnLi<#s#Oh7(%UScC;{<@{o&iJ_o{WtO_gq{*U%1U~Rgs!0lz$Iyw6 z0{g35+RO6!I)njMrk-ORrnh53-kbIhAl3fEYNwY`77>Mbip$0LDr?&`^5jPi5R#J&x z7%oScKKP&)8n)sB1=eE+_IrV8+hH0Ry>hNRxb0&HT7h-j$M(18%ck;iPhpP8;|mq# z+AD>Gj6U@Hvc8hYB9}zCi&pusP)zhMc`FZ|(lkDR6vC5XT@SXM8q~iGz7>|h1AxL; z#j;aRqmuY9j|^Ehbja}j92pQkDg`i!P(36{inl<{V2YxGLR1GJpO$d>rY4{3Pzm4~ zs_bh}Pk>m0d67FHhH@vz_a4;@tOde)2<=!=YAolR zoRfb&Zau3$fl`HDHYt?{G0r)~*J(av|1wIU#8@vAmq1?%y$sb7d>((wrJyew9F^%w zn8wKF)aQSLDd+>99g-gA2M`8PjTDh-CKt+yw$IY2(`tCw@)xR|i@|18l9Ygf2qFxv zVkkASL^<~;J(3vM8z5>HycQHTXBnt_ewLJf!;TI!LJj{_T|DJjbQm6aRe4wLG_N!?G_t01GS_^ub` z>acm8cU7H(i~oj{Zoh}S0@E-ilezZBAi)B@Ai0`?iqb$J_W}j+*)}}O@EPEc*Y^Mw z$v5ujdA~j76hwEB0UO9C58(-v1TRJeW8xU{>M_KsoySeFg&7@n3yS{~O=!B_&> zK@y(ew=k$L=$tbldI%VTuK|*KX>~pp3jnuMfLnaZ)3wiHN)uG{tPqt^u+?h$8sG#B z7Oa;5F$zV=q*d&5w52XWjUE+kCpFeQ*b4cUEox|LQwe`0OyF6RTXR-z_hnEW5Be_@ zwxjufRCKRk9tdbxRKp)ork$p~)?2C5D7?7Rv^6Qz_TGr=h>W|?rBUeDT0m_wrkNK# zNIKYZ{iTkh)dEViFrxOo#h74;zG*B_(O-tLv9A>F&x)9}8!+rbcTUb$$-o8R^zb2E zMSHoA7lalBcSiRiW?x#t<^)sJaGQ=jtGS(7)dCC(K?rt@78ziHMA;Zsr5O=&9KoZy z53Lo*xl6UnuI)ksb9MJ-$;QmGv$GLrGppN9gk5k$4qF#)h&+(yhIX@NQR5BtHyER4 zD(XS7A9Lfjy3o7~tg2poVmRF^C_QKxAi&ICH)f@o>&A=MjeYZAo7jr*T+LeD1-5}H zmMVELLRvK$sSgKri|RG@@%x#4AK%W%RNjPQYEJYnoGnrw(NwlbLYOtJ8>VT!m$l3r zV0k2OzM8kT8)jPHDdK6~Dw#k#&^TH*MvzTX@sRLNwkD5{P|5#}4E8sz6NcIVS~H*z z!(N5E1)p(uQ#7UdP0jjgQadW~C6|?4ZasGmwARnB7p|q=fa?izMd-~ysqo$yO>|JZ zL*>4fUCW(A%@!1yS0S3U^9NGikt-<2e=o3BT~{8xfl%Aa_tINan^RYbNpufdcmg^q zt(UH)oWgB*?iA&EJicb#z($!KDDNV5bz{K`Pmu-UE|hiFc0Zy7z5b#+*PD0+jtPW= z%Qe*?f92U-?m)bZ-6YC-yAA{*W&8%rU!TG}s>hIk_t*3A=|5w9K52C?{6c)=W!N>7w0 z`ot4-6PosFeZ{G%eRv(T4%u*vc&SvoZtWtCjv&}oGK5|@#?7&+0XeidqA-7@rrOvX zUN}*&!DptLT$j8?tJ}n8Y5MTR*rb*1iG-o4WjI6AUepG3SRK^iIv{$w)#MJe{(bFB zuBB%`O#6-ch>ajNz0rVx=bieBixKkx>K_#Du)Bv@-N#*>EvE#jb zW}2#G{3FjPOfw=^mj3|J6`HV@^TM33`ZXbvEW?LmDPHVU@SMOc=6(S0Q>TUu?! z9pU5K&~`ZRAK=z;QogmEDS=484h;?f-Jp)92XYNbA+=&4pMt$huCE!;n`CguTF${* zl_Lx3=X<7KxN3S&@~&SbQFjV_0)o3B_r{h>U?y?L8FNYvqmjB`LIae>H`rQ#e0>62 zHiz#G+`>+34eTcvf``@$!MH!UJk>K7@lMbBiqR{4#rT}DmOqzj&sr&edi4uNYOQFb zg3bQMUh%52KI4o-yZ4!P9^;$n6~RJm&^Ehs*wU!E_)4*z(cf2$E2&;yeHmNdxYR)F zn|e7x6M2r>`^6_wzcbl0QO6YP5cVduyjlHfFTcJ8v%Hn1+dvHK+u|$C)OMnnwXt4) z`L^X9%eUkEj+xY2saIMXN1vv9CDgaEH;z@2`3G7l^~QT+f*NM#)h;Y-U}Jv~N025Y z06mBuf>|48nPpDgn0$CoH$fii`@ec zUEj#GZ}=0E6+8--{BRo<2lr(-8(?2X3{r<3CCR_CM3z5xvf^O-Xvw~Alm*=dn`WV_ zUIA}F<;HpVyJ6;BC%BWtKN@=twCJYi0HHvs4aK(os2!y3m+kVM;j~~~?L5^{n#ZS; zfXI_t@gU0EX~?;*YQria-V#K|KyPfUN`*<6uUuk6wpqx|Vn* zD01>@8cDc8L#|e*0mbVRak_ZBV;iya5&zLeCVq1Bk=@Q!qQ?_uBy~=goR}2VLmPlj~!hkG+gx- zC**M^hneh5lDnDvF(eML>moA5rxPYM;Ca(b=Z)-*ne?T~xUxnYq4M zd>Lte*$dat*d_6$*ns*pSjr*%O&H-vn4ixh10B5K8^EGP$(hhekN1Gh;VnS6h#L5< z{MIZG@Eb+1#M1%770iSv^BA>15#D~xhd9Nns|J)0>GkZjl=q0A@pI6(r>~l1ka!qAW@-`lPg+Ou# zGBcv819{a;ZIwWD2_AX=GlM&a7FTx65Goj2wKVEWR*j0&gk=@mA=p9a{ER;*=g-+Bv`P~hAc{ECZY>q z+~9~6$!s*;CgR|3BA54G-MePys206+sXhHZPuuGz0 z1Nf%Gam{K~{f>A&m>23$A#|Eepmg;n3TQO=L(KgM638$KtEHK%k>xvvLch%ABU&tG zEMpQoat4bf`i>hThW8;Kj3G5#2CpZp^>euJF{50cNEHTUh29y<;FI7T^mH$c990+h zqan#IO$rLbl zZJ6O;M{(!Qo%Tt#?b*jWD`@OwJ1!R-;lp!>pP4^7f8vmRHj#MfZ9orrU#$v}k7N+f zbz2^L38EhNt`a=VN{$1whFy4@ zFa2+94}M2btW<&-zHtmw#9qZM1Rs$xCc}vDW2QauG9<$6GuNrluCTK*L?M=yVD4x~ zZe6KCdCsNm(B6)Aux9MIOhY_B9}bPx&}hrt%alvyU=i%j#d)X|xeL_>B;iCnO`35F zO9BNDz_1cNd|Ow&(BhJ-O)Q$fyrNWPtfw1P^G^%eh)%&@_Bl>x*O;h>xcIarN(XB6ylw|Hh)lVlqNHZO z(-q}t!ngxa<6c6C7HirE`tpK!8?_VPWFkAQRYc>ms*$#|?d(H`23A;n70LI$)$GM0`9Jyl0N*dx#(ZK3`4^KZE-m%aW z$Ia^+iD5=&Oolk%<2ZnCx1DTt#Vk57g?dIr0}5j50ARILMvesjHMFkpY*V)E};u86_Wj|InWMp3{f zC}q&$zNjI+{WD?%9(9@6D15izg6zA~(1Dap&BI&92&a9^imva`&3gMF4*}KAX~=?1 zx9{?SIif$fY1duxbQR5Mox3~!D|GYRhyF!E4VF90C>` z?Sg?o3Pu2m6k6M8k;|?XpfouR3{!9lbSegJu#hhbw#cfNa0hle!4{%RtbIB?V7vI6 zu?2kuG=cx_5k=(ObTf|lo6!P1r>I7-#VJbKP4mL=cs?qlNF&pz{%E!<0lQEJ$n ztq!ym0zU)DVivNgsP${GmS8WUB6_Jt>d@gTWVkHQcgE7A#|8wp<8g+hL!jP73Gk_i zd_EG+VE}1r;FIQASnouAGpU%KD$ywpRFTX$P>c`MC8rpO&y$94f?GjGHJK4qkYG0H zUes2MQexc*yRihptYK_wY zL3HSi7-en6iBAS9q_h+eR~pG9t!Ox7@SHlLz!uwkkOturjfmF<+Uvw+MH3J;F~H}zt85k zA{mnH$p9v#;gh&<$ohd8K&vthIUl|>%P?|7nx}*~?HfE<63l|_qg%Go-3Dz9JtX08 zgK|6t0(TB4hq7gqwa!7Wg6ll458yh(YkX(<4NZ1#^>J-@;=L?%)n`&ya1^12bIg6y zbnJwz1wlc419(m2B8>+35^J1?2!^?b6RO`=Aw&QkVaq`x00waIoM|js5*;@ND$w@V ziVf8b*u%(NEE@?DW4qp5aooPE;$Vf9=ME(MZ{&V_AoYEoAH4Pb=-_JdFgT@2MZyti zV5&6Q-4(xj9+c>!S2scW&UXK@9@L8d#@Xurzf) znENgo*576WC*TOBtN(#+U}Hf( zg24ydiJnP`x-17i3|k4h4X9h>Nj6rb!I>Kz68Nud`>T+-z&nYwH7_o^dAu{P?=EMZ z0ygZ2SwnP(EfjSak8!9+;iFv1_6{Wa}`y^f`d~dl3m)|iS zEPfrHU9|B!ldosi((T?rEx)PrPU$vqZm9K*q#iLU#r5FbX_(*A?T_R6Mj88z?z3Nt zY8!VpT?02caJ`w=TX^nfGx*Zf@`S%Z_(@Z0OT$!`R=c1sD+8)m~M;7-8^D~XQH}VY1HQLS#YeoK(T2jt!($^i)n_2JcoMTA5@K#&i zh?RI3wcjpEEq#B+x#ObwrlAV{rsd5rqk)%&F>>?zR_0(fOTm~nhFl3yGTtls+m~;X z^7o>iJ6VsP_F-0YY$(;n+Z8FHYkU|b?iwgDD`=N# zBe_?PL?A`Cv-d}TwE^q=0P3`Ddk&?ZI6ffwi2`Kc_+?b)X4W}6ub|m(a4&sU@hdJo~%bF4?UnmglSsbeF4r$I!{pk~^p$^@$8hT+dxCY$JH3Qz~ zd=D?eZU815DMMgTHvw+04PR4e7bDWA=_Gp*GtZjDB~!9n^x;vsjk5NsiglubRz-e~ z8-cP9O++qENgbN!8Pp8ex>`oiC~MyjIZ+#Pngi_n*;ud;T+gy2N{?7s6H5ldWF07;}PgdhEf5VA8o#kM5|cNq5B&cnWQ1DK(RleyC>5A zCp1G0&lut06Y~gxx?-QkO?9Q=X$?)}6xWA3X@_QRN}T{#urkq+_JJXAVRm;Rlghzw zl#TRBWv!hWf-ekC^G28|@a&Na?&jl*ct%Ak;1i}BYOGo~Kt)-E zI%)!cPD$U4Vq_v_C@L|Cq;;gJzMb0~4k#tArV;@D^s?N+_?d-e2jckS)NB!!Z}CZdXi zI|xIEBWKsR&;6BFFLL6xve22Mkc^|;j0R%&KY~4?h;Vy^MoaxF z8#1b*T8N8?A5>(M8kpa3JYE#NCOg@PBzH6SEhG+QwSR?-HA~|+Tv4Fbu;AXZ*`m~j zD>xL=D85~um@ublRY?9@rTnBQO5opc!?ft!GM$D@M+ZctlW811fVUczjxAX_#ayM6MlRy?>06xc?&+D((?g2u9&q3u zBYJuZ<)M%7%a6g6u-JVBEkQbh+ybW?$Uy}vhj2C~+bcRH9x}yS<8dpI`+x;!#J)2@ zUO$C82qzO@MMQg$D=4>iQ8y$j(eKk+IStFgnCSj<>J>aIX%|JbFhx1$9C|M5qw!Vv zyR41D5po94rcwVG$C$LfQJ01bWO{Av71Rso63NjCvOE)9Oqhow5g*vcO0*aZsT@J5 zVi1myBbjNvtCMOskU$_6EW7}p<69((4Q(;!G> zy$Bt4@PV7pOz4;)JdZ>2qfh2AU=6!P`&;b-h(#sX#K?0kxVJer8k5+3VA-^zi+&>9 zP{H1P|Ngm$_IqUsVSvDQ3NCO}u!RDZ!W;zqgkI?L7ho=viZwL?R!3wLv3=0m2bMIr zmoJ>y?pi3kN_+LXJNTJRmFSI*07w z(;(#F>SM2izGFo+)7s%9riPrhgO%s?dO>%nCp5M(4+=O#-QMk1z#NGABCe@VCvwNp;eH^|FYjQbMR)kF@d{UOPvKeg|8xNpeKZto47tL zr6*8d{#En(h#Ok8NCU)aNL5Wd$n#j3EyMc)GSFTOJ2pWtgp-4GL^g4BQ(pnA;4nY1 z;%mT%M^m(M!tThcv|RBBer8K*EqejZ*gZO6-@b;k4cexO`rz*EC*~IE&rq5VHUoH%6PN(`r>Ix-?2~Q4XkpKueG#WS zDcJ#XU&rhxe^>>oVWAUG8GlJ)&!B2A<#`B}5mXeE^gm=?LTj>yFn#|u%DiocW@Ue9 zygah7Vet_d>oL2CYwTM}E;@_~bCRpyqI!o{;$7 zz=FYZh!mWx8KP$PA4GR>)haoEQ_fq-x(vY(t_fxW`h+}%Df?F(j1AQtfrH+FB z0^+W5XCYeWHB^5?cM?p3C!$)?09_g@YeX zIyS_-1Whb_fuuylkKkII199f45SJ+z=QY_$Qv%#BHfqs0i3^TMmk4Il_d;NAum0xjIL!9kpA<6`W!DQEO+<1UTKZ=_#?SDREHoI@r0(7`ONlD7=f zIr||Jo+xxmdlv%FGU8T3R9pvynJNoe+K8Np@C~OE!Sd%Y`PwK#@D&IRC(!5W+0)f$ zjy-+4dh)0~>OBxWOJ*k=Cq_>g$*{!B^N1io%Wj|T_QYLZHe<3QaeAn})aiJ%@kFXc zmU@nWw+Fqj^2FRT;vr5;q-MhNX{%slHoSv7E zX-9Yz83BHvh_iR&Y}BK{#xhtY2@e55e?>9wu8nxI>89J(PQC}wsc@=_$W2wEf;c*Y zmJojz57qA>iQIa_DZ~I4ANe?tnHNrzpYUNPVg%xdHd#EQY#MhOmVao~olmsPEWu)#Qo^1U1=1scs9AfniN zI5RmcA&tHhPJ$_byOrQm5tN2lf(&ZQH#DvU znH7I=?gKcVgy2dDDh+_1FK^I6U~>>I7)4bp8qAI--g054;o;apm`A{XmWcM&G90Pp z1cIm&5J;|}1dUi7&`pFBWUq081v{q&FEk;lv8{F=pfAj`6`X2BI9l{dG!{7wM=0J- ztd9l~;RdL8_!L2cuwTLfg{*Lwo70o{F5#r$WIEV*YS197z4UH^c+{Z*fD9r50Maq6 zZe=t0P%_=x^F@2S#J!LFooK}%h>1+npW)gf#$yls?SwQ235P?0-c4eg!P9^^y~dN6 zu(lwGjokuD!7(OAAD8o+kuaEagca_zlcQ8z=fsm*N3iL4LCTK2dA38fxLp%rN{9(G zk~nn$xr<$KUzX!c9^AZet70ERcSdT^6ev18PCeWpNs{3qV9Ai^+jlOFdQSlpjY)7x z5jS;w$=KA?;HfjXMf7#^`aODzN2k1?r0PWfBtMGf zmdJ3HMdC9ODmGr*At0FqT+kiz2(iRJM;#O#=!VGP4M@SrTu3t>7`#w;KGa0?G#Ioc z*31}of4X}os1q0(7{#GbSxaBqguo2Dtuh{4J?rXYyJ?}8Jvbd)Kf51 zJRl7giRX_pJR8CAB2xD*)Qwv4G|Na}%Pf~ejU|R8amHz4jwoGL6gkTfIjGQa{bEnf zV_Z4{OvTV9sPWQ~sBIr@PoV9*v^{}3^U`)cZW~v?UtqL0NYt}OUrdW}d#qQ4@blnW z31cnv-~a$a^!PfAyuhD7Avro6eEHJrcwV|{tWO}kgrRQr#^LmU%&JzXUds=jR(#JpNcp((VIXoa%&rKcSCQ3h*iIjIT2M3XB=vqKT06_ z6dZARw3E>I%4m4@0rk8-=YT?yT^BkBXkVkF-%F~qa$yOYfP<9>lbR#^d$4kUWxqX} zut&2od&$U>zy{uu9Kk$K!w~FFb3D5Ze{M$BhwQRff=P-hB&k?$az+AD{z}*83;mrC zJbwfN+H;MY_4solTddBBw%k=w1p2wTsQla&-@Y?s5Z>9dWSHodb=iFCBOwc~O^|o4a)}VWW=E2PeTkKcyoH#tNuM z9@Wzw;hw~{2a6tbH}n_P7LW>J-FDQ4TDuW|oK=E*QI*WQ7!VL41L1OSp7u(ZURpWN zkU4qr_eQ`R^}n(IVM==Ru}Ig1Ut55~K2704Nf4knH@64>hY$xjak3bcz&-L)B7twf zhJh4=gXTaAR}Ig^H&BQ-ERVx1wg;#NX0;i7X>y9NoQTwI^Li(08z6=hjsKjOjkqeC z%A4Sk-i4O=U8quS1p7`6lEIb+`tKm62RkuOdT-Ek5|ia%mMYYx@vIG}vQjBeYJbPR z@j&btK$5BOo+?ZmOWJ0JldvxG=V|JIS&C5$fMp?G`1-W({pfm_XOLhSotQ@`QYy|| zXkDo_FXDiX2R^tdBoaC>^)4S}R10sEC&P4|KRKbdCU|Hq%zKeRK^rc{xqjmPn(Kuz+YWP2nHIj09T<4QFHx!inLD{b?>o6Lpyd zXt=2Ir|#;8^{rBY$0%n+Gp0}TfLn*5ArFs!p7Ei?q`wgd;nn!b*60a!Xnb2v2jRK- zjqEs<8*vff^JxUPeTSu1r#PMokwuclxIRHOEPq+#V^2d?!8ibk3e*BAX^Xd+Q4xA3FgotQtqkfk9Qgr#5$AophtdFjaf1RMt+{6J0rJ%A zWAd(;CZgj6Qm$;xtG`6sXZZ^wuzv$`JargDf&OwY?yFy5!fvX6$s_?tn8+?-oQe#h zBADr8uMaTy5Z{lYo^e{#StQ|jxiu64Jd%Ycmp+w7QWs#=D|H5li@LW zg$b)tXPG?0WM`7x&D>{^7?>-MVh<%oM!^d18G@Sk3yw3?eau7OKXDls=pGm#IO-x~ z9vB;7GatvL=>$xvkt_@DLO5T}RFPlL`Z83<0J+|@W(GxHLhXS?ik_THUAi4EzWLRI z+Qs(;TqE=ZYI)m4M4xbK5y;SI1e6PrLRy49XYfvZ>UKZ+r-&N$dFpB8!m;=WEWCQu ziv)pR8`y0Ja)9%pR?PKg%=K6@*GVbN z_MI}@4<@sHr=D%Kz^zh{cnG6&n2|7CGI4qeKhAlqGWi6P5#T$H?r*z!b`Ul|W@mFe zJLzD@FhI6Qzmf>?Lo^o6^n)2wJHT-d$Z-ES;hdhq!5Bzb-|UDykdB`` zej3sE#j%haU0?hFGY1ZhFN#VAil695p%5Dck;K5DNeA(38#or%>AwfvkA4e!9P$}Z z8oKow54u2&1gc6G1&KUMPV%m#quRcQ^VArP-pRMnkEr`F^1Yl)A`(Qvxnp?BM)giZ zK*d)Qy$4r?Vy_RK#2XWRJVQ(b4F#~pK&Yq9gBl7PK+~bj6l0GLq4?m6~1ar!T z&Bplr(@#e9x~U zI^i%`C;p?dL;Cd#n1Gn9Gn9scA`2eXn}9=1YC>MbF?@)hz~&Tpp^>Lv9yTuum$%U# z1|m%i4}lhpE{}46_~Z00@2O`PI{I1)e$83^ebG?Q0Q*4vd}AE;vK+%UFffq=Ak$#h z%b!Uhrr1Y^01fy2LKH-xxL(9N1%GTE(Zkk?PHt_?A76&5%*i8wk_jC5A#m6T90b2( z3WN8rZRm|T#fM?(2NN>BykUJ3!GjR&+8MiQzHyuG1r>6zOiZdD?iJUkdJs*q5zxE% zzW1rYK1$THFAetaQv1>cygAL$__~Kv=y9o+7m;=h>BL(QnR`>cLL3i`^WaRP4-?Xd zjXem+vgJnS;$IK8#Q7;t&5P+v7}g*!*aregVB)huL_GYYT17{z3HI2qp&zL=>nmiS zp{CaN5y8*HF~BG!y(n%x!92$y_>5_t5aV-tau@?HN9Tia))c*sNI<5&JDAijN@c)< z2$Gi5LDI@0GaOy~KAGnFDJ$@JZOzBlgm#;9toi`n2rbeSttI|E3+32P zZsDwMS#xs7xoySVPfkt_rm|*Mkh7GzX-bhk@MB|;9 zUZUVcL*_aDf>(m)X))Bl!bOZtJ|4U!^~*46OCBe$WV<8CpIm34ea<*IDgpB~j-8-1 zbVbnF;6MQKVArle=OsDZ#m0UTG0mXO!p6?S{EcH9YlXp0(<{iv9tsx7dx(7o!~zLn zY<-Ly+AmR4jzbM&(0_}b9JvjwEhe7C>}G;$cADMCVPdOHk9te z_YZ9-+(XQsAQx|#z4qU*V+Q?tf)b32{XLQ2Jp}$!19Hd1nEx=JSl05g8|Wt^5G)%a z59YJK6;z9$4J4gX+zG19%MKIrfvU@#0Q$HFbZ!R-vNfoN*p^yFg5S4-u0|wT#3IJc z6(%3y`-E|InU}YK;BcZ^hwc~FC_T0sf%yTvK~Thx;ut%?&Snw$X#kKT>6Ku-Rn8yA zAU-0M4%e(miH5in-WbD2$P*_AzqVioJWdWqmXM1$c119&#wxr7_zwp0;%Vn7Q~e~SR9!=&W4t{T#Y;(alllxdz2((UvFJ}T z`57i(W4;$P@UmDk;{LUIRR-g0q)`@ z*IWHOlYhh{k?CU(i=r5)#mExonU^SOMC<6X{8?^gM0#b69Q*;08?kPs9Si-^O=8eLC(;3QVK z@(5)FQ}Pg01FCB(1d>CbYZsk14WSvdFd9~^>TOEi)2Hoy3L0 zISef%+V2aWyK&LOu0@y&Mxz63yl#L6hUFJD4hj)Op@Z3+Y%yG}Ucjr+-JoZqS}XP2 zy!Y>!>`aoonOj5hD;x{qX9nPiUve7$39=;IdL1Q({?fP-aHC=(HgeMcp*t>DptHOJ z)=1BXxhIZSpE!N$SUBEU%cg_A?4H;{wQ6Mxuv zf56-SmPx<8KjfugNg_R6Gmp?{XzEW#Mh<{TMH4#6fm9?jD>-@neJdZSGMe|_Pnwj5 zw&JpEozPml^-IFPlN-qo) z9L$r%p$ivT`Zt(JX^yCTKygc|+nLCEyVeXHaqe)$zVe&OzewNAaFgecTK0bJYxi^{o5|dYWkMuVz zQT%JvqZVJ~L!vlE6mm=4axb2861>kN$*17VNRx_Dx?bE~%oXn|7K^1~zL+hJ7e7)O zH*YW9DDKSPReZcSlb0QNb=6(>8Nz{YiI8^Lm`Qb2Etva2${!#J86&|ClRuwft zisVE6E_1@o6P>DW^YW`q9^)kkr+yR(g3iG!Oc8a{@ZnPmmi<6ccm==vD<(4i`x#ki zk09Ht9AC~eDKNRgcc=P(r5-{xz(ZI2U(K-uV-jl)Y>c^|;|m)`Om{;5!-D+o2Ay5B zQ_xIuU7>fV_JWb05(=mLK^H4Lum@3FN7|bnaA?=RBJ62+-w_?R?I=_h`x3^HZgr&J z!FToV%X^xqoME5ytXU>y6UVWc$rdKWQ0h%i>^(@rtd8L!D|IU%3-gbF?&D|RxfUsp m;BG;^$}pP4|Fj^a18njMuakcaz5;FkWDX~U@H+rA*8c&RJEf2S literal 0 HcmV?d00001 diff --git a/invokeai/models/autoencoder.py b/invokeai/models/autoencoder.py new file mode 100644 index 0000000000..3db7b6fd73 --- /dev/null +++ b/invokeai/models/autoencoder.py @@ -0,0 +1,596 @@ +import torch +import pytorch_lightning as pl +import torch.nn.functional as F +from contextlib import contextmanager + +from taming.modules.vqvae.quantize import VectorQuantizer2 as VectorQuantizer + +from ldm.modules.diffusionmodules.model import Encoder, Decoder +from ldm.modules.distributions.distributions import ( + DiagonalGaussianDistribution, +) + +from ldm.util import instantiate_from_config + + +class VQModel(pl.LightningModule): + def __init__( + self, + ddconfig, + lossconfig, + n_embed, + embed_dim, + ckpt_path=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + batch_resize_range=None, + scheduler_config=None, + lr_g_factor=1.0, + remap=None, + sane_index_shape=False, # tell vector quantizer to return indices as bhw + use_ema=False, + ): + super().__init__() + self.embed_dim = embed_dim + self.n_embed = n_embed + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + self.loss = instantiate_from_config(lossconfig) + self.quantize = VectorQuantizer( + n_embed, + embed_dim, + beta=0.25, + remap=remap, + sane_index_shape=sane_index_shape, + ) + self.quant_conv = torch.nn.Conv2d(ddconfig['z_channels'], embed_dim, 1) + self.post_quant_conv = torch.nn.Conv2d( + embed_dim, ddconfig['z_channels'], 1 + ) + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer( + 'colorize', torch.randn(3, colorize_nlabels, 1, 1) + ) + if monitor is not None: + self.monitor = monitor + self.batch_resize_range = batch_resize_range + if self.batch_resize_range is not None: + print( + f'{self.__class__.__name__}: Using per-batch resizing in range {batch_resize_range}.' + ) + + self.use_ema = use_ema + if self.use_ema: + self.model_ema = LitEma(self) + print(f'>> Keeping EMAs of {len(list(self.model_ema.buffers()))}.') + + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + self.scheduler_config = scheduler_config + self.lr_g_factor = lr_g_factor + + @contextmanager + def ema_scope(self, context=None): + if self.use_ema: + self.model_ema.store(self.parameters()) + self.model_ema.copy_to(self) + if context is not None: + print(f'{context}: Switched to EMA weights') + try: + yield None + finally: + if self.use_ema: + self.model_ema.restore(self.parameters()) + if context is not None: + print(f'{context}: Restored training weights') + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + missing, unexpected = self.load_state_dict(sd, strict=False) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + print(f'Unexpected Keys: {unexpected}') + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self) + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + quant, emb_loss, info = self.quantize(h) + return quant, emb_loss, info + + def encode_to_prequant(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return h + + def decode(self, quant): + quant = self.post_quant_conv(quant) + dec = self.decoder(quant) + return dec + + def decode_code(self, code_b): + quant_b = self.quantize.embed_code(code_b) + dec = self.decode(quant_b) + return dec + + def forward(self, input, return_pred_indices=False): + quant, diff, (_, _, ind) = self.encode(input) + dec = self.decode(quant) + if return_pred_indices: + return dec, diff, ind + return dec, diff + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = ( + x.permute(0, 3, 1, 2) + .to(memory_format=torch.contiguous_format) + .float() + ) + if self.batch_resize_range is not None: + lower_size = self.batch_resize_range[0] + upper_size = self.batch_resize_range[1] + if self.global_step <= 4: + # do the first few batches with max size to avoid later oom + new_resize = upper_size + else: + new_resize = np.random.choice( + np.arange(lower_size, upper_size + 16, 16) + ) + if new_resize != x.shape[2]: + x = F.interpolate(x, size=new_resize, mode='bicubic') + x = x.detach() + return x + + def training_step(self, batch, batch_idx, optimizer_idx): + # https://github.com/pytorch/pytorch/issues/37142 + # try not to fool the heuristics + x = self.get_input(batch, self.image_key) + xrec, qloss, ind = self(x, return_pred_indices=True) + + if optimizer_idx == 0: + # autoencode + aeloss, log_dict_ae = self.loss( + qloss, + x, + xrec, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train', + predicted_indices=ind, + ) + + self.log_dict( + log_dict_ae, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=True, + ) + return aeloss + + if optimizer_idx == 1: + # discriminator + discloss, log_dict_disc = self.loss( + qloss, + x, + xrec, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train', + ) + self.log_dict( + log_dict_disc, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=True, + ) + return discloss + + def validation_step(self, batch, batch_idx): + log_dict = self._validation_step(batch, batch_idx) + with self.ema_scope(): + log_dict_ema = self._validation_step( + batch, batch_idx, suffix='_ema' + ) + return log_dict + + def _validation_step(self, batch, batch_idx, suffix=''): + x = self.get_input(batch, self.image_key) + xrec, qloss, ind = self(x, return_pred_indices=True) + aeloss, log_dict_ae = self.loss( + qloss, + x, + xrec, + 0, + self.global_step, + last_layer=self.get_last_layer(), + split='val' + suffix, + predicted_indices=ind, + ) + + discloss, log_dict_disc = self.loss( + qloss, + x, + xrec, + 1, + self.global_step, + last_layer=self.get_last_layer(), + split='val' + suffix, + predicted_indices=ind, + ) + rec_loss = log_dict_ae[f'val{suffix}/rec_loss'] + self.log( + f'val{suffix}/rec_loss', + rec_loss, + prog_bar=True, + logger=True, + on_step=False, + on_epoch=True, + sync_dist=True, + ) + self.log( + f'val{suffix}/aeloss', + aeloss, + prog_bar=True, + logger=True, + on_step=False, + on_epoch=True, + sync_dist=True, + ) + if version.parse(pl.__version__) >= version.parse('1.4.0'): + del log_dict_ae[f'val{suffix}/rec_loss'] + self.log_dict(log_dict_ae) + self.log_dict(log_dict_disc) + return self.log_dict + + def configure_optimizers(self): + lr_d = self.learning_rate + lr_g = self.lr_g_factor * self.learning_rate + print('lr_d', lr_d) + print('lr_g', lr_g) + opt_ae = torch.optim.Adam( + list(self.encoder.parameters()) + + list(self.decoder.parameters()) + + list(self.quantize.parameters()) + + list(self.quant_conv.parameters()) + + list(self.post_quant_conv.parameters()), + lr=lr_g, + betas=(0.5, 0.9), + ) + opt_disc = torch.optim.Adam( + self.loss.discriminator.parameters(), lr=lr_d, betas=(0.5, 0.9) + ) + + if self.scheduler_config is not None: + scheduler = instantiate_from_config(self.scheduler_config) + + print('Setting up LambdaLR scheduler...') + scheduler = [ + { + 'scheduler': LambdaLR( + opt_ae, lr_lambda=scheduler.schedule + ), + 'interval': 'step', + 'frequency': 1, + }, + { + 'scheduler': LambdaLR( + opt_disc, lr_lambda=scheduler.schedule + ), + 'interval': 'step', + 'frequency': 1, + }, + ] + return [opt_ae, opt_disc], scheduler + return [opt_ae, opt_disc], [] + + def get_last_layer(self): + return self.decoder.conv_out.weight + + def log_images(self, batch, only_inputs=False, plot_ema=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if only_inputs: + log['inputs'] = x + return log + xrec, _ = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['inputs'] = x + log['reconstructions'] = xrec + if plot_ema: + with self.ema_scope(): + xrec_ema, _ = self(x) + if x.shape[1] > 3: + xrec_ema = self.to_rgb(xrec_ema) + log['reconstructions_ema'] = xrec_ema + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer( + 'colorize', torch.randn(3, x.shape[1], 1, 1).to(x) + ) + x = F.conv2d(x, weight=self.colorize) + x = 2.0 * (x - x.min()) / (x.max() - x.min()) - 1.0 + return x + + +class VQModelInterface(VQModel): + def __init__(self, embed_dim, *args, **kwargs): + super().__init__(embed_dim=embed_dim, *args, **kwargs) + self.embed_dim = embed_dim + + def encode(self, x): + h = self.encoder(x) + h = self.quant_conv(h) + return h + + def decode(self, h, force_not_quantize=False): + # also go through quantization layer + if not force_not_quantize: + quant, emb_loss, info = self.quantize(h) + else: + quant = h + quant = self.post_quant_conv(quant) + dec = self.decoder(quant) + return dec + + +class AutoencoderKL(pl.LightningModule): + def __init__( + self, + ddconfig, + lossconfig, + embed_dim, + ckpt_path=None, + ignore_keys=[], + image_key='image', + colorize_nlabels=None, + monitor=None, + ): + super().__init__() + self.image_key = image_key + self.encoder = Encoder(**ddconfig) + self.decoder = Decoder(**ddconfig) + self.loss = instantiate_from_config(lossconfig) + assert ddconfig['double_z'] + self.quant_conv = torch.nn.Conv2d( + 2 * ddconfig['z_channels'], 2 * embed_dim, 1 + ) + self.post_quant_conv = torch.nn.Conv2d( + embed_dim, ddconfig['z_channels'], 1 + ) + self.embed_dim = embed_dim + if colorize_nlabels is not None: + assert type(colorize_nlabels) == int + self.register_buffer( + 'colorize', torch.randn(3, colorize_nlabels, 1, 1) + ) + if monitor is not None: + self.monitor = monitor + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys=ignore_keys) + + def init_from_ckpt(self, path, ignore_keys=list()): + sd = torch.load(path, map_location='cpu')['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + self.load_state_dict(sd, strict=False) + print(f'Restored from {path}') + + def encode(self, x): + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + return posterior + + def decode(self, z): + z = self.post_quant_conv(z) + dec = self.decoder(z) + return dec + + def forward(self, input, sample_posterior=True): + posterior = self.encode(input) + if sample_posterior: + z = posterior.sample() + else: + z = posterior.mode() + dec = self.decode(z) + return dec, posterior + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = ( + x.permute(0, 3, 1, 2) + .to(memory_format=torch.contiguous_format) + .float() + ) + return x + + def training_step(self, batch, batch_idx, optimizer_idx): + inputs = self.get_input(batch, self.image_key) + reconstructions, posterior = self(inputs) + + if optimizer_idx == 0: + # train encoder+decoder+logvar + aeloss, log_dict_ae = self.loss( + inputs, + reconstructions, + posterior, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train', + ) + self.log( + 'aeloss', + aeloss, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=True, + ) + self.log_dict( + log_dict_ae, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=False, + ) + return aeloss + + if optimizer_idx == 1: + # train the discriminator + discloss, log_dict_disc = self.loss( + inputs, + reconstructions, + posterior, + optimizer_idx, + self.global_step, + last_layer=self.get_last_layer(), + split='train', + ) + + self.log( + 'discloss', + discloss, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=True, + ) + self.log_dict( + log_dict_disc, + prog_bar=False, + logger=True, + on_step=True, + on_epoch=False, + ) + return discloss + + def validation_step(self, batch, batch_idx): + inputs = self.get_input(batch, self.image_key) + reconstructions, posterior = self(inputs) + aeloss, log_dict_ae = self.loss( + inputs, + reconstructions, + posterior, + 0, + self.global_step, + last_layer=self.get_last_layer(), + split='val', + ) + + discloss, log_dict_disc = self.loss( + inputs, + reconstructions, + posterior, + 1, + self.global_step, + last_layer=self.get_last_layer(), + split='val', + ) + + self.log('val/rec_loss', log_dict_ae['val/rec_loss']) + self.log_dict(log_dict_ae) + self.log_dict(log_dict_disc) + return self.log_dict + + def configure_optimizers(self): + lr = self.learning_rate + opt_ae = torch.optim.Adam( + list(self.encoder.parameters()) + + list(self.decoder.parameters()) + + list(self.quant_conv.parameters()) + + list(self.post_quant_conv.parameters()), + lr=lr, + betas=(0.5, 0.9), + ) + opt_disc = torch.optim.Adam( + self.loss.discriminator.parameters(), lr=lr, betas=(0.5, 0.9) + ) + return [opt_ae, opt_disc], [] + + def get_last_layer(self): + return self.decoder.conv_out.weight + + @torch.no_grad() + def log_images(self, batch, only_inputs=False, **kwargs): + log = dict() + x = self.get_input(batch, self.image_key) + x = x.to(self.device) + if not only_inputs: + xrec, posterior = self(x) + if x.shape[1] > 3: + # colorize with random projection + assert xrec.shape[1] > 3 + x = self.to_rgb(x) + xrec = self.to_rgb(xrec) + log['samples'] = self.decode(torch.randn_like(posterior.sample())) + log['reconstructions'] = xrec + log['inputs'] = x + return log + + def to_rgb(self, x): + assert self.image_key == 'segmentation' + if not hasattr(self, 'colorize'): + self.register_buffer( + 'colorize', torch.randn(3, x.shape[1], 1, 1).to(x) + ) + x = F.conv2d(x, weight=self.colorize) + x = 2.0 * (x - x.min()) / (x.max() - x.min()) - 1.0 + return x + + +class IdentityFirstStage(torch.nn.Module): + def __init__(self, *args, vq_interface=False, **kwargs): + self.vq_interface = vq_interface # TODO: Should be true by default but check to not break older stuff + super().__init__() + + def encode(self, x, *args, **kwargs): + return x + + def decode(self, x, *args, **kwargs): + return x + + def quantize(self, x, *args, **kwargs): + if self.vq_interface: + return x, None, [None, None, None] + return x + + def forward(self, x, *args, **kwargs): + return x diff --git a/invokeai/models/diffusion/__init__.py b/invokeai/models/diffusion/__init__.py new file mode 100644 index 0000000000..749f5c3f6e --- /dev/null +++ b/invokeai/models/diffusion/__init__.py @@ -0,0 +1,4 @@ +''' +Initialization file for invokeai.models.diffusion +''' +from .shared_invokeai_diffusion import InvokeAIDiffuserComponent diff --git a/invokeai/models/diffusion/__init__.py~ b/invokeai/models/diffusion/__init__.py~ new file mode 100644 index 0000000000..d7706c27eb --- /dev/null +++ b/invokeai/models/diffusion/__init__.py~ @@ -0,0 +1,4 @@ +''' +Initialization file for invokeai.models.diffusion +''' +from shared_invokeai_diffusion import InvokeAIDiffuserComponent diff --git a/invokeai/models/diffusion/__pycache__/__init__.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f528d77f34feb82cc4d7b0e6d2b5932ac68f438c GIT binary patch literal 304 zcmYk0u}Z{15QcY?h!Qxku(>X6nrvY!qIgXj5$vv6LUzK9Zg!5{J;C!Od<|c~he>PY zD+q#<969`9{$ZHmo1YeoT(Cahf7>_iU*qs!2@f}1;eySGwJ_2oR`$s{k;0^pqRjS} zdF?P_V{zX^Bv*83q3FmL=w9d<8mxLUU_)iF>vn-p@^8t!QVz>{I=QV^(+d0@^;>e_ z;(eJ8r&=3owH}sX(|9m?2I?cpj|o47KjAjn&q{5m2W3M9bn3y=6SOg?`H`9#>JUmD cQA#r(qt?~-bvV1?_0Gce7&K&TnK)1X0HCZ{qW}N^ literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/__pycache__/cross_attention_control.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/cross_attention_control.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7836a01912073b7143dba52e4b19f516d2d5b88c GIT binary patch literal 20594 zcmb_^du&`+df$D`lfxl7r1;SD>S>KFQIfmsI9l1Ysy$f4#z zoqH)+)1l*rO4pm#X}3+0wi}~V8w61LNSmVFK7s}fuuY0~fu`?Xv_&5%5FrIx)cvDb zudV)m-<>~=bqk)kd>H(s@bDbI!Oxq9 z;TfJ;HCw6NpI?2VtLlP;7$9}Rm+?44&ZstoAnFcoImd`_y_%& zslM*YaogH9mmkOTOt$6`6U-v6vaN(uq?bc2I>Pr{uUgH8f_3Oa} zuU4&Y2AIQz3a3;m$8#!GTFp?k>Wdq9RR%pDz&A)DNgIxtHl=LW+Q;zzIaYiMtBKeC z6$5DJtS1%>Kk1oQ2A`}eL}c4@uz3!UB#rIMcnLr2CH*lVRLV;Op>kfv%L1Xsy)i!z zWF=xd%M;zFyf=ZTNpI4h!n-N2fTsdcRZ1K|Z3jo7Uoz{>q0RvuiUofbwdT5qf{2}1 zp7#!;?GerxsBQaW3kJ}8%yV_Vn)Qx*C(wRD>YeP?I3+d4s%h`EcLr|{%G*i*kbd87 zivxvKJ?5Q7jl(#6L}L?2NYHors5eiB+0>uiFUp_;^Nf`@lml|(Nv^?ihH_LOsX6o#+l@iM9$KyOV3Bn z^{ZcfK1y7D?X@p`t&e~@S75cI?(rD%K^h6MNAPEXZ!_ii_(fSD>jsc+vl&)wyo*1~ z^Lq{7UjP&CSM&uiRGEWyOG*-idV2a?Dy|^+|j&b}ex6zh1lP*YCP@OcD*lwQ^IY zHQEtggzlYMz3!GPA!r7gsLf`vH@&6TLB~>+x?gT?ZWPBAiIw6RSJOVbR+~pX$JeeRtMS3A@=gvp1pJlTg@XGtk!%LM8$o2+@tEjpnTi+c8!g5 z5JY*dOqG;>Yts)v-0jHW4Ei$~Kvuz15Zqd`7M4n z;x<|e7)ab{H7kBG6&+gj!`+Y+z7q(kR})x{;xUer?4;?bNtU=sijF$T7YX4hk;5^g zq~5VnIJ?SUtpV57exv;udJ4EDGl2c1nK$EK=cD%782X>o$!^Lita@;-nCF*!^}!h= z*ePS(ylJUikCs4UyKeNJAh%sN9+*qT)Br^Kr)3U(s3S~{B6(m5Ow@QhRd9OA@I5@a2b&jK-N3zeH$rk!^R*QTbne2&p zve@>`r(Wc`%eA?avu`&b>lVwua{v8!gYZgP3@uz zqqASA1*N?y$}8yo0|F~+wvWft-nX{;ENbpQ)z1t}Rpl_{E@!%FUDr!U;LOgWJgKpX z<>WpOtfd#QH9S^&8%EH{@l&Poo79=rvRn&=G<7be1-JBjZDY%%uPdOd8qNVR=E`*u4AlJ$o*~Vm?lVmrZz0onYYXY-b1_-I0U@f#Zm-KO?JGR2 zKrq1Zs`(xWvhefO+HJonLct&Gb z%00pX{qeyA`XlulW&6?-U{J4B{K)odjVRfwRs%nTT+OmlP;2`Y@|F0*JP(Z=m_LL} zm(-eSrek*!9cLp8N!zNKp7o11iEsz18#$oJFL;7lgadc!3Q(kS;jG%BI2|CYuZ2rs6W(euH$Sqhta%;MOR>{0s+fa5l5 zL8Ba2)-Jn#eBe?1%u=ff8nFf;#g*-o&6OQ$7yDSXSL^=UXWFT_3-a^!^ffJr;*{L> zRS0dT;kFCVh9DSwbh!dphT)1`m1-p#br_*&toYL7BnIvj8uPpXjy|2W^e|bkWH>r-?n=> zr(>!q(YQs1CUwvPTyjLe)S`ok%|aTwV)J2iJF-ja1j>E^-++ij&CKEVQF}Jtg<+kq z0{+|k@UZtZvnzPeTXEljUN#H8%#`iG0pBsDYz$@gmeX-|urc>R2Rrr^pwLhVK*?9(7f^;TKSaD%e7 zrks#pOBm;U3?eHpKqHzotw&h-`FQ1@s)w!w{d)1(h%HBPN27!oHDOPLLf=x+M5!bv zwM<1nfbr4sVPlA#=aFjj0O1+gHCPFGU=$HtGusQyUE*73rPZpdrYd+IXZ5d2Ys zOAaH4LW6pq#b09bWqB8FLLyOLVF5Q?{Unn<)FekuN^pq`J>cM3BX8x+f>lVQr}No- zGM_Big;Y8PHU80jf$7n)6SLW5PUBz&O@=-^56|Hn{9`1;#xl=_`O$%iGT|p_rt~xG zS$~ZGRujy3E#(jky(CO~IWHC0Ts8f1)?QVl7P0JF8Xx-+*LKZ@BJ5Cm7wic(p+S z>(FNV$_$OrH}w2&aE)SZ-c$;eMo*!xCFrl`B6Yj#W!n`vepcO@IDEX3A@*H*Z&c5( zmN)C+`972fpf3>VfkGw?h)LV?m)u%a<}ELoTJ^40jX49H ze`t+T5xcM&t}SMJYkI951TKYNV&P^$OQ{dtSYR>$h!{FO(!ohx?&`z6&X%C`G;kCk zz@5a8y2WJpBu0mUYNL2qiV^v5pg}+)lpSwB{lgCD59=gKYKpnR8>~-rZu`eRY1;er5TyZQAz@4VPF#f?Mr@p>Y9Sv__cqH% z01Y?9Za~NTI6yIflWk?{*FI_LBjkfbR~Y0#ZxyPS@rTMQB@JOIe+e1N%=lUzbKRzj zDZio&;F-TBJX|K6k@kBDY>>5+^c*j7A1nwmY|2Z%m)J^mAd`3wQn*atv$oQBliA92 zGNP_t?qoVfCrwqfliW^hWt&zfy_H3sth|4zW9(!*S^fS#;uS(;XAI9al$5a!rHJ&B zR{coQ;ZRSA<2y=v{%xG3hmY`i_%RZ^a#sAX93{&2jkPj1LiGB#*ea*qz)zG~DSO^Z zqY)+i#zuG-7K^nq3_q-*?y^D#Y`Gb1v;rUR081Yho>nz%l;4t5Bs(10=m9n%pyMzzz(6r) z0H0u~2CgMLDViCe{lqyt&Y?BxdWrXv#46T4Y}&~IY!UDA155T+cKJcqHq_gm z2PusiiM9^`RlS2%^lL{6F+8e2j?xEdz2WAq$h;Yux3$`Ds>rNGHvLf}XMLs%z1Buo zm;$f3_@B^`kh;gsS!~7A*p~LiA9lp|>g50X9uUU+>_OM8BM_$kKur|JrylHw-^Y8P z{sh7JCzXyu4MR113qj@}5M8AL!>$1K>{r$B6gc(M#w ztY8UA!b)ZB*dCEnENF548RL*~--Hrwz_BSUI?jCu*5?*UPDV^oUn~{ zOFPMf2H}S{By%`YVJe;p-+6G@cj4P-e9-77!3qVma3IuuhK-44paEURvZCw?{0ds2 z6vzwiMK;)1Y`Yvrz-Sz=N(cjk$)(g-0keaho*CHiZ;vA807Z*i_>JG^CgqVl4th2N zyJX%x2r(gQYfIRlcuPYRyNe;$qtEcDcARUPEBh5$7N4`%NDw|#zUczpmh(%Lz z8eSb5h9}Le4b$-BpN>MXqanbPJ|yuKz^@@gyaZh!Xvg~?yicE_egohJ3*EepTIaA; zh^V`nP^VG9OF5-}56s=CF7A6VRgD}@2iUq1sIOw^hb|J}tq5ig{aNa-p+r-dzsMZr zS49A;`%Fdvc8Xh46CJc%8<4dKX1`j6X$U81hF~QA<`C{MZ%NpLIq9^Y`(Xh5G#wY` zN6eg2s#Nk?l~PIDE2&Z`n+ahaC0~(f7ZFC(n=CFfQB0~#HksUE(xa<#y446I1Vq2a zdVO$6uE4TYB)^1ja2Sb%QYIcxo|!mmr|tAY8rDkuOQtiqg4B#=5RrhmttO54Ixvhu zQV1Y{%}e%3N| z?5NOHn3|>D0R7Nn@v9)01n|29gCL~ zi{_Q#aMV*olQe2Fd~gP`v-$^kcmW}7I#zOM*#vj*5Eal;ACGm|^gRd^A!7!eFwsHO zm1jX;N}vQ171ESECr6%zIjpB4BOiMsv6bG+Y-P8`wsM^@uqFcxCJmNt;Wr6QD=QLn z3h#21iJ_Gx;{2^soy1NCdKKtnDcec)@?#xKZDCu+JLBuQtvp5_U(fGM+&7fp8Q+-< zr}QRu@`xHqY)ym(FX^T38J+yjG`$tuW@lpC+M1NQGo48<{hkh6K>d!nbD$$z973e% znA;#*UIs!`w*9xnM)6XU!NCJ&4T*t7Jb;2x7k1Nl9}#@mggJz!kPX;JV8g1xy2Bu1 zT1k9uKbB1!=!6XWvs86d!a01C61rK{Ow*_4rP(V~3oVwJxt*;u0wu6&5_E6dCZ=!ZI zeO(MyR|iACyPmknTD%yU8>1+kxw7gv;XqZ-zDi`}jI4KDqx}R9hEj0TkA?~?Ua7aB z;s(zy_F7_TWJOEGISEyarU^ls65^P2KvT1|H&9LeDJDD>2+9*p2@aK3bF~-WP{IBL z>JPKl43h(_O8s`=JO@#lMq+4Cf%;Xx&w=KYZH+|dv!eyQ#{+~b#;@X(rW3!8YMNLG+sRTs(gGEQdW1jc5$+7F9C2L? zfeoWn_gRs?MOOJDzLz3v1-~no)Zb^}bPEwuqpL2&l!nG9ZQ|Wi9hl#5`_6HfKZE|W z8OJVw`A>>Y2`P*o<9FyQCb3G257|jNAI+l%e3syGj;a0wCzcTFt$GsqAp;kQ7sW3@ zVCrNZ2vyx^z)cDB4Cr)dZ$lmkohGE$Zvvsi#Cj44J?q&`({sLQe#7{N30D%`Q7{JW zq%@h>vcoiq-WJS6kgP$P5SQ@9Fyp1)H4q<=dEZ8iKms`sHl(@t6LYvgf^S|*C*HS_ zpTu`+PB?``Iu*3Kk8t9Aj3Hp_Fp9zq1Vp=}f1VsVoLm?6#A$p3A_haW-<;BpSR7F6 zT?6a0-UXpV;GF0|RqI>UEqKP|)M2n}Er8u6EHi%9R=Y27vH@XYL)G|NulXEV4I5Q)n1$`snhXY$Dbo;-Fb?--vvntM+qFB4C#!*PAa*@i zL{vTe9iT60jZ70RglZZ7v~%tX7KCaj?8eMXAlhO8Qg*f|g8)A@s*0P!s|6c%#F$Cc zJpC1I_fAXQWMn@iUxddo5~bel66^Q9#bP!(fXg(CF*5WzR5UcH|H6c4IXV_gv!9py zHejy9XQdyd7L7A}+Us8W2_hTBFG>Jf%@9hDh*JNN-Ex3vt~X$ONl0AhL3CyxbcHz1 zntom1w21OCI_YP9CjKk?7isl0a?3msXOLkyOaTfeqt~GEfZ$n=5QOz{*0w&f9P0;G zBKe_{RBW@0n2n)Oh}o|o<4L4-)%u1}1zCe{J-20Pq6U31&LQp{mPiPykh(6vVSN)s z3*^8lCEHVo%E0{&3CwhM(p^#qTjaeo%YMSRHSZW<=I)nGm=8>&JsX0kc}WnIFN9-W zD$K#wX6nE#1bUi3jTT>n0KAoWJ8^4X|K2h0h)4{}ceJptzii0FK^2B1TS(e(BhxXr z67L!iv!TA-H7TwEc(!;JL&s2`~c=w7Ug3*7S@h7+o(y){#Fjs z>DWBw&P*o<6HK-0kXzjwR8CIDeEiGE$9Mc=3Ln6YFbo4Lok} zY>tBBH9>gC3YSAO=-aVZp;15wktJS$e=sNy3v?Fe$NzicG*?{|=E z5HM6AX6*SBIqW3;ISws8kMedFx~OI4K2E=5eeC?cV*Q~#N}zzW?;5_rA43A+20@v; z5!B5=ki&_CpaVMhBm^MnsO=oob6DDh!XXS@=Rnu;plivy&w;MNO9#4^0=5| zVG4z2xhGw7-ga)~LD%rxqicv@6VQGiQz~YKW{<+Z2YU4(DybhZ>4P?*LHiL)Wv7T^ z>SHE?T#^V3{~pTj6NU$onFrheKg4PAa>4-YK2)p+m5z)Y_X6Y19^xcKPjV4~ri%<= zC=j4G6EM{GEwmwnmqrkXBT^ymf2APpuvaZ5R?ry!$w&B6(EoN#nydX)PCAJUV+~<& zVA;^-l63W^vxEn_-*&9~Mu4l`u&3||#2v&IQ=_?19OlK1;GqBYnX%kwt1=FY~S_yhLAt2K@rw7&b;$dK^j*Y8gB{WMz_EOx3R)3Y_jqLtk zLD_c*Na8{k)KDy?P=sJFr5eQd+>D_Qfy$4F9dRU{NDBbpy)>>;iw_)EXPs{#!giTl zDy4oGEr#}o=t3==2QYAvgvK`Z#4{0?YW|Y>a}XBbd4d}~wtx{`IDeo^BG`n`r!9r6 zEYjf{QS#;;1`<8M)$x+nDPT|RBl{^-{Vgt+;}(GA?X#aC`SoI|mWIR$iV`Fq*YFLV zM$$FC=R=SW+UeGLanNM+ImCsYUkL0Tf&wrr0jz^NSB^mYgj-u}wje^Tc5OH3p;h#G zjPSQa|8Q%&>91kXh*(AseQAXX1taMhj=JWTy&!gI{0!=e`(2T~Mfu@=2fgp;3sFVt zXW3vFO>oP-*X*AXd}L>7Ylc(;IMcVL?;}1h_M69H#yO5M&V-Vg-pDC*9gTPQt>V|~ z@!rX7@Yfc)WS^MyT~33iKW*f!f+N_)*auuwt$R@hx=mSBN+_Qq>;GMj@{f@0!sZML zpTjrcd5p1{m*UqEPpU7(OU~lec*!Bwm~p1?)SJs`Kjmj|bspCUU``A2GehQ_ZoqLg4<&*(c0Ks6-oi~A++y}T;8&nNz#>JEf{s2Bn^>x# zD4;*lp;7*;FUbIelm4u9aJE|5{c02^;VyvERVpPhZa`Yk8^OF2+&;V)OynS=x-hS%EM&J~ zhf9c;q5c3K;>aaVwM%2vA9~#viHJWrd*i8|H*b^r_>Op4@Y4KeVy>C!EK> zKr&=RC)FhW3&d|DgRmf2n?P5{y>NR0Da`3GpF1PZ@V0hM<;ks-HkU)euwW{G3inj# z415ZEmFlF$RG!dAb09Zwwuk{8VGMJ~C%QKw)IY@NO2WxV7vND4_9n*9-{Py^X6YEh z@#{fX)l#uA;+qSukYEHnR|69o{H_GVef{cbx1G3R%lO4Dvhq zNZRZoYCMnp1m}>%L@i@xaL0*+$-`g#sDe2|!KY$AzMK$7O@5S8{~S%0A_o+|iNI!< zWBO>_lpA*-DMNSFTr?m)CBS)E)Jfb#CjH5 zvxzvl9Ln;N8%J&extx?uN{(ey$YnbDZoN!rqFXQ1nM98DGMy=@$I%d}wN(h=|LGJk zBaFK~anPvRn8_r*Q{V|UtR;nT+QZc}T_3XV%t07Tw|VR;ThpCsmdqGiGab~LgLoiM z2cX~2bPn*)ZOwM_z5HBfqL-iViW%PlJi2vZP&T&UV1^%QwlmjRfQWIh`!p|4hr;ol z+0H>;iCdq;+r#*s-#UW%9f3%Dv~##~1h$1kougt~IQI78x7lK$bF4EJkAXfjOtc2MR|yFQBOz` zrW5h)O-XO}DOQva=}DVM-r`)o#pIWnyv(uW!d6cjDg_dc zNPI7{*Let)tGGu}-z5f+>qb+{rt z6nClSy%nY4z>>_E1cY`?LWN%67%496_1`bkJ~1KsVsU}XC|()KRJY}HujN4D;hw2! zk0#or1hW@hfPv;JQMTV$iOkt2&-8K^$!F?V-Hem9VQqMn23amRIfu?9%S?Wpb>#9J zEKK~Be?rEQvmFrgn zmNomau!LPahUA_fiIs?X%%B=5JcWTmw)!6G4Jm(^gYXOC88kg~6(?&QL1VBPK$Dce z0Yoc-y+jqy@W1!HtLnSRK*AsukWRJcP)4cRU^2R6D0_!X!q+%xwvX)v!4@H?|GW%z z^AX@tW!xWRT-V#|p9*E#n6dPv&f}?QzEC``N^H){fjIkjH|QSbOmtvNjXBCn&{tx= zgN}ANN@tf3Yr=4pg>YZUlZxy;I?-!H2M7LI*dqP)r6&j6(b~I#D>M8nVnFe@;z+-W z+&e1JRf3~{zZtZsW5)w=UaK2;t7-8xbKDFq=rG1ZJf&I|s`8O&q77ER!yL^<0~RXU z6Xb8tV7elSYX^_k(SDaN$9k$8T;n2VF$78V^f!@r1R*u3<#hHFiczqd0w8eDJxeSL zo0y-k{B?I>?%&Mt6(iGqD>4iSq#S0O>wAs z(A`bh6ow}|$Z0P*C0PWZZt-85Tac5Fxh@bSUo|PpwmiTT`!)6Is;`R0`g)7u`RA+u z#(&vh?B6tb`|&XO46ptTh+x7=S%2~C^c_2Q`!0Un)XV&SEeraA!-OaNw4OEkjjY*k zI_z^MY9e^WMDWt>w}f-ddi6Kp$Bue#6)35bb6IU2=0>VYJO*_!Cc_GlgR4Y7%NM{L2U?P_bK$m3{y z9Gyv3HBE6U^Dq&*T8C{=Ux#`aK~3`QLd&lUOr~8oui(o z8rWA&kZL_HQerB?EE?;cSNU3|gR29tnOeuW?|;8PEHb&DY9o_;|0`9DWNh^QVSWbX z4i5JdJBX5fxK*ZlUnGOUM8nVf*ZdxhLmizZ`N`gRS~bEj$rBTXZAj4pNZfCUm>p3?)BF&ScJ`%RuL31&?yUwJDV3Pp4c@vJCgapC@L><%>icnk~*5 z%VQjAEycYg=dhmdCIc{-_V!jHAB-avW%iQ?eC5Oi3sWp4?eSl~K=2~wxD8|ehl9I( z_lqAt>D}c=-|*){32hE|0_9e^4rL@XkBdoe?(#3J)=zkZb7UA46IPY8Uo#xHum76 zVv-7oHSzE*KPzb_Du2L_id?R|Upb+oC_GY>6_po{Cq1VMggi@P*{fGI>u;OfRSlNK z!^*>*+XSgv=Mq2|rpc*PU3}j2mCcwfj1?tE)rNf`0wyG9zn@9lkq~dAD}w7Ie58p- zx4z@Gy_VN?K6cK3v5c3wKb02p-Ysx{#r(;A8m zP=YW3$bk_G&&b{PoZ(*=;IY?RFt4`>vWb>W?Q>WNt|15^|5#5JAK#!C;W%evoZ2qe7ZPS2%+|^dBtc8!3RxSaFWd=J ze=av>{$n?u&wlWSeJnZR+ne~YCYr6e;^=t-bEm6nvX^|MmNjN`OXCsb zA@bvyGK#v0%1d%n)dp!18FiBu>S(v9`=}cFc{COYK5Ck?aM6S*%F?R-%fm;%!&`ZP zJCNu_Rg2Ro8)H04EC1=#NTlfDmsRc1=3{f9iY!tpnpQR7^my9avbkBcFhkbCT3vN6 zo1dXfPKM^H8G4=w=p1BHok(G!N>OH2TLj9Xrb`b8Q4GMVHtiDKbhzZQakaAG)3Cdw zc*~Adlmjs~>^Lt}7NrOaQSH395Xu*0U?X;HwF|F~u69D}1Dt5vZU_uju#C zNBt0n+{3GX3eu`I93Rb5%kkZ9cgNoZ{M>OH-nP^7I!@bdyB(+NHQd0t0qQ#)x9gnm zUdP)HqvBC-gW9nip&y1-D_nNS7`Mal*(6GrJ2mT(MKJisR0`1dGqnj)1z+Q);Ac-8 zrEb&mCq#Zm)V~)n%bpPmXQs^Pi*}< zOQ^@(Gf+vR3}27=f2RllgUjP6A literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/__pycache__/ddim.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/ddim.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4435ff4af259c2dce78ec14aa1d50f0c6343dd71 GIT binary patch literal 3044 zcmZuzTW=f36`q;Bk|HIFq^_2H3DBf`k@DCF!%a}bfzyOFYM?;aB7^nLQd(-c%gioq zSy%!^Eg*Su(f$Le=u7@Xf5|))1^UqXt$8h)s^83#3axqSi(-&*cw=rfOZml>xRJe&d_68m$ zpAg{*|C|Uva$Z>jU$Cd7S36sM`tj#aK6~8%S9f-MBb2zhG?EUujs)4n%HDe!*8_I z%EO0`Kc5?<+$2t;K=woJu*IeAaPh3*$v?zUkXPh_mb7Fg9kJ57v?(d9Z^+0t_Q190 zzOY`iZwPk;!7g?`v@maRX<9GZ!IJ|nf~k5zqL;k1FoE`aOgZCBM2T7nCX?hPHy=Gq z(NpE*@zFR?c9^B&TdM3lN`~K(e}8m%oQkJ$xdwQ4|*W;R1=n*gz7*hi0k# zP#9?9EIkw=9v@6zD*t{GjGu|%K~r1CFto_o`%4Zw;5<&_g7bs>Nfri4{s2TZ1z>KP zh50a(F9Inz+gZ<2+1)Jm*|$`=$1*>|&_l$@S`8>asd*T%C2_^-rPx z;l-u#(LrE9{|yKY`Us#8X-rD?XZm|e@xEZl6mrRCmZ1?4pzYEUWJ=`+r8TlJA}~fy zX_dAP3gi|-Li2RER$mAZwH2KfGT<81xBw_|dIVn&Gu43`e{dBPrY4E%+H>k}DG0vYilL z{NIl}`UmuJ{p+W5f{-`=MAXRJV1|0d#dQya#`@=Y^1q@S9U-ko!j^+6`-*#vQ1{)IgDD`MCLcjF5q-$ zow9Dg{#4H%xxzkS`TbAH?`T;cd74*tsuhP00%t`B4(7P>iP85{hIehY0nQhOQ#YKI z*(Ny845wi@&Dj<>Cx)|XIBT=*qBUACTY~9(XqOwMy+g{ja9&%pog1>vCE4x`+4hp` z)(zR#l57uuvR8Jx@JG3G(Y_?N$ZWs#XHCtXtd;NB@~!aK3$-?01UXlt}xYL5S!L9%P+?nCqT7Afy+k=fny zZn-bWP?ySDU;YSCOP#!^K~-ABsfbRMh0@ZaavPF2S}fnMbm*kFgOYR{peAXdsXS6w zi7FS?MK-TKI!$z*sTT4o7)M1U6Z_r^wW|W~p5>;hj3_y-KFrLaysC&t!ZQr$NGX zoi#OfZjxyoIg&M#;jyxZ(9r=x zt*IrZekuA)QEp-bLBtFBWBq6D9L|+FFb2krcLU8i47d9q-)(_z zQ2PpBSIn(n*^GV9ym!C%+&#Jn{Wf&Fj!nCCn{^qx@~t-LI(@r$F3jjL>A{G8um0%K zqbebFvh}Ov?8^`E(W~nNK#DZtT-CXbF*I_}4bBk-NhL8ITk1X`e}X}IX~vIaAe0rO zUHw|Kn)5~x+yMv>7(jOh z5;-0;iYZ4_6WLDE#%XHh1G??&Q`6?Xrmsnxv@eOfq)yxPW$9~syE#r9N9mHJN}O1h z#D0JOb7un}*?IkZ9`M1zx#ymH_VfS$&wo3sgM)E+VO(DabSGt0v(Bhw=m&n}Oyj82bQJhwcyGCn}(^eIse3!PxRmD>qNyOnf}p6-)%Xzm%H3B{&-F3HCmpoW3>K7wmsNF@4*E ziQtOh!1IaVKsC2?`-Rl>9i$u#%A}O5ckp~C&sPSAcs>;8xQmp-!4Xo9RPQ9myLi4T zxSHpy?fLG&dn_?^Y-WwgDLlKl9F$iU&o$P=YI$|OaHi2%tKW6v#KL0Z%=+x{xz&{u z%jO&Rb7FRRb@s&Vx8HEvY%o7} z+x)FJ-9CTo9l^~v&);#&?CrC+-gHy-wi~OrRd2rWwp(Uz?v_;PlvHV#)Tw3ryW?x; zcWUD5TD3N_cw%v7EnI!38gyIdT2;OCmRs()^QJp)o}Inp_Sp+vL<#GRi$XvoagZJpAaC7v;n1Oi1>HYLp49Yt@+s8A@}jwMO+^ zV`Zi`vrr95$*#?Wjm4Q|qJHD)V1@trm6@~E%0jpp=nINCTWMCq)yisZ`8+9whZpOO z`&Mi7iwivMK234!Gs_Rvo}o=l-6L_)xu~?fx=?AHYgA_HmBp2XC{sPhm-Q$cROe^b zmm5)fvA#017G;_I6`DiERfc4Kd8QE+=T_HijS7vMS*b_6StgB{8trUUEA!#%N`-#7 z`ZEtNHXc}+QHN`c%gl1+>~ggK{>7Pv)!NMRgYh`uPsQQl?0SO+zUnbrQ>ke9V;^~B zHK@|^Mm3zDnX5*l53xuZi;eRq7sGnvv4+|-_}=x#YPB{;&hUc|^Vz74&TO?YQ>o9L zsRrxI)o6?{2xsORmBm_PwbH28>Z@T?(5kLhmKV=fUu9Dy7_5Q&?|0cKDAsYs?W?U+h>I`Kf!9X zUaqY+%5!J5lFBo+^DC=i_4q3Z2HdY#m**p2b%a+_y#=q|d*aOMO7+BYy-{7Po%nFL zx>TKO)K9QJR?k-7`_PF+i?FY(ZS4N`o<9);Yb$Jns1z*LXJ(hHLB*PMkGiM6`l;|5 z{>`bY?H|R{{Mi`nC(=r^l9P!>Vkz13f@CYTnEYIFoV48<8}C$<398R5&Q)oC+8|8i zv%rHXKk~vG`MW7f*U!wXRl{p3tJ|w^ipOYVy|$>$$4;oYVys%VZGSSnk$-di_5JLN zpX#4trm6{$RbrZU1zwN}{Dnl24l+SD$OZYJ5DXA227^H<@B#Vy9VRxyUz*>c`F$@W z=92>Q&*!I8c|Y&vllfwPFkcGp7q|z&`_**tfWZFq$qUKpOf`Ff8s@!VGT6i0T(CFT z2l&qi`-3a^UkDBa2l+n$&@Ylp0N0eq#5#3}C&js;Uz z*C_Q~6I@FzW5Gkgb;0$-#z{LKoFKLH}ijYa7%D2|0jdng4_AOCob=f z;Lhh0l)pE)D|i>_`+~cJd-%WKz0do>2ZDQp_galt1n&#(5t&w2h zr3|U`xpJ_zkFeaha_LZOKlu)`x{EKVH%E9XhdEx%DsvbT6MT3rPXb*49=*Yqf>)A?E53a4RSuI(NgNW*VT@DmMr3DnGhj zt0@DJu{nJGT38Jh9Z(cpIt{SS0!3$6>(%Bi`{sky>YDOD@W^}X&@|;Jc^n`zu($x) zsaDQb&)1`ZDz6w^o61B33yaTGYZYONdNkONF!GJXl`0T&tsWKKLal({!bVhxpQ^P0 z1hEQLRISvhr!c>`yj)=wuUCP)i_Izx91N>ZuP=txI=Ff!m^upRS_g7Pxe9n_u~Df+ z!-hYr;J3x*j39oLUJJodQNB`{Tb`-c$wThi%t}?cK{|oju~2oLhz94D7uNuFhRK3x zKnT6cu$-?nqM>;q$v8XRj|#T2Dry`}2Vc~x%k^k-9UQnuM^ru7*-$CtSdWJ9Z$tja z!o+`FW_0Xi%Bk*f@-aJJFa< zhz6cf8#E3e=JC~VabeLKvpl<6dA7Q^aEAUY({y3xcEWgdWwsgww6zje7uu=Wl~u-; zQc5<)ju<(+)`&*PM5C%&oZKfk`b91YgiS1R4r zka0O8uU3ol9~Ska*_*l|JgQFy)@#*9+^6tCQo@r8&gxx$j#fZ0HO_?h=^5&4CNP?< zqOZa;%K4;%#}#~B!6y`aOueB3kn1$!+8R?P;5dqILv+cVZEDE zFs)!&xwKm7tgXdYlHsFzVPt}EY4{-mRaIB9g@c8pm-64t_+D}%>0i#IGRf|5FbCjksn}b3Q$J={jE=#@bpAr!rJ~eRl z&w$SxPz)E6VYZdr$ky@=XbB*gFSr&YUrKLe1Fw~xOl;&D-ckxU2rhrg+sHS(O`kXU z09gK6Z%^Wqw^3+im(oupH&TsE3m6M*%Np#O zm&SwLV~Jolv0c!L1FhTzm8%|1sNMS#8-q1rsIg$uzBv@^QOS*g&D|PZD}QhMNj)UL z-*koXKVA{g%Asj1Y~ z9r>M^J`bEmvpeApB#5 z1Kvmn29nYaGnL%OmAQ4DbAxEC|G5HYd?p&JKOJ(8vuZjyudu{bFcfFg_4?M=`-Lsu z_U#_~IIPZrV78RIuls7{#w{vlehDUNCYqQn%T=jhuW^^7oJEeLPWn-nNItSMs`B4_kycq0>^ zFSGMN_{PmQMME9Tf7lxa3xEkO z`{|Oxl$RvsE#ZW%|Hnx$W+?e`#`E6F@xAZcm*mwuheyVFPduaAg=1W9r4{o$QCs|d z%6l_&xkynZQi_!5U(RKUv}bR!Ibj&5GdpdnWZNOWKxc(K>pxBKy#f-6@E~Dw6JmHN zwdr31=nCkQ0@_$g*%M%5JsBPZv516EzRd9=nP^@aBtFKm<8fEYJ@!quuN3#)d*bgr z_g$N?)_-zC8oce7)BRsQo6zr6>Qr;jUFFAMGKeW6Cy9 zl8#?a zz3{t4jNf34Xt1>(q#9n!-%ROz^-LlhYmu@ivEg@9H`1;2{fQ?hHZm=b|05gOR%X+0 zWiKVe8ye}r-^{c!!g)-(-^$#Rc>3Ve`&y~`yOXyIV#WKrPY0C!HOsAl>8>T=bNUl51oPj8e(CYQSzjteek#F8U(sk1UH zS2N={Rn0*-$S_G+C_240hs%($=hk2*v4(WqGDO%cJrZa7AW3(Xng_zzJ zK^#8CY(>7lw-$$uy7*#lS!KlBfgd+yq;t2qwj9F>Us9GWIKM+@hxe;MuO5VdRgvV` z$Xh%M#C5HT3f)F;S)Ab_`S)vC9w$;_+2xW&{&LBJ(YxMrEW^z3<&u|q(@P%!sufcu za2JA#gjUL`=`RQZlDh*E9=PEZ$9q(kl%XTt}jA_`o z%Rc+$sq(q+tkB#bf#GP*uh~Ia)6px3F7aC=zWd9f}nhevblSbcY1LSFbWL#EMIZk_|Yh)EU8|iYb|5 zZ=G9KO*Hg$#i*8cZe%@)`UEAHa^R}V{QAOxUaCM?NHxc8rEK}A51470bXceYk=vJw zrHF;&7EfwCF$tb{C3)%<-}Uq}eCB$3f!LJqI+l(t#j+Ly_g>*AJ$W9(bT zlj@WX{w1M*kUYmjy2OTf%e&mX!n*W!-}dz_OBtQM=>!`0jlQ(ELEN`?if6}?_BM!9 zH`10%o58*VY};W_+amlKie*}D#=noa8;2JZGb-z|dY`os3IC>2|E>ZXmQp_{+ReXp zqz0bu@-IHD!8t-1ZgAouVqmWsIj|59ynt5=ZGOK zSw^N(3I9G1Q+c=clIL8hVr;b8N2}}bjLyN$fn10Rj-{l{WaA!9bcRT5n$7`Y>r@P_ z^>4sAO~+npDFV1h`SbaOWs&jTaHMO&i#Orc4>jiPpCXaY9B4N!Fat}8FDD6;KbU-! zv*D+b8)=PUXC6}UA~dc!Qu7U9`Vyz@mVY6&QQ-88gyAgwh)ZnMC8&*sWZ;W%TY@GD zJe_L4OsStsz*~T?5TslA@x*5K66vjiwgB)xqZ8;?XiIk_?f4a+KwbPI@1fuf%9+gM zm-K-c;!)ZTC`PQANJR(~-ubz;siNBr{bff*C#I|z2#AuiQL^I1F#kv&j=K5kn;I>d zs90OAnS63(rhazoY~|J}b(6Q%HBH*1d*KRYlEurV)UW{;@GdZO)L>`KBWi!shfziE zX{Q)Q{W1~efxWT8xCco`-N6e&;>7OoMBgnWH!=;sk#4{|ZAraC}BQ>#9oB&(HBf8I?kQq%T*m>`VZX9f-bs%{9 z%9a;?NIa@}MDeXOGnhZso-0kw>je`%ilS9MgM4iiZ z_M}bbr}=Cu>9~VBd2{}2@z1@q(`Z`avsx%X8|7dXaMtAztW@O7c82{`eXyeU{_<*V z!4+mq$Ip?-+BvEWcJLi{=vZRZDl+as^4_|u?4PTEl*Gt`eQ%@W;?_AIY|kJ2cy;Z9 zCjJ_}&UvEwIrtgn&jBu3ctZy~4t>eYoBPw?nCI*enQV?a3==OME9TWYu==Kq>4?-o26mC7Ym6jWRJLRa zwq#5&Zry=7YZ%(-{8`%J< z;b)RSYh;}_hPP5@L)8my!@RfcWr%pjb-=gBy9ka!!Zmp!BlW%57#YB^F^5bDXz0c|o@b2NFw# z7J`CS4&+N7ka7FFA-)@kzfnGz`9_&SK*c2wtK@d64(Lyfy#0<0{}ZES#A^7T73(hk z&b|wOhZn!VScb0>i8q{%`1N|VzJ0^BDYNc|l0Y`RSKkhs>`DSRCwMG3M~zQeJrDUJ@1}iwZ5LVC^UubesN=Z z=tC{i@DCN(qRh3KPf=!lEdaZiK+E`>Ch2NZCI*i>{SwhAqU3z|4=5=X>xt13{tuP> zni2~lOxOkUi{~WyLhk3JiBZlTXlT!fJ0RB{*`L>5I7T6(tbcZcu{Xuy^gNy}@ApgI zB<#geZyazm?&Z7=nc`-qkDsVLk>OABwffxzVoezV0x)5{L#jY#aKwj9@i)^3fD(We zaMc1ZUqs;|7~wCYHUqv&IRYa5H45SkW8gb0@ZI513vW9N{cjaRw%-N{Mmi#<2$A(> z&OWREj*#2}6S8n!cpLl&4TH9|p8_Rc_Ti=OO*M-K_2Vpk_*a-ryZ=7^>NTXH;IDa5#qQ;jj{{fm-4ra>uO<`)C0E2cVx5e0A-=tFc;pVpHXk@zJv< zL*Mi%qiAYt=rR#^>b%r$91Z5A$t2pZ_fpyz;GAmc3d`7jwTM}9TF&7ue9v-T$^_{> zNKu(`vC2KTq3JETHQBjSr(f}-91^dE$}DO^lDk8&Mp;z5)R!nnq`I~`cLqHUbWqV; zQ1VHmr$My(BCLQbSp{SCTiblWn4jUFqc7nejnynMltR{!y`dC=o8-iJTUek~try6)Z0vjd;7isH$B?Z@tX+P_K=5;?G z6S#dv^-YeTh$ctNy8ao0t&_8vyp;GnI?ui7FDANlVDXJT4XoUQqv{TK6}NByyG`SDOkYT(gvLc%R3nyF<7`;iD)XXy?Yl`*}Bd! z%$dv0V+NAINsC|*Sl>1iJ?OFX;mOk>>f)mHwM4>8)Eea9CDjM##rMB8_ z&Q;1_7-MKlox^IFjy5IczGS3-?(+V({B(D|GvD^J-KWC0V^+kLg8BwlbFM$bAYyDV zONme_2zD+I71I_5INn#PE59+!OMmT1ms7*@3bw$vQO3;H9$Qa`9R#08J7X*w@N#pv zjq&z;eXve&t^?}?c!B)D{SA-v3IHNL%K-l^2K*OOgu}Um_x`EZCu(iz?f`wwf%?}c zXUlO^>Bo4Q>L3BX7Op-^7Y+7noI-7sHfYbf(@8xl)ui)WuOR%b@Ih3JwWv(5dh39U zRRqk*2D*@Tv$UJ8)rja6C9voUCBCRl z%K&NfffBNCj=sZ1D5B>%W#}?%LPz9LXeN5nYk_N#Ws7=uCmPucXu&`bgL3Un zz9B*wX%pOyTOn|b#Em%@C+#U8Aa8)kKHC^*6yaSI79fn#flR>@fpgU=v}@ZeL7WuG zV;U^dfms?xuMsU5@e|P;$!(6bipKf4jUJ7*(DR^GS+p$t@W)9R3rP7qSdA~onR;|= zU-CeGgBzu7VCi#xU}=zY2BfV)sU<75$WuD}*~YHTi3WY&+%4H2d>7A+oRJ!T^CDa` z1A*x9J`Yz%An-9D5fEs|5mHgc2{nw~eLZudEjy@SaW}$0B7hIm){3)bll$=UYD1E= z(^Fpfb>$NGAUwfbMg#37#lq>^!K|DPXnalQSObk&fyQ32*s}@`Q69}O)McAh@FqM; z>L)17PT&)X=3!fYLUg9;*Lx)Bz`UtR_(~#;JJdv!LLTF&q!I->V(}@JA1SCQFm{l( zvO9HtK{2Bze_Ju5=|88~O)B7$V!x%}2MAEX6t}33VDLsKgqs?2@JHlJlstv-K|^U@ zL<2F0NdiCzo4%rgZdbYeM1KgJa|0dTshADRyA(56Z1lbxhMO27BegS#UWWNaMIo6< z!2|a8qs}(W?$w|79106w(KG(=<^7j)QdRH}>_OQ>)D3MJ+`bv@gLLBBu%r_kNyn*i zf@aZnkU?TIa7tZ3;hn9VS;{Wu#6}SRDTaIADP~%v#Z(^jC$&VDS0&V8=qZiG6AE5d zAgU}}RUqIRrHzt_((essRvvG&R8&xuF}kK6rO454r`V??4tMXIgXY@qL2tW=)Ha>Z%CnnI%KpM;_++#-8Qu|)+xt>EVrNXQWRv#YDi z;VNUZcPk_kD_efFK%- zb#&*dX#Q4uhV@w$ZyWR`Vh{3He~uuR@Olt`7wLbeSn0RC>299Fx4dPhqmA1oDksVW2RQJDAf?I&(!KnVZP@nJnkj%fnNlS~JIA-%lGI)z2~IT>}UV z7`(iU7}EUr@)=b7pD@q~Zw2!PU%>kL#B{of5yNHMYtdB9934#jyi_ErY}<6O$A-(VgCkOwXI;ZBKOJX`H=ly4z; zfAEmyI}+#nfR))(c5o*6VDPZzxyt1U9tlpFp~uzn^OM1cf)87cqj4!8$v;YMkI^pG zsxjv`SPV`FAGLhPf{z7{GrLo92_FwWVI^Ku%>>iICoO)hnTA{k`dx~K9tObG8XcYU zW?@A}QKkidD!h#~3$-pT!koZT`)7oRRPLSaHGnaM>GMNv#SfVN2#Iw6bB zChMb8-;@III+T7Z8}1EK@B@ALfnFfa;-Mfxd=MdNCXjk?$>P}s#fKXH1@RY)DEj6X zDN{Tch!=!En1jHHM0Hq7!?|V=HJ#*AVF?vsaXC=mNK^(NPt>yRWQAfd&!0e%wf=;e zj+APoYz_s5#xNukO0$=o3h^lHWz_(w1XolsnR1K=>entaLz%Gn?pDck%`TF262p`s zeUI7@#vu(;U&R;%CDzUTmpJu3ncf)YtKrHOPyZmP2R0AN494;uuzUx$<@+m?TZTH9 zA}r%sxw3Ic@ukBS9+6ShOT4XIrRPnVgj{8(5cTsYPgiap>(pGdnt#PkjHOeKuP6{1 z9#(LKAR0#pypB>lh$d*8B~-$?-ruKST>*2AXt9on0gFRS`&Pq~l9}ZjSh3Y~nvaUo zICJ(4Je8!)F9zTo){LvQsVQSeMx%Y|{ZidWL)4Wzd!B=15T({u*TPe(QU@+<0nH}R z3@{?$&bCE{9hhVmXGWauROlFY5_!K+MlJ~W0&I)E+?mRXY%KF?Kmg;v9)6T8;SZ@M z;jBBQ5ayT+E%jB9?cS19)6l+ zlRB=q3%R1sq3i8m4^v0Qj4I3)l{xOQ2r+_3rWy6A5hu7C?X`!AXF+_x_Dsb|6Ve8j zB}~j1Z(DPGLX#sS!SG21Pbdf!*lM(?o7Q9ZP~BOu81=|dBs99f$W$W^w4Ba}&I*VW zM>6mZb1;-;FHz)o0tT~wJ#W~PjfB!sgoKj-A1Su7kVYEXuP~oEgqrx2TzZF=iG7>Q zy;;g+yi!^kw9UyaFt%%KLCZToG%1ngPkXKYD1j+Yau_wfv?q8*ip1#J^JwC1qJ6WO zF~*<0WyfbPp#v+rS7#cYeL<_G>}S2ZjW;pnyPeN5{b4EyKd9i-3e*rpUA6&MlwijX z9W`Wb#bhl8b)?rAA7g)oEmnOE!{AVp18I3w*;3@_A;yfxbvD6nvSXXzpVi3g?Bz33 zHnwm2Y11l(uRWe>9@;uqW>(x+#GI)9_K!wPm?*_x{VN3Y7j7*f7y!&7eO*R}*e4}z z=KqtVr0Nwy*rX^9T!C?kNwK70YHFYvj1*B8B}WeYgCKGfh`I!RFj?6^%O?iL*(^%I zU~`bs1`9nUIa&!7R1ZlMhN7FpmoRt`Mxb2D%lx*=XuF;^w`lACDHMP-ABJ|D_i_R` zgF>IIjWOpc&#j~4yHW;<0E63Fpa9J4yD3Om0C6+5Dtz|k^UZyS4<9an^s$FdJy?G1 z^!?XA`oLrNz4u`~k#4}RsU2@>sj^%?-V5BNu*^=@7Q^bzSra<`uVdxT^TGkpkEq24 zuRAB)uyd}BMxCj6r3V;9lVa}lBy?=xWQ^RvIKzN)gg!q8;25{1=w(}B%vkVZ5n>fR zgPsN|tT2?sCe|7N4m!*9u69_IY;FMr*t_+!+YBuKlGdk$H*gCt=U9_DX|ns?8$5qI zmvfr+IbQ-P@fQ8$Bxkf@a{RwFk8EAJI~D)7r?o!2D=q2{n_0paXE)1|6tm7$>@GR} zzLqo#uOcEL6=MA=&899R62%Hb^>9{gYL+O=c5Sq)eg;cW>@fSbR`@JK+8*Nvl=-BB z2Nk?u!9xl@pkNDBRmoq|*i90FIEAC${`FrkVzD5RUGvJVWAx5O0NUu<*ai}H&TTU? zTb>y0^jlmxHh9GpOC;woLJK)aD^0Sof7! zC?%HzP!8_+uYI`%$c3|{eN#Obfh?w^lS$Frqg#91i@bXpv5k4Fxwk(5VL{lFPOmlZ>LKr>ZFq!t8V6@Rop!d%My^^eX}O6B%-CgZ>{7;)Okr)u+Sx30 z#tw_T%D~gN#U&@trt0?_`L0@0^$#fC%(P2`ydLxujpEYaQt3P>TEC5<0Dbz;B%g&Z z+HNC~rD)U$%$|-{D$T1tYO3s(C{1`dQ&_IAmNDDX%5+J`!_y3MGk>~mBGKHZ3^O%! z-<2z-R%5pgC!6!vlaM{e>^6_Rhw%o#K(EUMzj^gLtxJt31UM7XC;8G=)SoR&kqO$0!l=eSOUfn(=MCUH z6k=1GQo2kofhRdTNvxPMFD#BYg=6EMR22RI-EFhgKC9vQ zeg(#kyISupFw*V}FZQJ3tq6(c!Ba$`(f69yzf+gDwI=F!rlIX9kZgTFLu5flE1muW za)zM?d?yELrRhDJ@KUVus4;jt*Wf*yk8J~G!8^XsNy3L9AxjL^M~me-)gq{6Oc5n8 zZW-|;e}ftk$>Xn5&|2(GYtMIXZ`}Yc&$rDY%{hD-1O={2W0v8lcV_a7gPOUb^_$ zosBe&vMidJI~y`;vBV};JQOS4s*bK`SB%BG`Fm;CZ&E(v0!LmG8$Eq-qk{fviT0Xg zriUxvmF67QZ0)CRKP6VD)6?_c%3v%ef;*P)T(fod-oC&-SgKBS7VZ5!Xwn+6DX?y3 z!9uuswJ}RWvy9X+?3%q=0Rk=Q;2VlcUPr|#+d>K67ISmoHu^%=4^Dfmx$9_0rN<2C zo#EhA4o?8UI))pw%upn-GaF`F> zVCmfApg{B3)^lqtvzq_+3%oPuA>8N}oQvR^aP zM>U(pjkXWNlW+{8rec3W0Aw~`k>Nls&sqL6i`8dCZ7q!6@Sqi2?mwRBf!1=e>_k1=&dKepeIubyIwSEM59+DNBbjR{T!3dspGi@l zrFkrTB>l!#3KMkK^R{LM97n0dXfzm2f6{4KAab?zjMDjIY+4x`DclY)UJCV{gVMR? z-`{G|b>^<}8S|g9^~JMyVa69U&UDk;rdG~P7f?Mg(=kB_oxV(#R-RoAIpAE^`KtUZ z{S~-4=9$$^fU^iW`!TYZZ;t+HzI`#z}UVECFTz z?5CRhx5B{QPy18?w5qhnY=3PMVbvIhhXasl$l_RMcn%h*1L!||ZPph2XIxv@>cTNX ziXDjC3ZB_lz84f{PqjnGHV=I-nl$y0tWx{}nKQ5IuMX9VRmAb7QXal*2P_69L z?5SwT{Lrun+Fn@Vgqd@3g3WAHXe?GMvmqYM!f34Xi4tQUFL5HDbrU%<{e&jb_PwvK zqS5-=GX8T+R3E#bb8A3=z=(_O)9*ceKJp$uXVUE`@9G0+MgtGWmGiI_L<|a3b+Vq( z3?HEeDMZUO8F>2Uk&GzW-jvCVxJk_`w$6B8SvDSh^FB(!ntPk~ajKFj&=bBh%0ecA zlvr~PY_YlD3qgPcxq!k25H(Kcf1D0)$kf%JLAg$9>(7AI$fx5d8aDn$QI1i7G&yiz z(QxFTrKHiE27`iW;T1_?1RUz#NgoY_lh^qrGSxY^hklI-C^auf3= zAU#r&Oa8`iYbbu+^6hwHUTbpb3&~o^TI+}3j$8Zf@1nIv^V15}cibvsNZDK;wLA32 z_J?A;QNKsF_A<_=9%o&A-lNYWHYUA&cJ-*=ihbw4D;Fk1?7r<|u&u22Skzv$5%6&| z@)RXXegJhyOv_#c{G$Oue>{q#6=ML1eEyTcHeAzpgxYUHQy) zGUK%oJLJwM7!=1}0h#cMW!62Pyy3~~uw(B&-}q#=ye@5mXTpJ=T!?rUmz}{qo;%Gu zl{2Ids?h-UjaW=O+aX~&r_gLr2gPsJu*70aE7j_V0__vhj|5Ba-)zS9lnOTpk zRATtI^?tBT61a{Pd$QlO6=n_@b+7VVIo~6wDf+#XolL=_#9jyGo6DoAvge~P-{x%s zGw}*CiLKu-ma#5}=S>*507=Ljh0NZVj`jWhov);d=%>n*H~vmv;W)xi@AQ?QE_%&_ zJ3(seqU*t&z;gRXU|Q&zUM1>(mci+AL_wvdX@}E~5&7P0A?|=Q%b^FS;|Kx~8KK}s z0MRvds}&e95COyeKYsppI1d_sBCQk@&_)Yj|@Uq%tD18g@=5{0F4qNy?sv z0S?&1dI(l$(C5Z}7;kyds<@HEM6n{-C!$+(smC#_x{OS2wO%>r{+)-={Abz?|D0fI zqDOkr5gdf)^tmunlL;$wlmCeBOyiCaVP4gU|UX830RNT7;An zZpdo8xo4-1YD8e4B(6033nb-RuLyp3}#KqJ&PmO_#m@Zt9gR&Co=!8I3f74=l zmcDPX0FxVzp-!Ae2bIOWP9$q8 zi2RX}NNtX&t*QejYpylcLQBP2nkYBMV3e}bjWIyTKYStaLh=Rg1#D??h&%dxt}))u zD^+7SjzhH}d+!BfJHKaEcD3YEcO0(cFc`nY34{Ltm*GxZ6jS{Iro+&m!<0PI8fgu3 zp5{N?n@*dUnVj0UG;uL?G5Lk$S{kL{U9$33>L#f&eC^9$?#qA0PWh)MPQ7ZU3^Rxm zO9IBIVeSEK6)7l%KSd0GWu`jcalQhx=9#A>?`++n<^%GQxs9QV!-OWQ2o#jAczHRF>Y@#MaZB;7#lYzsN@S(xF$mJlWu802Dd zZ7*GfasGp5h4Bh9x-U@vD_$rWf`p&qvhsLX)&EB-Ko<3bdR3G;scbUlBy2y*n;yQP zN!$x}OGnJC3>_ z`($!IZzes3!ULttIsCh^L@pQ8h_f@f*NcUsk#FFH%Q++{GCj=6AspJ<@)ZC!d8?Q% zp~NBcs-nMLabx?UdRuf6{u>(377rtJrL)Duf2+s-4IchGrFF;gKj@J^Hi}|jQ}CM# z-f`Q8UssmjQqWm@p^c;Qnrtw&=)S;UhQCXs`M|b4gFOOXlvpuhFUO*hUTMppY72$` zMQ!;R^`qOM|Eje92K|^C^xu@;S^eRE*W+uQyq)C`T`8N1cWmQgXR#nB{Qq&W-1w&+ z%|0Q8pqfB|od&e%8(5Q=VB#MDeqj4T6#RXQ$-kW3Gawl!hSLUOqTYA z7E2b=K+EP$hBJdF;-Zy>P`Hga3{bZxITREg}}6&ynE22yorZTvYpMAHL{ZYpflX zIv5iQNf8GyBTB&#gh96bE+z~H8##Qd4B}UR7{p)VxL*YE^B;CosZ+LMDo=>N1SOr4*T`hxA4vyjde(2V@kW@Idd5UfX;TFN`}9n z#l7Q(Zlj>?{^3j^H0tC%!YSbn+vxvoZ=>zF(2113(?+)K2lL>&KoN!*Ovq>0q0>0D z2MpI4`QHU-*4OV6F@%4p{{4}HZz?cYVcj(drnA%E(++If6V1~LI41HLjb-2 zziBR-y}1MW=n;6^LHzcy>f5Fns-^mb$r8&%I8=(K&UU*bgER9zp=FNj+oVqp0FKh^ zUL)Whdn7v-y+%n}*dJy{`d`&m)#C2nJK&2i%26BrO$H}neh17h4&6LkT3JoMu)ZTzj9F#GPxrw5 zq_Wt#_%n*t71%lZclFtCEBLB{3@_R$V74}G&~wV3SMXD0!lZw0>l74t`3IT;;cuTq zrt?)G4HP31%tn70Vqw4E+`V;bVwLGWL7$*|`_BqBvl5{D^SKw3KsE*}@UbVfgARj} z?l**%j8Lmj6lz`AvcdWjA6=+IoO}KQ{N*4UNwwaAEJ!1?cOVRV=SecHobCLQ_e>&_ zb{>I*Sa@T(d4mfv-hf^l05|SwTc#@BqoZ!_AO_PU4QB-J(jqLU(il&{EeY@CRpgz& z!K7zK7hCJt?T?9C} z65FQ7X0iQJt=UHMcLr1AgmuFSJyV;oI3~n(I2H@Z7rl!GlL3h*G7)&#!N`>v{E9{1HPROg(gora z{c{aCAK0;AA5*`^V2d9f*MGjxO8Wd_)?&y{UX(;AhYZQc86Yd}@GcgBLCK5RivyQZ zkPycEfB{GvB*fsw!Ogt1u>x2^-h0_+8wtRP7h7WkcqYJ`9F6N?bMbE9iNTNG*|zU$U;p?DGNDwGX-5 zZnYq%yyA2`;k}rCF~ig7(im>=(_Cs$3~;(XmUuD$Li&Zw3%M8aFAU75;i~K~HXDPM zC;LJkzRb9Cv<8hEQ+lDmc4lH<=b#C@R+OE@J)buwNzUr5%kro9ttFB~sml8wo3 z&d|Bf!jl4*OwhXB@Pquuu8V&B_2lNB7jTz{WuNLCQVrtRH1@WJgFV2dVQL$8d08L( zTEq6XlV0V|BG4=jmf1wq!xmMkfftrkArU~{ogFKM%$~xVp<6zNilA3jbd;P ze{f^0R$xSc%@?_}X_0X>S)$at#^O4Dt%ZcI{%fr<)r%g|7)Yah%2_~_*}>tJ3Vuqk z!_J^xqU=wU#^2jeq1yU}W{wjvs5mC5_|7blUIvK4MG>n2N@E%H1IiO^KzTgTJh2r} z;vv6%yUtEWPBbUQp5YF>N1b^(c2?B|>7+TSQcfUKJ{h*b`h2@rx3qgTL9f1JzZQ~< zZCqgI*>2xF<7~56)F1}A)%urD#aSR$E{AG>!w+Nayy>E40x6Qn7_1vQ!qX%wl zYkR@o=@=nQyRAPe-Dc_Tc7OKLpVUQ+WcGl9SUBDU|1T!RY$g>V!KLn1DnMes!?AN~ z!4`?S5u-Z%<{0y_7J|)j4u|0z<61e>6Mr@usp&ErtYNx7FiLHdyKEu~#~|t-p2syj zN2m&B-hlVVaA3?$zxn?4`<}j?x!l)tQBa-r*_}y{gve9K6#5DQtG>oP_gv4d$9}v@ zGoa}$~QX}iE7tg5+#6{DbH zeZj6GY@0}VgG}r_JCOB{yv*L~B{%Z^@SCVnV-}Kmvw5`yIOP?raqM0ZxU%sTBl9S5 zp)<}Eg0Se8kM~W6`qx>8T0Yjl0rk&(@+HwRgF0=qhH0Ixu|FXvaijDvsilszijr&4 zhPgRp+a^uwZ}|7B;hK4tj5iF!v^l(7mID{LZXq^Rt~JNwNI6iddmT__a(P>`MIAlH zpBZmivvHh(OfuQs?+oCT{z>sVZpT}cD;NfT?D;?KIqw=giuIN)pXF#QwFJhu9lLn zyqL0X;n>M2({w5Qlkvsg$=KWGUUPrd7P)2jxMrLMbEabm=?2;wcKOc6RQMDFh)+wT zm66Y_mZ5XCpTSOa9UA!a`r?AJWaDKY4aw*gV6(bxH-6U5y?=KzxCLfh3QdC@7V$MG z2k=wlu8WW;?gSW)vzDVmEDM!5;)q_oBOg1QQESE<*aI7!>^aZK1(;kOE#ndl>Fj9l zQ>AgX*2Fw`CP2g-E8q5TKK`Mas?o;3i5*qbf<9_qC=XTPh=MHus-QH3QnurbBd^(W z$ag|-?XWP=gIotSg}*|k0++AozO6zEdr!}MGlw<<&#=kGV8>|*?}{CU`u>!UJ}|n# z(hGib81}l$%MB~&y~1#ZO`jv=*|Zk-r%lW4vN$lKesLi4N5ey2HuEj`@qXqF+-LJ0 zo|(~aEK{Gvxu)xQxezPN%i2R)$HQJhC-2L{v@Y|O2kX9Yxg-X@vGPm)cxn{JKc*4e z*d#rD?&H`C0x$+nRZe>c(h9J}K^I=6yE~zU4U}^W zt;#vH;=^q&QYK|LZt^a!FK{*kF5~~HsSr$VyP@^o!ph@~?W)ypS3n<|%E8FEU(=LI z%Z)`Khgq_+lDa2q#q?D40P1_rS(Y?OQ0#-11tUvbW7OTbl?114=EoWZPZT*XBpbQS ze2}>WRROt~op~2~prar4V0>2uw-0h5)dwHuzK3wdE~nBB5_Xq?PUl=bYg)9|&&mMz zI$hhKt7(O2Iz`R0Z?4C3fa{o7y7w-e*1fh?=-iB~p=P;vI`+FqIoMDwHr&;a&OS1nl>iM><1vXPQ$7a$1`{b!7)n5mz%E!=SRtLCALYGm{ zjW{RwrHEARmeoUf{Tfph-eaqZD-!Wne`A!z2@|7sqkF!o{d`mSqOuhnbIj1_7xZyW zf$_?ITCrUU9jYj~6r3puGQOEs&lfYsc)mB#OW*2i`j;&bgtLF@X@var~DewRyT zj3`G~NR}Y>B8*FPjO@Y&sW0R1*A$S$gMkktA}viR)^`Thgq;26Yw|AU2i(CI-q=D8 zQA;(_7aW^Yt~c_q9B}x%iK7lV{N+H#y?&^ra&JyB@G`gC$~Ta8Ii4&9ht_7sX=jX&cA+ zLQujHvA*KIRirqBrBR2RjZTgIue&JzW!8doQM}Dys_j#^_k`%&gBu&!*}i_8 zWfb14;5`Hv8y=?S>q5Pj!<)Ul2xJeXytjuMuM)BZfu8W;RV_%u%(3FjiqsG_t zVOnX}IN;d0TQ41VoBkWhqMdJC^{;Bn-_MgA8q3KU%2uKy;E!kUHj#lEm`qJ@@m9v0 zL>gNnO%_C>$YFH9TLBTihsGt&j`2OQ-K#`@PH!+>PYd;#fNzVmmRj_gs~lS!hupcM^xj@<08D3H`WRw4UE4AI7v!28GjSr zP7FGlyaNVt1`UQFe+7oT2aU!-V2E!p#1B7!n;+fbow?+hX5q*ARtA!TXf>v!8bw&* zS9dtWLQ|=_`GM5ojS-kX>F})XPKHrb6r&x-LiPB0(4Os`-Kmn(1p`-T{b^~3|F?pB z75uz{cPsb>0&YpMMXL2_%VBXIZ!}n~RGpJETe82X+#;;Q-&Y{VXH#Q6vOFU~ZE1Z} z#o+KocPVvMPUK2sq^*w9y+s(O^p@}34bhNQ$rs$-EEhXcs*Ix*=P?>HLF_P`-);Hm z1$Tx<2#Y|CvWwV6&v6F$M{4Tx3jVPIJCj&oYx0*!dtGbtG!efjKPOPJx+jS>2{p=- z?f@*XBHbFrO&{;1{=xlQSueiG1&`Fa4S-)?7e{pBh3s}6-z|5&82X+Yr26)cI5z|Q z*;1Wgd^rO`j(r&mubvBESQ@q{jDaBuqAX+J1#(>~OF55@S|+@bOTFQrBa_R8*R+P= zfsgRj=*D<0%~Q%fHO%S)h1oO&tbEV!Y85bZ_rnJorj^LDQ;M#!-Nj`vXd`lY40?%N z9#d!ya(PU)A>KP2QSpd7m&Z&{|0)Y}0-sf*pqrvyAj2i9yWu*bOP2<1BARE(=ECoJ zn|$N;e#bIE7ll)-)vqv6?#$C!ZsDz3{BJt?)0D{x0?RUd0Ub|38Xt*%`k|+U*&3#>a{59oC6#>u=_deH3jS4A)5tn68z7 zS@-7nzIF*;y`{5R!EDhyb`hsQ&wta)#<%Khvv+J?&@v|?wEeS>h<1n(c_rH38m>$J zEGUC7Vxk71787rS_(GEF71;+6S6(f)B=UKK)TP9CSZwPG4Uho5 zG`pf_4Au40#uzs4`zw@6mk{u7YANS(cZ+;?$#LTDrZtUzR1rer1fi%X7Hr~$Zu?8 zPpqW>TMmKNf8EuAl6@}S0Z3HOVKZZ}dFq~t>~f92z+N+Ny2V8EeOnJdaz+o+s=oM))ab!#w%%#rhO&l z_-ap46K-~A*1|)y@|D;Kx{%nYssrSk+Upz{ z+j09TeP~DUf329EzPn6NsX{m>7XLs#e}i7Vm%#Z4n3V$zq0Fc#d5v5KD2WRwb#vOi%DhwmzODke%kgxSHKs`A zwCm=B`aoI_;lEe#ACxU8{_G5-oau9;K2r~0r_E8}+zm1SLy43tgjBhtS#GKy(r6mj zVAqa2pw0@YKS$yIxO~p#I3b|AYnaFvj}V#--!Fddja+{GGCa#Sg`C@BX5dNxKt!>w@; z5Efi`n@>V^iaU{P?E(Seg#-iyUj_Ie5A9M1Q#qg$JhkISG;XErxJE%mdyWJ*f=_}j(U07i7VJ8S+ z`}Kci^T(8Cck}-{RexGd6&MIVqL}SlqnsbpRTk2=3ix>g%`0lSuG@$$Fq&s?FH+$-6FFTa#K@6inR zzxVD-bE5N{)cSX_hi}uEutadM`kS<6Zh)ZuBLuEz{nfvZz_vmn#PvT*&@uekp!5Dl zA*K#*ZFo?JcrpsFjHyFY(d(fO^%ylmdQ6{2msTyHzk z5k9D5MQDTuy1O$k3gcZW`vWR_8*lFI%kJwD-*$+I@Pn$W+asCVZS9e$g!@$MBi~)E z9ijm2zGZvturrXfe?kMQ1JYI@=GRDT-xy|F?(53*oy&A5&Adj?2Cstw%~8@}`cl7z zl5#_wqFCGfz9F2}kLCRI){`2z?vI-jJI`nzJ6$X@HmK8A)-DAE9+Ef<4N-aja2mmA zAR&P{j?_Q!zwm>N^ zf46LsXFR3>SM@$FjO3jQfC5uUZjB0;bH&WJ+(2`n^$Tz0vfHH=-J<&Rdf_;A<%rMm z;1sB_yp~EWgKmnGLmXQ%i;PnMFZDh4PRGoHU_P7|fj4~)pJK~M)$$tT3!0`Z>2mth z&VfOx_>H)e!k7|@8#zcPpRZdcrB*HCv01uNx31geBZ`E#k;o_;d8)+`KBeFuEzloV zU^03mI}I^>QEw$BZC_-hS&RxbDFT`GbNDaym9aW(fm>iC-T@h<^*J*%l@4=k$Z2Om zQ=x+cQevns(Rvv8!i|GoOHByQAK$vZ-?>0{2Sh}3UcHH8`&e+tNKkJL1?*?GIQT*h zZJ4DyYy)is+X^gtewjUP8+#(XG32-$V{cqnPEO3R176nDJhO|47WRMYIl)Z2y&>MJ_YMd<)X}I|D_bD`Nim$ zTDxCdXIEgT@dy{fvvT z(Q-Ha>}rTDgQ6)^;T=YoLD@lt;i0qs*Nr{VKI;EQW2`X_lVD;&^d46C!3Z=TlpNF< zjE76f%}L}+6Xe;`+Qk@%39%P;>43X$5weD{n2(EZ3OU(y2%GT3^HueYjI!CGB-w@4zb>pa1NbBvV zWA4d%L;lb2+SZ$7P-exxU4PDR?@xP1I{mtm7UXTrhT@g3|5}Oq&q_?Cywt{(8;3Ry zGbdAwmh7>=9{&&be;a@#pb*Yd62hp)yCD^dgs+Y)nC`rq1M&G^}%sg z{?XP^dUS$%vefefjq5j$H-=C@KOt51R@UyXlF~K1miHGrH(2SC61ZViN zm>B-50Q;{g_zeZWsDQsjcne{Kf+wnCsEN5ths{DbDoT5-4d9VZeO2Fd4^#H|PxbY{ znRQ(AI}8%->ZrT+e(u=!x2go;EGN^wR4y^w@Ed=f6`yZ!xWYhj)1|4@QJY{nfC(cb>Iy^W4ajzHi_SY-{G z``VX!l|K?YI1S488^f;}KySWX3mofayj>U^m9}s#FcF7Y6(8s7-T2C|@?xwDt zn5k1YI6_ChMZ(9`N;{cbqo2^@x&otsBtEn??JjE25i%TBApI0>zh13`3$u=PGKQX= z^X)t>V#zTBzo1x`lrkpYZ!7J06^MgpzFC}wz1Yomyvjy|08QGv9?6l?k$XVS<;qStw&=nXYZYMF|nS*$$v$aI9)^|7QdqE|F!Bf0`hiZ zQMTz+vE1Fkl4L2#u4Be0ey3o0+H}|A4K{mQ7Uj>yx4PHOXD<$`=AuEje&hXU%1w@u z`DNApmlPOlM0`y-qv-JE>+zMbx=+s5q^(3-g!l2)d6BPLp}Elv1Z%%!j zX8eknmDid=@_>b&oH?5NRK1nDLoK}xJp`a4^ z?E{w!c`;dxc2R4_x-C$F*1PIqbf>ZMt9GZQ92U&?!pW z$2h3JLIjK`2fI+`MoABIej+Y+JDj4t+LS%p)z*6E9lG+4+FsIy?8%pr@PK7K%g1e# z`c6Vaat?%dHnOqbdC!bk(a^wH37#g#s_sn1*^QhdXeG)A{U`=k9~7svF{USK^eG{& z8X7nVZ~05RxUCXbKGGojQVK3Er!|CncJZ>g3@@B&UUgUbz48SgYb*kJ%_qF!Of&@D z<$6~(uk2i2? zM6pK|Jf^^4=dxl3<4mpUIz66L<_8tb>TynisBp_EbD0oQ0X9VQB z*ui$<2Y7KU=)kVGU?+GeA>qG+mB`K+wd+{q2`cshD;pkEd7~?8$;F3an#)vP2nI7*s(c>JIdux^89w?x4dt4$m|BDGgLvkqz_6iYi|4@PNA>nA3bqXSD&Kxk zLtf;i_`3&?vK-NQeH%Z@VaaU+$j5E~zmJF;K(-J<1LtL42j@#k6sur+Z00OoUbDFZ zf+0&@axUAXtHLJ6x3Pz2%X@dstIb;g4EQgj&}^aspv%%g;N70^5|x48J8CerZVq;o z-MA~Eu_!+T%~Ch5vQWRinc~es{g&a6bfL07t#q@T)@&TI*%(FR zP6)xf>|Y+vI2jte#JAWCS^p3G>~?<#+l%8`i};efQ>Sb>opfB;54r|^j(@s*Q&Ld3 z0@Ei?o;r!cypt!xhn0utlc&(-;U7sRCb@^qB$ZB+F)r2USRVM98aR3C^eKC#@50|C zw^EdMYM{&4;jRoVT$On0vh8YG1~OG~*qATn|&`_H!{9y%;Dy_Q(pVemu`X1v&l^k_A z8^|3&&po$)3(S@NF!!i8sJpw3of|F`6W^0NlFJrGz1!R6`QP-?g>Pr>fs*Oe;bO(tC}RiC zbvs}2_cE28-xvQ!=0oT#Y%TAPGAAH~#^4W*Y=5iV#n))xO_?(5KR22THA2EMeLtcx z^kW#KeMxCoGh`ZHHi2Ko+M(*h0?!0lF4|1n<&ttxoew|Bw|%VN6bVd&(`C{!bIvm_ zq5BKjfz7zCi-DlveltNaIo_;~i;Rdb@{xP`!+d5J7CKSmRXl{+uO#RbE9i5LO^lI-E9u;?+H{zd)KZ{_(ZVvC3E3N*I_vYNW4!)+2ZC}d<;4F3E zbEvNl$*HXlr`3yMqMIQDd4EE0ZLZ*NY$|TjVNIEFM@T(HoawXi|MS|z27E5D8cjdE zZGZ6@GE(?`an1cw;Wd#t{y^cIlA03B z7G5_2ho5&M!?!z zI2tfTS#3sUpVijFhP#!LFe}MQbrOj{9*~de5OA13z;G zd&n3B;KR;tZ5xDrN^_Z0kXNw7PW-$^uRzl|L7Ms7UUToit;?P5XsECc`V8vp_5ggD zNZetmt9wE6OQ@8XV!N~}xoRu+%^{Hz8j?8WlFtn^+b}PI+R`>aM>5x$4axjOs`3V5 zfH7YY0gT2bF?i#^Py|91<&6xcHz?a<24hZipT1GoXQAA|VF)6(*y3KnAP;L&^NOCY z7k0T3uhZ(Nz_|k^n=Hfnj)aU{M?zc|0|ths{%F-DNa|P{55F@b5)QR*_;h}G&3`mX zz14-)@O))aM!;zvYpbtdWE*flpu?*M7=MSNUv4jEpF7XE>+FUd7RK@J(Y1%e6zk~G zt=lQb1!}pe8%KrM9RX`-s^r#*VfxF;@e%+6@h0XUSZZ)tIbOes)40035;ndEtTMa4 zxXg7Kbz5$cFJ6mtAFN4t2Q33LKTN$-gIhU1on;vLbif(5Mj5mS0rpY8?Jl8iNtO8; z*DJ_A)%iNk>NR>?vFBf6ZY{q|%N;4A-u!tD-hSFLiIIo=?@F z+QIJs(A@nbK{PnewQr5}nt`|aJHxA=(D3q?7|pp8b=w$XYyUd{>tx@>6xzS~TqILn z$c1j98VRh1OoFeH1Xj~cQaM5E7u=jOIV{e9oVy>48<4DCNY=~7iOC?T98d~i={~Na z&=se0HH>P`5?XN5YyS17){2i1LFuF6Oy}kMt+_ z8cq;ac=9D4dsWmWIavGf4O(jQCJxWYz8%1yxbcskzMAjwByLuOSP``bpm(wuRTRF& zwclv%eJ_;AkQ6Ceg?WIXo}^MSHE^nVn;jo#aJ|`cPhh=s@mAg0I$$CzywqjgpAKg= zn*)qzSXA&Q+CoWp9d3tY0XCHxb|W+3>KNa~R!x@~E?g4Y%6LjK&5~nZ8(D~GnCl#z+X?A&$|~}w?HQ;`dszNe zfZPy8XTvXQ)%vq@XP%AnXtkP7L47NBRj2;0R_pty1&*X)*Nkyg+sBIG*GpXC2^nQZ zYH&Vf^bXuo+I|cl#Fv%XkS;`S!dH{0^=<21NQaqNZ$O9tbpEI{35VNuT zi27**|F+=lCn)W&sRwS;;1z`fPU(#d=HZitRY5XIsoj2>>Xk_iY<-jGu!kI=5=40foEk~2?Gh;~}hfmn%!-bA7m#Hq+xZ}dD z2SF?6RTviy-1I0{A_*9Vh{PJJtIPFJ#E5A-MP%ahx;oQky?Ix~&!V7DQLA zAMbH;Z3p)#zq(Rgn9-3Y6r~itK&PXtmV=e!4qw)fw~_v~Tfgm|aEu$ttLnp|zPCkQ zQcN_Xlj)2oHmblKw0cXgVi-EktLSKNYNwgzXRhzbnOslklYyD_#%kL?i&R}qW@FNu z=*s@$;*(vwPuAXj`a&qx*&>-(({!f(g_e&F0pX1V(a^)~>3qaUi0~%8dQ6!_qZ<}< zE5;6j-9VI2l)KwiDp6RXK1%@klatc5lVIp7;O;My0Pk3`Sm0tRe>`1GyMuV<%|bSp Y8TAw2<9>$^wPRcVkB$r<9>3y$0)N-zTL1t6 literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/__pycache__/ksampler.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/ksampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f8d88acc70edb365cc1201e59dfea3e1c5979569 GIT binary patch literal 7538 zcmZ`;%X1vZd7t;r&Mp><2LS>kz!7DOT*oHmC{`SqCM8pX#4uW^43?Zir7{^#4}e+h z&Okl0B)FNaN(E3^(h>HZ$S@3Gm`l&9hP{y%*Y{mZvC?O*BT>}BEQb=>(L3Z`vnOlL+{-!SCU+%WMp zyJpYYuyl=C%<~Iu|{ll zruNN^23n?B6D`fy#rrff*0t8`p!bxAySp*>L!R|h=I@5Q@Uu<-VOVU(JV|5!KF^-! zS$ZS)cfP}t&CPzEWU1dx9`let`?lW;can6=PYOQ@yInuZdb?qfJnqK+)1=t;Gg`)9 z4|}`anCB-rtk!Pn!bsB-9d%S^-YAMVEwJdj;qH3)B<5(WjAo#^^dNnb?Zj_9cvF5n z=J)Z5EX957uu0tYRCGK zajYGg$J(Mc1m)3cU1+sRv($GqextNXy{7G0s9^=!YVq$CRssH%MrqD#`$nnn8$)w6 zQsZwGdSNjWy_V{&Va(7fw2obB7-cf+&;&Q^Hg~Z@PWwOk3RlYzW-m!mgnKPV*hR*p zZC*#WFr(eRu+x5T_ZhEIgB^82gVVZ;#1-yUTjFdl`$n`Rxh+Z|~%arMGsUiMeQV zD}WSrp9Pa|HnDO}%e%UzFW@(er>oEF9_meRNu}`f|Q7Lt~3w-~;W%ePpx(ou)a@r@P>5^&Z zB#%BsADqLT6FENCIvV(`m(X#^Z-X?!in5g9SxeG1=7A)1%M|(^xRz}7LVg8PKf|~@ z?rutk@kO)<_YU~|n1y#=qV)|FO>OYf84}zIf+S6fAh>m>n}PrNT~yQ*d~QL0DA>&S z(~z@(boX%>?SPDS-Ul8CcRZb6#oBz9zUg6XufH2Wj+7KSs)0MyI|HeLqJ~MJ!~}|L zai4l@`7yZ$r|sf6)gXYt=6MkCYiN;Y;@?1x-Xys^-I2d01jp0sUk&EJ?r;&Ee0&j) z>3)bizk{MQjtE$_V^T+xX?>k9mS%@atz&f11~@Pd4ZtTUGun|my@H1U zu_z&g0@8t~n;8h`L0xvy(@zU1ow*ekV&!R=7I_fy4Ce~Rs7A93o^|;RY)D+r0B}5E z@x@`{8nhJ(9tSMm457dm2TX`r1aTS(X9u+3%K7hMRV5arZAvy~c6kgSkAor$l%BTG zmlL?ujfOF=H;lo`S!zBvJ|d2eFPc}ueIIweivolq+=L{9fL3WA+e6@K$G~j@BX0uj zfUI?#eY)C?$IYSKuwqP=4SSzl4_g;P3ZWuNSn(U$tv zswg7Puxun_;7LG?kYdHZiM>g|RH8;0({iN<@)hbLdm)^DO8k_1B|NGIBX~@~?O;zh z3dAlFwKCF`SFnf_(0RSC`@k~CAW`-7!Q5Grjz+f(2}QDKRY$kt4_7TUXX1rs{(ed*gkTOwW?9|a93~}?7H$b_Vta5>SZrVpF>BB zI~k9W@>ju{RP#p9Y1Er?HKRIHonhZ#vvIZKZ5nKj%^%oUqc&QDzG$#-vP*2?fYz?F zMRpm`KgE{VGM)`~g)&aQ0zk{KovHs|-_k&08{P?Z)ci#JxH~AZAY&l8`ZqfiZkG5ly zJmIkTNfO2U_hmC1+fFH16Eu?Q4}b}U!Ko1@+s!AqUFhqTI+9NXBj`NLwGRngP*$z$3PymsonfcWT_*s<-*O*t6Ne|eRM?5Z)K`dO^5Er4S0Qetg z@J+}y;p8)H;l_3Kbr=vm6;;X4fE>PYHQW;>-_AH0w83}p-t`}j+%6wpIJ|CYSAFU7 z_+zg6H>G&^cYo;rR&RaXEG2yZ&$K&!tI0`Dl-(!k6H{>VW1!?nY#Q=RB#a|zhVctV z!dO>+g?i&ZqS>z8Gqx*cP6q~R6MT(ESVNHazYknjq77lBAImf@eEiMOeE zkBawcL@nz>eU$UWFNp647(hIdPNh?EM*>Vj?EJ6eK!8mW=>~oz*6tUcI}2o=HR}3c zVItD!DDXL!!}Xu{A{peVH*E?}x~2E=eN-Sf8iOB6UIqY!WLf*jSb|pqNp@g3-4ej@ zORw~Pt^ck5XU4ELtd})FmNhbWp70)rMYIj#LX`A%VeSQwfFaxAZY=6&y%KR%SvBdU zgxz4PpRh2Eh-u{cFQ4x{H?yAfwkH0cF!o*+RxiwVq@h6o;*2DY;Xth^2_wQDkP9L$ zDE}_n`8E}sRFF)I*?t}e8BewnAlgVs;NQX-?S>TV@eU1@s8dnP=R!Yg+N6gIbczsm z^f5Lk>*h3wX9jqqo~pEdW7NF=vZe;p6JDu#=g@|bhxG0~Ek-V4sjtHgZL3q-3LO)) z3%%eydfF%$2W?zPhA;}tEDQBgZ`-8GB zpMOx!7UbvFF&8n@p;2C8l_lwj4X>sPXImD-wSfSWS7;!_^Y;5&Z3s%7yWhkomGUyjIkhU0#DD=oj_U zFRyl{7&$m9v3WKBQ_{kf@@jdde4$)smkw>FZ5qSt<*MB4KSMKD zr6sryZCq$aBs$}Z>LfvPxby2MJ_hk|(CLq%@Jz?T2pg9`7!^j0Y`=6sg`OyZL)K$w zqH_Ua@qV0trYkj?7+%ppN~S=dFpACokv>Rf(?tjkdwb(l+GH$#8t0v#x7ISb1kDLv zN@p71Bczfdx%J7<={mW`$;QKHrEqT)s>ulBK;P7Ui1;hJSM~@2Bxoe{3#fcis{s zhzI~io{GmP@Dc8qF0BP=MK}RHCjb^64^x%~-DD?LR>hVL(j!*3jdCc`#}=#dM2Olb zl7kZ!b^8q43*syfV6FvC-HJJJoJ$qHL0?$T;{t*3X^__ocM)nm!A$<=H1v;Yl-Z3_ zX#@ENy~6otNrZ^J^`Zit$22meLarQBZJ$OqQ#efs*@Ha(un(MxrD%x7bKk?BN0K3G z2p^C_8^i(N+l05F5LNT8pBLU;tE@Vqg=MhDmC^&LH85hk-H5kR0 zWM6qio2qp)kOf{0(mbbNuL4k+9&qp(a9aX9AJ9Cq?n?3-$}ga7V}Nv#m^*hAA{}xL z(?s-^Qy361-uj-(@n832zh<=0YRj8Fr;H@C1> zAQ5|H=if=8V|HF|ilJTQ=nf_G)-L1(|IM+>lPKIK_9Zwmbmf)R#O2viK{t9g`(`kveP3w%* zj+dZ+31}yWKL@+O11%y4GZGTlJ0VkJ|hR&6C613c~0(SWh*z z=J;zg;0IJtdRV2}KA@WP8!BlQZ1Irs52=X)50%-z^cJOo@1HsD10;c^ec?wm>X-_N zX+(Nq<^|_;aQ;IoPASr&S81E>qDFVlP*BRwm?0QyeBsuT2L)?)p{d1O@N{Qq`eab4G0a5#MpBqjymWTaY6QGr8}MMyx_c24uD3YH~Gkobt@ zi}~y`t_~^1*gxZ(&`&JE9po_sP@VfT0Et>jOa1_ZogtJYnF8%Q gurWwh-6dAaf0QiEgPR==jGNWVv=p5NsWysgv|dEkZ6TpJ!FN#?tGB8CSp_wNNm|C~a9kZs?H z?k~bYJ*vztrr?66Dq!4Xk!g18S7)C&?Yg=T?RT7j7l@d?5nvUmhgCA;6gltik<}Hwp909eRd9LrWuf z{dg!k&=#9WFnq)sl65420z~WpX~UDDf4a268nnf_@El)VcF=R%`rx0?AFhA@eBK}= z_?HF~ZUb|OGcGP%0Vh{(`~uQ+pOl?3`3nU~|3rUH{zyNir)=6eHD=mOpP4gfR-d(I ztF!jV2G_C&_H?Z<#Vcif%*I;G;H{q$;Pe#7P#p7=0_P3Iu@uKXZ35>l#i=QdbK01Z z8J)42F|%g&tOn~eX3g2ktOHq!nQc$I#ah`Zv=cU6&+A3&gmUAMiKyrn>!o?ZK4H_1 z>1MH3Y!oZdXRBB*wl)FwULw=&_`~@K3r2j-?P9%4iWaYJldVOMH|IS}=<#RnU=I$f zV4p7bYCyY*ZJ4vA=6rcR!ag*_-}s6evAQ&(y)>c&?X~$#T^tQsZ!c-RQEV3LBMq=( zz5h(6VV2qOU~FWc11U|&9+KIm#VM%TF7+{j4k>6!&a^3uJEc(q%x!|wQw^lx7$r4J zQd*FE+Cy{7$|?tchNH(i*AH2?_YwJmmfF}r9zd9yabyIfq?pfBR^Vx1^u?>7g+K&A ztX6pz#N0oU8lspfRzM=_UYdsYT?IMfDstL9nP(7Y7Y>uFkd9vD=e`JzJxn^;fC0hi zzMBmpiF&%EN0R0+v73u;;)1rML%9k<0T`DEw?af|R0^!9VUW3VvC1_S^>SwtKfw+! z#N>X0Bl60#vi(WGy?E$@5*{2#Px4W&0$Nl%a+67(PVygvO4(~bOvWzYdE7A?nnQKU z;5HxG`YN;=RKI|)3td0neI{+^Ahi2-n7~5WJG+;se zK4jBcp{cPZB&)qiirOZbI)zi1BaB3=KMz8Ikfx&Y9M33m4G4rLO?!0!S}a7QKAMCf zoUeC9Ud3m76o@Q`F!c8Q%BRx4JAs2BIQ9kR8ywdMUg}F0$I=-=gkXGnp|rS2QgDB- zF20M4?P6ceb8~5WE+d@LmHZ3ey8GgB()tYF7F?wM{4*RB<%dLbex5!-=CLTsGjC^S#_vpVe_=DwQ(1 zep#dWLb)G7Y2xJ*0RS@q#U{qGBxn9gs8Qj2cp;z`jAXk`9I}qMiwCde8blRm` zjKOKeF5vZ;UD!qkcs4!$(f?0?=a&*#K8hbA!2~#mlZ>2Sz;%u%=lLoUC3uWHJkHMF zMDiArZy~86d4y6tW_u0sQ+QQTqrj>)H}r54r6S?7&Up~Ib1y)Ff!=WE!pa+~OFb&i z)fbv}F-q>J0Wa($uT?UPn#>&mswE=W4 zj^ZW|*?|iC=}c-C@hg;mjN~IEdq}>IZ)~B<|>_)LP|f>zETZKC2czQ@HPTYOraj6ARZidKe&Wrn@50$N3rJ~A5g#M~B#DK;>A$9oJ1XS0|CbRI^ zdH&YFX6^3R0>|`LrRBOj8N$43c>^LWy$>Ll3WQEsQ%IL2VJ0xmWGCd&ZdCy9R(~Ox zz=}j}13DrSTq=$MBL)5v0SH#VbHc)}S8Vgs@fex literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/__pycache__/sampler.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/sampler.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d749938ac4a274a6f78acb9873e6f93cced235dd GIT binary patch literal 9799 zcma)C+m9sIS+84HS65%A=RUKuyRKc^$+o?oHI4%cG8-qhH?~P8AzorKI21K~YG$gZ zyQ}w9_3rHCG>JTG;Q*TeMktAGF(Z(`0wgYiARhPwctgCP^U4FVP?Q7+fyDOm`_Adh z%E-C<;N?l&Sr3F6>jpEK)i&2n z^|aP)JngpAan>D^%A9tgr zZEmM&7KeGto7*9auP4p;Z4>oTw)~vx@+Y2qT6Hz%PonRCm;niZIP%UE|x#gbPlu?x&+W#kuP_n>%aum-EJ>Me8MT=#TN z$r}Djs88wI@L5873U3yBh}GG&YEwGWhRW%tAJrSR#yfkM&9GV3!XLK)r%`=&kxO4vpNVh zJun46B&-GqtzpT6k_TCidcF3|P`xy4Y3|BGmTYw*=`_v^f6+C^ja%Zo8a=7#Th@-n&j_n! z=jMSejFx?9?%0%b;Gnc{Xwq|3a%c{$+|_HoZv!^5QFs;X8b!d-&0Z`E3VNh_RY8Cr zl$EElxY*0}kd%~g)27b;1ZbqCCC65#RLc?$GjBU2 zFL-lWg+W0SS}wDeY1?DUvNO*c%-%O&HkI@^tZ>U4K$;GU%oQY6PRrXj4=oe*kyjM1 zC>(kBhIL9u@=C(Z{e!Y798?aCL*U%FwXaLqux#mgX+@P*->0;CyzR>IwzZR`Y-++i zuj#&EG`0Y4u=;K1C3{dDc!QD})wC#%`#2-Kae7vi#_2g46Z+;6U&RDJaR)#6cT``u7In4@d3Zc9aF%(e*gU(_kYGSiA{%WlCkoQFA*OwQW+1(M4(6JPHkI zUkc;EI0PQzwsO1WKyCn~Eo_=Yb0Q;!W*ZupAR-5A{s0>C^F$sbLWrXzreShISPcMU zdEkr4`xtH|1Fqqio>?*XK0J~ErK3k!bT`<^!Un#Y$(kN>*z9+D95Xk2{5|aP_`00U zuJJrPULwz^Obj6GB;9^?qIf83-~YHg-A=c{_p2tC6&D^ilimpqa%ED~`Kc3yoRkmN zzN9*Z&S!iXInu_U_$5k}wfRD7CL6x!0m!;48IDGc@06}fgFM+$dHx6=QVe0rldYs1 zwH59wux`|4IZtzBY3wFY;-`pEpY%e0g!)v-wxb?EpK>1~@;K#M-5wMmJ}&KMuP<%P z7cWvN?S)2J1EW?ww1w3-@3Ii7yXY>uef)ZDl$anIhD3DG7 zn%e@1gm-uR?#vmBV+B~^AH`SrPZLoh{S2kl0z8S7^m)`}U1cqEa&9>FtWnf!ZS6iP z&0Nm(VNZ{Dn_l*Uva*jbSF1Dz1v#DNX%iThC(SVKHsOu*!8oJM=r}GxgavB(X5FgL zy4|7Q-tuS$?`q9|1})!aAWizn=pu}x`TTj@*{4Zk(p;JRIT#W!5cVzpQ9PkH59~u@ zANJ(3v1|+$P`Z#?C_N_@C~uOJEs2E|v@G%##iFoabUdCHm?aiphh`Pl>&BbTzG)iW zIpOGfs?Kw&&M@bAO{}Y{*6#~Q6=9Z^81(XOm?;2fPp#*Yu%W@02d4%rSjA;=N~~-; z12TI`!V&hSmB1?$ORt&G*ac8e6hTYE1NDUuS{7x{3UffKq6%6Q70@YB1FefG&}lIZ zIwNL4XT>b&oR|Zh7v^iGnExa5>*h<=;Pl{(I4#a>l1I6)ru9+v#fWzykUtxID$Rqv za{*b)Ne;IXnOU&W&x4I9Z*D)50RVy|3$DP7dKOE<$)?jVSQ;s;OnM`8R#uYE*4Vg} z)7Sb@moNk_qS{_eIsnZb{#kI5iwalnu@$S@;bT1QrTrWT7^&xvPf044w$yqI@lmPC0x z^c*Je;Oc6xf>m}M#9Q%!$UbnvzvEjT($-gN)+y7s3D4>0zU#ZbO$e;s-UEie3;ccu z==&4LwSMdrt^ajA>wlb*^&`i({@W>AKXfYAe>qj_Kb@NOAI_BZ1E+3%-uQtXT)$1nNO{0sfpc&i>TK=A(#Zs^^CklKjH*zz_yRi4+uk4#$C$|qEssQJC zF#|X$hX4OuAX& z4Qp^J_r>DA`Lyw+Z$dm*2B*<>1tzMyWDL&abQtfqnNDtvy2#aWa5L6}Lh%5K8XgNlN{x3jkpC&)=# zk@4A{2qWO5k8qSFi@Ov^iaBCiJZ2%}h;+|emi8wLj9g8ns2UYnNV+WE<-dRm{FjK_ zOUA}RC5vgBg%rMW^RSDkmGnu^>My@d4m|%gB3~gwV$@jRF&@%`S&UFv%wz$Yd@JTp zQ?9ewPI}y;WGM_)urCbBT_>*=scG~daWsm^X_SfOG~o|LBoML-lTNf1%Tf$8kB~H0 zrUSV4+ig8o2M)uQv|(_=OpLoaD&bulG4pz(l#7T<+-1`4rM<=j3dcA}l|nE6ixg4kz4^W&)EJ zGtFP7?_8s-VxH+G@JKt6-6I_+mP}1P4Er!Y5fR`Cn*B5G>=uZ(XxVdjJPYvTS{4jL z4Os6u?gEU*noS0eqhO4vi+sm~(W%lNj7%~;ojY^3gSH>^bARAZ`{3oHzK7d4mn`rH zaC+9>>b|4suI)7DNryeG?%CNWCwrFkx!UyKN7qggK}S zZ!nFTWtb~9;juZSsydM%*d%wQJfGeA8bs5h5aTQwNJst>@vS)`|sS7&8l z-~5TvYCa7l@_~ReqDq)EM!-p%1&w_CTVh%v;MX()9?SyC=7E1?3YLI>6@lP@s4M)d zqu+DDItxfY2lOidxfbvZ%p1N^yE9W?o~8i?}-h%*3Dk8dV{G zfo~D{4I;lughWoE5C1JneVxd8BHti#n#gYx`5hv^OXM{ozX#G-(YB7a8@2&FFp&r? zAd_$gF&oO1Ibw@Y&t+K!Qis(Qatw$a*(n-fzDX(7sc%vJ!Y&NVrnY=ur=mBAsF@%D z;BQi;(k|*K>F0kynF@IiD5b0zMHZ+cq9)|td<-&_5ky(jwL6gAcPj-S?#-$RM-?9S{Ob zxXy!n{=bS^ z=?^U--_)GTU)hcX6|J;i4=P^J-FKe$pEPUu)hp-K8K4%qrpd(0X}!$!>MyDC(0 zyzji}sszNYa>N9?)4 z{Arm#u}=wuzCBtKr#RW8cNsy)7Kav!7D(+;ih)9aut35jZAbzX0m&!JbQKdp9;zB# zuieQsIKwszZlruCz)Flg;ML&jHoRmVBPM@&xT4V}td#Nal)pP}6Tn;xnj9Nei@>oc z*oB6kRGePXIh9(KUJB)*)_=@#!tB_7f={gj%OlNm5#mKw8z~a%H-@GcAEGy0mm{kH z99hQnK-tgL5 z$3E(8?j-ARE6jt6n3GByo+fEkqmbvs-lR%@I-Zfj1TRJ%m#*%`U15GS(p!_FcC%}gZ$N~55Bp#G_oB}w!W(mBqB;Y$dc4Y4a~f* zgo6%U-R+wjm}u#O5x@xhgg$kzKFE3O!3^$4FM@-GZKvn>Z$`bsdBT}dukKm%Sl38u zX;zeN`#I{GNv9z2pwA7gJs-zu@a1U-EgN)N_@Z&mag5x$`Jf3jhLwuHX;7e#SvjoO zeRCA@`y%SyKoAhS&e{9Uc)gV)ynM(_x|Ddc>1x z1L!yAql8~=MN}PePxYbrK)88t~KhY zgdeJmGmo|b=b~<{D^<~`QMq#E%2AG*6?niSC*xQ8A#GlqyTi|}gdw7CZS8gQ0LA*u zqQ;AIpLG-tmL;u(mtm8>$kmi|xN&%)uHu`uBu{Hy%C`@E&jIo`is<-2ZtoKxk8 z{xeqP-Yxv^dGlV$^J>MKU9ma!u`wfyVaU>E7!FNRm=e94%KaDyGd2-?%`xJPlaR}q zHtVeF|1zxV|2Jg%-ww+-;Gz)CiT^o3AL>6qoTG+%nU4HPYLwB>leRwjBHNUgh!CJi lH{GB=f~el!YpU^=56$B;_Og+Y>*5$*feu9VZ^@b~{vQuKu%`e3 literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/__pycache__/shared_invokeai_diffusion.cpython-310.pyc b/invokeai/models/diffusion/__pycache__/shared_invokeai_diffusion.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..644fb9f1dceace8413603bb522f5a9276bb2d899 GIT binary patch literal 15344 zcmd5@TX5Xgc?Pfm7Q0+7FQRynXc?AlS!>IrFNy6avMpJ@#7WIuBqyCs90c5hC6~lv zR|i;$++sV4Dc4S?o@x8iOOlqF>2$^qy*#upZ5}eyX4>iWy)$@9C!Kcu)|ouDv88_h zIl$gX%1+vcE{Ox23(on^fByS9XiZKQ75skx)<-p`rYJw9%U?vdy3ky#E;g5{OU)D26U~#= zlg;JovS>5m-`6}<1(;sJ%X{Um`}>#W+1>+Qv92vCOUi~$<@7w-Y9p7GQ(g76H|w2Y zrHhJ}W0S1>p8Bf7rr7i?g-!2idusJTnFw=0Unav^EHMF8`|ET6g{M1FVoz$tQ)&e52F!f+$Sx=Uy1K zxaTmh?sojhsqt1AI&Ku9n#Ngd-S9YXFz-N3{)(JKa&p5v@Sf9bv5xPB$?O&C5a7xF z2&`o8>P@$ul?Q1zfODqb>~*(&)y2Tc-0Lmo`LDG2O_wvm#=2YEzJ|7DowvGP0E!&_ z<&eFED|{M>r&Lv@RJAv-NS@AAS}fc(+zl^>1=d+^=vluAqQf$o$^4gEVbqrRYy=xu zy(q#hObXE^5VqO!Sz-dBW~ZGT+jPR+X48xKt`jwyp3|;H$*~&;3)j%E zQWW92J#G(=T3Pi+5LcM5;eBG*URhyb{Zorvoqd(>_O2-nNq;>5Ic*}PO*pCIgEBctFT9K zx6|?N6KyBcwvV#&X!`(rjIE%}Y1sx4ud>Hcex{#)f<1}+gZ=zd>{G};#6HcQ#(QUn zpm+w*N=PjBEc*;vJ$yv%1@>9go?{o;bLgYuJ;I)6FWgdq=||ZNi2q8mlu^)2f?zy; zrPXY=0x+=q2v)gh<9BV-v)xA1Zq@B*vk}_0tci_G)Uv@lHy|`@*ABrrzGt_6x8~XP z7DrQ*N1JYB*WAEfN1JxrZ+Of`d&ry`n0yyqh+*(-XVBPvDy80Q9_xS=SBj$Dh{_Dz~ZdDyc2O3(zFlSYeBu0oQfa~AV6xZP7s0R2`_-q@mis? z-U`=})DEb!YQuYv*${Y--mwrDN61~lH{;P`~k`myh5(> zGF4bu?Y76GU0$JzzND2;Viv~_Me?I)o{7ucw5n-km7k{`=Y{BXKpdpM9HMPfV98t~ zba*Eq`bmYa$HSVASF#^H_i_CQGE9vXM~Nv_;~Y{g){y2zxgHr?CUPdjT2wUmfs9Xy zS1+_(?l!?3&;l0(Lv#W!l6k?aFgj4oTb+$f$_sip$P|JF#``-kwF|au`)+fcx%SOQ zv}uEgq^V(_+c^(mu+enSkMNHZwH%TFyOkBZ0e*)>AUuJmxYq{1dd%Nl0kmOc`;BeS z-#rgtA!VS3J3d2~O}s2#kyBbpv3pVb0!MrW=;n_hN$x+2H;6}I zzwr{aIu?4@J0!bY-`VIim>bl*iq6kbH;-q1$zBIECCCZ(KqF{GjziW;NS&E^QY(PPauf7O zvf zRUu7)kC)I>xQs;mc3Ia|9T?SA^R5O{#sY7a?v_;TZuiNfv3pdOkevNdL7`VX0TPD-q%YkEa-GA=0SBTEr2bge$kdo=A z6`^jVK7`p<$(2b=Ph{uC-&Qs^_jTzOT{e#Ii=ozuTZ^TAjeMdoWCd=O{ko>ql zMG~Z9tCwQ05c%F(D6 z+&e@=0Kfruq3BmG`Ym)AKEIN6o9L`{Ge|Uek-27r9hf|gC?-$*@h(jl(ipm5--DMb zCMxO9l}WCo-_jH?IhC?;vBv+ zWUvrq*^j87M3KPa-{4_G=~)m!O0PgEQ(=tE!{i|5Fys8~T(1}xV44(ft34~q#nv9g z*0$c8fa-;z43SleOMC@ym>EOe*0Bncy>f6ip5PZz15-${T5R1m(z->AX7SIXwvg5K z-z%cDI9`gfiFh(D#|B`Uil<1?yG0O0Us0x}`OXEq*5RCdSKo3|n_A}2=^7ONB zp0`_=0>9Y^hhhc50Gf`q90zxp)D8gKul#=j>JaJ<@PS;@E3dq23$_rx5v^S~SUj&@ zM_^A{Uqe>DouX;eJxDEN`3u$pVUgb-+MPCepjb-8HSEWUbj}Uf?Xf2+_Bl5kqAMvi zkMMrZ+l{)7NFw4KEB5Q|?vTsYL=3snM!bhYp^|XiECfX3k?-bl4=$Q#vzCqcrVUz# zhlp}mm<>RB!}U9!q|;^#4bq4ZA{X3qLvO`?t!}R&9Cg0$E+T~8Xp%;;*Wm~vZsT$y zb-^>XyMdStd`F5&tO);ZMO1|Lz+Sx|O{x{+mR@+ zJAqg^hKD}cQDZk?HrDmAw05{|F!CCwph~60$qY=E`pVHk9HIt6;G}+e@&pCh-CERu zJU?PtoCv}7kUVlBNt2xK1xb{cwqc9QX7@|^V;2Zn9$@0CLB07%6TW}Oz6(hZG#s&8s!Yi2)EwteS@A`_|My#$4(ko_qX8sbe*xkBkey}{*y z${pkot&}EAj{hopNTx)LhsZhIKPk(CZ0`h~p|AnTDC1;r%XRNBG-p&cX5cJhrg8$# z=mh@^jYUAtvdF9}BLr|SukfgDYGk}+zswyGX2?q@g-=g$kaCQOiMYcyKS{9@{i>`} z_z8hI+#{GNd+J+=1_(1ny^V+pwS!aGee-1)%Mdd{9t){$L;FeTCEW1qod&jR?3 zke>u^tQ{gaXkcfwwwdH=ZY0BjMs2ebY$ql|;DqEU3Eae}`Orp{i6cc;lHYNnoAknj zSkyE#OY1ZuPQ#=m%D^H}qaml6^9@Qy>7IWJW&eaLB-3~r9;qbq5|+YJyH6jD{(Fhv zkrfyb0ncI-;52nV!R!cMr8ylrGd8!>Ed;oL)5X}_&jF`q4Ezh6YHOV1efTRBQRO!% zA&->rP-)8bhY9*4XYb&}f59afyiD;tIqxgK+`LCL)Cdxa7=+|vjt2W7k_cXSr0!{9 z{`1OPDsr00l|@e5&&6t=B#nsj!z8NiDZTu=3S7QNzDw^a@2X2mgiTR?=^aE@-qGsV zG-bJ4R%Aq`tlfjmcuCn)q>!toq+lVC4KM`oyJZK?gltk4Pw=!^n^vW!|Oa*C-jI%5PH{&5gf_ z@xCy)?7PUFvn))u^}mF_GXi^$z8s*rN$A=4aD&qR{6XhIRuqLCNOOCSKLj-`S| zhzKQ8CHG4*l}u5_gztIS@NY7!2UI$F$g6^9r`x!1F z=uDFgBWRgBC7pE5C!m;5IJFV<64&?9OLnISHJFIxn_Gx@BGqvj8_?+Q=|b70VFaed z`G$I1M{vT3^A4#YS%ZTRY0ZEeb88hnS%fpuKqQMfWSu$#k2@&{JAlHMiQaU>?d0V6 zRHU&l?b`*~92mF-0cjQLv0@3og3n@W{)=43FAb5vhi@VD_Ug2OWQYnm`KV8$WX|54E}vHYf91b^vwEvu-YbED_9`?6)9QSpen0 z7{RU&CN|VsPg_5TezB9gm4{K1v2zjcO^ElPALu>oz~$o!lIg1W8?Ji`igxW4#KH45 z-*bab`&vc=Vm}iRLF`n95rw43;({R#rQsFsq)o!)bk;bv)bYxAXkEHwXMqs@TS(x? z9w|rrO79LFOd+iLTj;QdMM&K{webV#-{yXTA_Iu&Gz>*SA zWWbeT(3RK+#*DULYQx_%wxQ$T^+Ar!;V>7KoKzN-IUEz~nF02I$pmEjw#pwBZ)^M7 zZAF-Y`B>Z6(`O6&KNPSvcu(y7D3J+^1@;74I55tO2+*4Q76ox9lx@PTxvjrQ=6)$E zFzh9L4+lLlPVsv<90pkEUWzSopNuEcJC2H>yc`$NdMd_j_KUZblQ@(K$Y)S9`-7Q1 z;=Me2SNOZIj%YL;Pnb8;dy}F@=YIuQCJ4Ho7M1p?v>@hg3O=7qAlL;c+5rM~4EtAzkuNL`1}`tP!RLSY%5D^{{dG>^`y@^qNM!Ynqbe}t#YsZ2Hhgo~w za`9RgRJdS2^jvoWVY)QP@LV^y@>IQR*ynIgt-C-+KVrMt2+rG02uGyuj{Th7)y~^p zqYy^yxvn9ae82!1-hHy(J@JqXNQM;h4(Ui+zxNOB;@3Ui@6*Pw-{-`6%F+*=mkQ_7 zC?EoqT~or*o$hz-`Uo?6tTM%y2w!AD!UuJC_#eopMv%-hZ@sfIh@T(|AZ**j0^ANw zCz+H-yn`~W+K7U};0+L6^@X>UXaUTLn%`(A1(?Ciq4PAAvXt3dG_nZ1(By{z znMt=M24iF2&D}wm7WYO#cSFoB$pI9YD+EhE#a1$b;hl7TFLSUrqOB(j2G*eqt7gdx&)n^x3f74p=&MfWtYZeTFS^S++iwK>~i6;}iOjE!t z*R6bF@){AA6oa5+GD28>pM)h8Hw0upJBUSslY_y8xKt^eBf>zZa96k0aGAs<1j{LL zoCf|esRq$v6?ba!81B?!6)lVyrsWn1QVr1woR<{_0(#ue@wdp>h;@jowQcR25smmYq^2` zSpmIkGy_2NV*wGVg&zy`VBk&=S%9dJLS&^76%T=kV1YD1Z@6nOB;u`-wELJMd{__O{ZQ32|iBq`YgB%h%mWbW|P>46-2&48A zRZJvvICdY6$HVwZRPtp?`Z0?{i@;YnJ&iCjIhKiD2Qvxpmy;uw1%bwRsoxV0Ab2+t z<7R_lcf7FU(A#YfXUR9c#>Qrp6n3+saaL)f_n27a5*{pz*aYm7TLfDWgCIJrl{7P_ z!5Y@?;Qmq3oL0Yy7NaF!L2H~VJNI!ckws-kjyREupi6KBMQ_ruFdY;FB_RZQ3{Db4 z7TKYvJ@q7XMT8?Eq^NK}QQJlwj`q5@z(qJ$t%CY1;47TI-nVWmI3fZ*VmZ98GZ<58 zPr@Lk_SD}ZdaM|7+k}4~K=8kygftiYe%eLhHYIeJOs-Kr6^P=nLSo9zAO6==PvSL; zdy0@JX||*sHtGB!c2Q_sJItbX!YLYGmo{Gbb6S@o9%j`!ZB{#_p3dDX*nCs~wnk28 z_9GS0cS8RX1B^vL^J5WE@Ny6526{!DJu-l|9AwjSYy=3^Gp&v=l-T`74feiuaHcqs zA5W0!1tV^fQJ~78Q~Ld~GQv=LJc+$?U>)z_hz@w$jHf6h^`_ICj;ARa+nb57ma=6$ zv#4NqeKwvI@zoa0j~PgWV~`Pf@VEIk)>%j$`E+bA*PBDH$Aft>8?=sR7~r9HpcC;P zOeeJK2ejsT$1!dma`?C7IXFqj@=wzm(Nfk>Ye>m5 zgM<_?6}BK}wkDYMUamLK3cZDRf&VBbs7mqtZS_8-Hy_W#2%O(4v*H55wul}|h#N1$ zD9qEh3V3rVF2+l-NoFx1z^>NEItX;j{w18;QeC;o@80%5IRf@ zMnMcAUSGkXKVg0LlWT}AUP8MhzwLQ#h;CR%By**-6%ph;%2p{MeDYhA{1GJ-(@RXb zKhFP-N=SsmKIt@*JlTdMz)1pBPC!xkA*Xm$qDDz6JE>g0?59e^Q``hL z6QYR6H^diJ{nNgwFQ@Nd*B#pelwy+r4jsH|xr3PQQ{jWCL0{M=mEA20BdHeQBcQPj z0oZhdbG>1bmoGymiJb^E#^!#Okpk*6O{FNS#o0E!M&K0EjXuX6Jgz4*z=QNW(<$N; zr=Z0l|L9cQbc!n0ubh;Vg1yN%JT|t{$&57DhY0M4Sn2RZKt!7qI#kvP)ymDmMnjHV zc-U&RWwDGv&b&5{Ivp2U9-(Y|EdYs2%vxE1vk8S2#5)+*=6KLE%h zl-a7`ld3-kWMU7S6H|&K`+T18r?0nKK7Wb|$w`sl2#|*7&r(8mAz8az91NAmSH%&| z<5c@aO1?}9(JM~IiQ_Ds#JP+~+w|0+gn|e>poC0d{w+#qdqYH%IYmE36pT|Cgu9fG z&CJOf;G}x6^z_LfzCPP*G5!-OK0TV8q3nZ5=+lSrcac~M&}t!)h9%J^th=<=td+$f zTAYwB-8nTqS1#wRDXpNX(?6$^(k0jw8f*$v*FH3j;?K2J4JA5C^nCHZtgR+As};aPi7 UE#)-rEbK?D@f`~m<81za0ElTcJOBUy literal 0 HcmV?d00001 diff --git a/invokeai/models/diffusion/classifier.py b/invokeai/models/diffusion/classifier.py new file mode 100644 index 0000000000..be0d8c1919 --- /dev/null +++ b/invokeai/models/diffusion/classifier.py @@ -0,0 +1,355 @@ +import os +import torch +import pytorch_lightning as pl +from omegaconf import OmegaConf +from torch.nn import functional as F +from torch.optim import AdamW +from torch.optim.lr_scheduler import LambdaLR +from copy import deepcopy +from einops import rearrange +from glob import glob +from natsort import natsorted + +from ldm.modules.diffusionmodules.openaimodel import ( + EncoderUNetModel, + UNetModel, +) +from ldm.util import log_txt_as_img, default, ismap, instantiate_from_config + +__models__ = {'class_label': EncoderUNetModel, 'segmentation': UNetModel} + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +class NoisyLatentImageClassifier(pl.LightningModule): + def __init__( + self, + diffusion_path, + num_classes, + ckpt_path=None, + pool='attention', + label_key=None, + diffusion_ckpt_path=None, + scheduler_config=None, + weight_decay=1.0e-2, + log_steps=10, + monitor='val/loss', + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.num_classes = num_classes + # get latest config of diffusion model + diffusion_config = natsorted( + glob(os.path.join(diffusion_path, 'configs', '*-project.yaml')) + )[-1] + self.diffusion_config = OmegaConf.load(diffusion_config).model + self.diffusion_config.params.ckpt_path = diffusion_ckpt_path + self.load_diffusion() + + self.monitor = monitor + self.numd = ( + self.diffusion_model.first_stage_model.encoder.num_resolutions - 1 + ) + self.log_time_interval = ( + self.diffusion_model.num_timesteps // log_steps + ) + self.log_steps = log_steps + + self.label_key = ( + label_key + if not hasattr(self.diffusion_model, 'cond_stage_key') + else self.diffusion_model.cond_stage_key + ) + + assert ( + self.label_key is not None + ), 'label_key neither in diffusion model nor in model.params' + + if self.label_key not in __models__: + raise NotImplementedError() + + self.load_classifier(ckpt_path, pool) + + self.scheduler_config = scheduler_config + self.use_scheduler = self.scheduler_config is not None + self.weight_decay = weight_decay + + def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): + sd = torch.load(path, map_location='cpu') + if 'state_dict' in list(sd.keys()): + sd = sd['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + missing, unexpected = ( + self.load_state_dict(sd, strict=False) + if not only_model + else self.model.load_state_dict(sd, strict=False) + ) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + if len(unexpected) > 0: + print(f'Unexpected Keys: {unexpected}') + + def load_diffusion(self): + model = instantiate_from_config(self.diffusion_config) + self.diffusion_model = model.eval() + self.diffusion_model.train = disabled_train + for param in self.diffusion_model.parameters(): + param.requires_grad = False + + def load_classifier(self, ckpt_path, pool): + model_config = deepcopy( + self.diffusion_config.params.unet_config.params + ) + model_config.in_channels = ( + self.diffusion_config.params.unet_config.params.out_channels + ) + model_config.out_channels = self.num_classes + if self.label_key == 'class_label': + model_config.pool = pool + + self.model = __models__[self.label_key](**model_config) + if ckpt_path is not None: + print( + '#####################################################################' + ) + print(f'load from ckpt "{ckpt_path}"') + print( + '#####################################################################' + ) + self.init_from_ckpt(ckpt_path) + + @torch.no_grad() + def get_x_noisy(self, x, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x)) + continuous_sqrt_alpha_cumprod = None + if self.diffusion_model.use_continuous_noise: + continuous_sqrt_alpha_cumprod = ( + self.diffusion_model.sample_continuous_noise_level( + x.shape[0], t + 1 + ) + ) + # todo: make sure t+1 is correct here + + return self.diffusion_model.q_sample( + x_start=x, + t=t, + noise=noise, + continuous_sqrt_alpha_cumprod=continuous_sqrt_alpha_cumprod, + ) + + def forward(self, x_noisy, t, *args, **kwargs): + return self.model(x_noisy, t) + + @torch.no_grad() + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = rearrange(x, 'b h w c -> b c h w') + x = x.to(memory_format=torch.contiguous_format).float() + return x + + @torch.no_grad() + def get_conditioning(self, batch, k=None): + if k is None: + k = self.label_key + assert k is not None, 'Needs to provide label key' + + targets = batch[k].to(self.device) + + if self.label_key == 'segmentation': + targets = rearrange(targets, 'b h w c -> b c h w') + for down in range(self.numd): + h, w = targets.shape[-2:] + targets = F.interpolate( + targets, size=(h // 2, w // 2), mode='nearest' + ) + + # targets = rearrange(targets,'b c h w -> b h w c') + + return targets + + def compute_top_k(self, logits, labels, k, reduction='mean'): + _, top_ks = torch.topk(logits, k, dim=1) + if reduction == 'mean': + return ( + (top_ks == labels[:, None]).float().sum(dim=-1).mean().item() + ) + elif reduction == 'none': + return (top_ks == labels[:, None]).float().sum(dim=-1) + + def on_train_epoch_start(self): + # save some memory + self.diffusion_model.model.to('cpu') + + @torch.no_grad() + def write_logs(self, loss, logits, targets): + log_prefix = 'train' if self.training else 'val' + log = {} + log[f'{log_prefix}/loss'] = loss.mean() + log[f'{log_prefix}/acc@1'] = self.compute_top_k( + logits, targets, k=1, reduction='mean' + ) + log[f'{log_prefix}/acc@5'] = self.compute_top_k( + logits, targets, k=5, reduction='mean' + ) + + self.log_dict( + log, + prog_bar=False, + logger=True, + on_step=self.training, + on_epoch=True, + ) + self.log( + 'loss', log[f'{log_prefix}/loss'], prog_bar=True, logger=False + ) + self.log( + 'global_step', + self.global_step, + logger=False, + on_epoch=False, + prog_bar=True, + ) + lr = self.optimizers().param_groups[0]['lr'] + self.log( + 'lr_abs', + lr, + on_step=True, + logger=True, + on_epoch=False, + prog_bar=True, + ) + + def shared_step(self, batch, t=None): + x, *_ = self.diffusion_model.get_input( + batch, k=self.diffusion_model.first_stage_key + ) + targets = self.get_conditioning(batch) + if targets.dim() == 4: + targets = targets.argmax(dim=1) + if t is None: + t = torch.randint( + 0, + self.diffusion_model.num_timesteps, + (x.shape[0],), + device=self.device, + ).long() + else: + t = torch.full( + size=(x.shape[0],), fill_value=t, device=self.device + ).long() + x_noisy = self.get_x_noisy(x, t) + logits = self(x_noisy, t) + + loss = F.cross_entropy(logits, targets, reduction='none') + + self.write_logs(loss.detach(), logits.detach(), targets.detach()) + + loss = loss.mean() + return loss, logits, x_noisy, targets + + def training_step(self, batch, batch_idx): + loss, *_ = self.shared_step(batch) + return loss + + def reset_noise_accs(self): + self.noisy_acc = { + t: {'acc@1': [], 'acc@5': []} + for t in range( + 0, + self.diffusion_model.num_timesteps, + self.diffusion_model.log_every_t, + ) + } + + def on_validation_start(self): + self.reset_noise_accs() + + @torch.no_grad() + def validation_step(self, batch, batch_idx): + loss, *_ = self.shared_step(batch) + + for t in self.noisy_acc: + _, logits, _, targets = self.shared_step(batch, t) + self.noisy_acc[t]['acc@1'].append( + self.compute_top_k(logits, targets, k=1, reduction='mean') + ) + self.noisy_acc[t]['acc@5'].append( + self.compute_top_k(logits, targets, k=5, reduction='mean') + ) + + return loss + + def configure_optimizers(self): + optimizer = AdamW( + self.model.parameters(), + lr=self.learning_rate, + weight_decay=self.weight_decay, + ) + + if self.use_scheduler: + scheduler = instantiate_from_config(self.scheduler_config) + + print('Setting up LambdaLR scheduler...') + scheduler = [ + { + 'scheduler': LambdaLR( + optimizer, lr_lambda=scheduler.schedule + ), + 'interval': 'step', + 'frequency': 1, + } + ] + return [optimizer], scheduler + + return optimizer + + @torch.no_grad() + def log_images(self, batch, N=8, *args, **kwargs): + log = dict() + x = self.get_input(batch, self.diffusion_model.first_stage_key) + log['inputs'] = x + + y = self.get_conditioning(batch) + + if self.label_key == 'class_label': + y = log_txt_as_img((x.shape[2], x.shape[3]), batch['human_label']) + log['labels'] = y + + if ismap(y): + log['labels'] = self.diffusion_model.to_rgb(y) + + for step in range(self.log_steps): + current_time = step * self.log_time_interval + + _, logits, x_noisy, _ = self.shared_step(batch, t=current_time) + + log[f'inputs@t{current_time}'] = x_noisy + + pred = F.one_hot( + logits.argmax(dim=1), num_classes=self.num_classes + ) + pred = rearrange(pred, 'b h w c -> b c h w') + + log[f'pred@t{current_time}'] = self.diffusion_model.to_rgb( + pred + ) + + for key in log: + log[key] = log[key][:N] + + return log diff --git a/invokeai/models/diffusion/cross_attention_control.py b/invokeai/models/diffusion/cross_attention_control.py new file mode 100644 index 0000000000..a34f22e683 --- /dev/null +++ b/invokeai/models/diffusion/cross_attention_control.py @@ -0,0 +1,642 @@ + +# adapted from bloc97's CrossAttentionControl colab +# https://github.com/bloc97/CrossAttentionControl + + +import enum +import math +from typing import Optional, Callable + +import psutil +import torch +import diffusers +from torch import nn + +from compel.cross_attention_control import Arguments +from diffusers.models.unet_2d_condition import UNet2DConditionModel +from diffusers.models.cross_attention import AttnProcessor +from ldm.invoke.devices import torch_dtype + + +class CrossAttentionType(enum.Enum): + SELF = 1 + TOKENS = 2 + + +class Context: + + cross_attention_mask: Optional[torch.Tensor] + cross_attention_index_map: Optional[torch.Tensor] + + class Action(enum.Enum): + NONE = 0 + SAVE = 1, + APPLY = 2 + + def __init__(self, arguments: Arguments, step_count: int): + """ + :param arguments: Arguments for the cross-attention control process + :param step_count: The absolute total number of steps of diffusion (for img2img this is likely larger than the number of steps that will actually run) + """ + self.cross_attention_mask = None + self.cross_attention_index_map = None + self.self_cross_attention_action = Context.Action.NONE + self.tokens_cross_attention_action = Context.Action.NONE + self.arguments = arguments + self.step_count = step_count + + self.self_cross_attention_module_identifiers = [] + self.tokens_cross_attention_module_identifiers = [] + + self.saved_cross_attention_maps = {} + + self.clear_requests(cleanup=True) + + def register_cross_attention_modules(self, model): + for name,module in get_cross_attention_modules(model, CrossAttentionType.SELF): + if name in self.self_cross_attention_module_identifiers: + assert False, f"name {name} cannot appear more than once" + self.self_cross_attention_module_identifiers.append(name) + for name,module in get_cross_attention_modules(model, CrossAttentionType.TOKENS): + if name in self.tokens_cross_attention_module_identifiers: + assert False, f"name {name} cannot appear more than once" + self.tokens_cross_attention_module_identifiers.append(name) + + def request_save_attention_maps(self, cross_attention_type: CrossAttentionType): + if cross_attention_type == CrossAttentionType.SELF: + self.self_cross_attention_action = Context.Action.SAVE + else: + self.tokens_cross_attention_action = Context.Action.SAVE + + def request_apply_saved_attention_maps(self, cross_attention_type: CrossAttentionType): + if cross_attention_type == CrossAttentionType.SELF: + self.self_cross_attention_action = Context.Action.APPLY + else: + self.tokens_cross_attention_action = Context.Action.APPLY + + def is_tokens_cross_attention(self, module_identifier) -> bool: + return module_identifier in self.tokens_cross_attention_module_identifiers + + def get_should_save_maps(self, module_identifier: str) -> bool: + if module_identifier in self.self_cross_attention_module_identifiers: + return self.self_cross_attention_action == Context.Action.SAVE + elif module_identifier in self.tokens_cross_attention_module_identifiers: + return self.tokens_cross_attention_action == Context.Action.SAVE + return False + + def get_should_apply_saved_maps(self, module_identifier: str) -> bool: + if module_identifier in self.self_cross_attention_module_identifiers: + return self.self_cross_attention_action == Context.Action.APPLY + elif module_identifier in self.tokens_cross_attention_module_identifiers: + return self.tokens_cross_attention_action == Context.Action.APPLY + return False + + def get_active_cross_attention_control_types_for_step(self, percent_through:float=None)\ + -> list[CrossAttentionType]: + """ + Should cross-attention control be applied on the given step? + :param percent_through: How far through the step sequence are we (0.0=pure noise, 1.0=completely denoised image). Expected range 0.0..<1.0. + :return: A list of attention types that cross-attention control should be performed for on the given step. May be []. + """ + if percent_through is None: + return [CrossAttentionType.SELF, CrossAttentionType.TOKENS] + + opts = self.arguments.edit_options + to_control = [] + if opts['s_start'] <= percent_through < opts['s_end']: + to_control.append(CrossAttentionType.SELF) + if opts['t_start'] <= percent_through < opts['t_end']: + to_control.append(CrossAttentionType.TOKENS) + return to_control + + def save_slice(self, identifier: str, slice: torch.Tensor, dim: Optional[int], offset: int, + slice_size: Optional[int]): + if identifier not in self.saved_cross_attention_maps: + self.saved_cross_attention_maps[identifier] = { + 'dim': dim, + 'slice_size': slice_size, + 'slices': {offset or 0: slice} + } + else: + self.saved_cross_attention_maps[identifier]['slices'][offset or 0] = slice + + def get_slice(self, identifier: str, requested_dim: Optional[int], requested_offset: int, slice_size: int): + saved_attention_dict = self.saved_cross_attention_maps[identifier] + if requested_dim is None: + if saved_attention_dict['dim'] is not None: + raise RuntimeError(f"dim mismatch: expected dim=None, have {saved_attention_dict['dim']}") + return saved_attention_dict['slices'][0] + + if saved_attention_dict['dim'] == requested_dim: + if slice_size != saved_attention_dict['slice_size']: + raise RuntimeError( + f"slice_size mismatch: expected slice_size={slice_size}, have {saved_attention_dict['slice_size']}") + return saved_attention_dict['slices'][requested_offset] + + if saved_attention_dict['dim'] is None: + whole_saved_attention = saved_attention_dict['slices'][0] + if requested_dim == 0: + return whole_saved_attention[requested_offset:requested_offset + slice_size] + elif requested_dim == 1: + return whole_saved_attention[:, requested_offset:requested_offset + slice_size] + + raise RuntimeError(f"Cannot convert dim {saved_attention_dict['dim']} to requested dim {requested_dim}") + + def get_slicing_strategy(self, identifier: str) -> tuple[Optional[int], Optional[int]]: + saved_attention = self.saved_cross_attention_maps.get(identifier, None) + if saved_attention is None: + return None, None + return saved_attention['dim'], saved_attention['slice_size'] + + def clear_requests(self, cleanup=True): + self.tokens_cross_attention_action = Context.Action.NONE + self.self_cross_attention_action = Context.Action.NONE + if cleanup: + self.saved_cross_attention_maps = {} + + def offload_saved_attention_slices_to_cpu(self): + for key, map_dict in self.saved_cross_attention_maps.items(): + for offset, slice in map_dict['slices'].items(): + map_dict[offset] = slice.to('cpu') + + + +class InvokeAICrossAttentionMixin: + """ + Enable InvokeAI-flavoured CrossAttention calculation, which does aggressive low-memory slicing and calls + through both to an attention_slice_wrangler and a slicing_strategy_getter for custom attention map wrangling + and dymamic slicing strategy selection. + """ + def __init__(self): + self.mem_total_gb = psutil.virtual_memory().total // (1 << 30) + self.attention_slice_wrangler = None + self.slicing_strategy_getter = None + self.attention_slice_calculated_callback = None + + def set_attention_slice_wrangler(self, wrangler: Optional[Callable[[nn.Module, torch.Tensor, int, int, int], torch.Tensor]]): + ''' + Set custom attention calculator to be called when attention is calculated + :param wrangler: Callback, with args (module, suggested_attention_slice, dim, offset, slice_size), + which returns either the suggested_attention_slice or an adjusted equivalent. + `module` is the current CrossAttention module for which the callback is being invoked. + `suggested_attention_slice` is the default-calculated attention slice + `dim` is -1 if the attenion map has not been sliced, or 0 or 1 for dimension-0 or dimension-1 slicing. + If `dim` is >= 0, `offset` and `slice_size` specify the slice start and length. + + Pass None to use the default attention calculation. + :return: + ''' + self.attention_slice_wrangler = wrangler + + def set_slicing_strategy_getter(self, getter: Optional[Callable[[nn.Module], tuple[int,int]]]): + self.slicing_strategy_getter = getter + + def set_attention_slice_calculated_callback(self, callback: Optional[Callable[[torch.Tensor], None]]): + self.attention_slice_calculated_callback = callback + + def einsum_lowest_level(self, query, key, value, dim, offset, slice_size): + # calculate attention scores + #attention_scores = torch.einsum('b i d, b j d -> b i j', q, k) + attention_scores = torch.baddbmm( + torch.empty(query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device), + query, + key.transpose(-1, -2), + beta=0, + alpha=self.scale, + ) + + # calculate attention slice by taking the best scores for each latent pixel + default_attention_slice = attention_scores.softmax(dim=-1, dtype=attention_scores.dtype) + attention_slice_wrangler = self.attention_slice_wrangler + if attention_slice_wrangler is not None: + attention_slice = attention_slice_wrangler(self, default_attention_slice, dim, offset, slice_size) + else: + attention_slice = default_attention_slice + + if self.attention_slice_calculated_callback is not None: + self.attention_slice_calculated_callback(attention_slice, dim, offset, slice_size) + + hidden_states = torch.bmm(attention_slice, value) + return hidden_states + + def einsum_op_slice_dim0(self, q, k, v, slice_size): + r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype) + for i in range(0, q.shape[0], slice_size): + end = i + slice_size + r[i:end] = self.einsum_lowest_level(q[i:end], k[i:end], v[i:end], dim=0, offset=i, slice_size=slice_size) + return r + + def einsum_op_slice_dim1(self, q, k, v, slice_size): + r = torch.zeros(q.shape[0], q.shape[1], v.shape[2], device=q.device, dtype=q.dtype) + for i in range(0, q.shape[1], slice_size): + end = i + slice_size + r[:, i:end] = self.einsum_lowest_level(q[:, i:end], k, v, dim=1, offset=i, slice_size=slice_size) + return r + + def einsum_op_mps_v1(self, q, k, v): + if q.shape[1] <= 4096: # (512x512) max q.shape[1]: 4096 + return self.einsum_lowest_level(q, k, v, None, None, None) + else: + slice_size = math.floor(2**30 / (q.shape[0] * q.shape[1])) + return self.einsum_op_slice_dim1(q, k, v, slice_size) + + def einsum_op_mps_v2(self, q, k, v): + if self.mem_total_gb > 8 and q.shape[1] <= 4096: + return self.einsum_lowest_level(q, k, v, None, None, None) + else: + return self.einsum_op_slice_dim0(q, k, v, 1) + + def einsum_op_tensor_mem(self, q, k, v, max_tensor_mb): + size_mb = q.shape[0] * q.shape[1] * k.shape[1] * q.element_size() // (1 << 20) + if size_mb <= max_tensor_mb: + return self.einsum_lowest_level(q, k, v, None, None, None) + div = 1 << int((size_mb - 1) / max_tensor_mb).bit_length() + if div <= q.shape[0]: + return self.einsum_op_slice_dim0(q, k, v, q.shape[0] // div) + return self.einsum_op_slice_dim1(q, k, v, max(q.shape[1] // div, 1)) + + def einsum_op_cuda(self, q, k, v): + # check if we already have a slicing strategy (this should only happen during cross-attention controlled generation) + slicing_strategy_getter = self.slicing_strategy_getter + if slicing_strategy_getter is not None: + (dim, slice_size) = slicing_strategy_getter(self) + if dim is not None: + # print("using saved slicing strategy with dim", dim, "slice size", slice_size) + if dim == 0: + return self.einsum_op_slice_dim0(q, k, v, slice_size) + elif dim == 1: + return self.einsum_op_slice_dim1(q, k, v, slice_size) + + # fallback for when there is no saved strategy, or saved strategy does not slice + mem_free_total = get_mem_free_total(q.device) + # Divide factor of safety as there's copying and fragmentation + return self.einsum_op_tensor_mem(q, k, v, mem_free_total / 3.3 / (1 << 20)) + + + def get_invokeai_attention_mem_efficient(self, q, k, v): + if q.device.type == 'cuda': + #print("in get_attention_mem_efficient with q shape", q.shape, ", k shape", k.shape, ", free memory is", get_mem_free_total(q.device)) + return self.einsum_op_cuda(q, k, v) + + if q.device.type == 'mps' or q.device.type == 'cpu': + if self.mem_total_gb >= 32: + return self.einsum_op_mps_v1(q, k, v) + return self.einsum_op_mps_v2(q, k, v) + + # Smaller slices are faster due to L2/L3/SLC caches. + # Tested on i7 with 8MB L3 cache. + return self.einsum_op_tensor_mem(q, k, v, 32) + + + +def restore_default_cross_attention(model, is_running_diffusers: bool, restore_attention_processor: Optional[AttnProcessor]=None): + if is_running_diffusers: + unet = model + unet.set_attn_processor(restore_attention_processor or CrossAttnProcessor()) + else: + remove_attention_function(model) + + +def override_cross_attention(model, context: Context, is_running_diffusers = False): + """ + Inject attention parameters and functions into the passed in model to enable cross attention editing. + + :param model: The unet model to inject into. + :return: None + """ + + # adapted from init_attention_edit + device = context.arguments.edited_conditioning.device + + # urgh. should this be hardcoded? + max_length = 77 + # mask=1 means use base prompt attention, mask=0 means use edited prompt attention + mask = torch.zeros(max_length, dtype=torch_dtype(device)) + indices_target = torch.arange(max_length, dtype=torch.long) + indices = torch.arange(max_length, dtype=torch.long) + for name, a0, a1, b0, b1 in context.arguments.edit_opcodes: + if b0 < max_length: + if name == "equal":# or (name == "replace" and a1 - a0 == b1 - b0): + # these tokens have not been edited + indices[b0:b1] = indices_target[a0:a1] + mask[b0:b1] = 1 + + context.cross_attention_mask = mask.to(device) + context.cross_attention_index_map = indices.to(device) + if is_running_diffusers: + unet = model + old_attn_processors = unet.attn_processors + if torch.backends.mps.is_available(): + # see note in StableDiffusionGeneratorPipeline.__init__ about borked slicing on MPS + unet.set_attn_processor(SwapCrossAttnProcessor()) + else: + # try to re-use an existing slice size + default_slice_size = 4 + slice_size = next((p.slice_size for p in old_attn_processors.values() if type(p) is SlicedAttnProcessor), default_slice_size) + unet.set_attn_processor(SlicedSwapCrossAttnProcesser(slice_size=slice_size)) + return old_attn_processors + else: + context.register_cross_attention_modules(model) + inject_attention_function(model, context) + return None + + + + +def get_cross_attention_modules(model, which: CrossAttentionType) -> list[tuple[str, InvokeAICrossAttentionMixin]]: + from ldm.modules.attention import CrossAttention # avoid circular import + cross_attention_class: type = InvokeAIDiffusersCrossAttention if isinstance(model,UNet2DConditionModel) else CrossAttention + which_attn = "attn1" if which is CrossAttentionType.SELF else "attn2" + attention_module_tuples = [(name,module) for name, module in model.named_modules() if + isinstance(module, cross_attention_class) and which_attn in name] + cross_attention_modules_in_model_count = len(attention_module_tuples) + expected_count = 16 + if cross_attention_modules_in_model_count != expected_count: + # non-fatal error but .swap() won't work. + print(f"Error! CrossAttentionControl found an unexpected number of {cross_attention_class} modules in the model " + + f"(expected {expected_count}, found {cross_attention_modules_in_model_count}). Either monkey-patching failed " + + f"or some assumption has changed about the structure of the model itself. Please fix the monkey-patching, " + + f"and/or update the {expected_count} above to an appropriate number, and/or find and inform someone who knows " + + f"what it means. This error is non-fatal, but it is likely that .swap() and attention map display will not " + + f"work properly until it is fixed.") + return attention_module_tuples + + +def inject_attention_function(unet, context: Context): + # ORIGINAL SOURCE CODE: https://github.com/huggingface/diffusers/blob/91ddd2a25b848df0fa1262d4f1cd98c7ccb87750/src/diffusers/models/attention.py#L276 + + def attention_slice_wrangler(module, suggested_attention_slice:torch.Tensor, dim, offset, slice_size): + + #memory_usage = suggested_attention_slice.element_size() * suggested_attention_slice.nelement() + + attention_slice = suggested_attention_slice + + if context.get_should_save_maps(module.identifier): + #print(module.identifier, "saving suggested_attention_slice of shape", + # suggested_attention_slice.shape, "dim", dim, "offset", offset) + slice_to_save = attention_slice.to('cpu') if dim is not None else attention_slice + context.save_slice(module.identifier, slice_to_save, dim=dim, offset=offset, slice_size=slice_size) + elif context.get_should_apply_saved_maps(module.identifier): + #print(module.identifier, "applying saved attention slice for dim", dim, "offset", offset) + saved_attention_slice = context.get_slice(module.identifier, dim, offset, slice_size) + + # slice may have been offloaded to CPU + saved_attention_slice = saved_attention_slice.to(suggested_attention_slice.device) + + if context.is_tokens_cross_attention(module.identifier): + index_map = context.cross_attention_index_map + remapped_saved_attention_slice = torch.index_select(saved_attention_slice, -1, index_map) + this_attention_slice = suggested_attention_slice + + mask = context.cross_attention_mask.to(torch_dtype(suggested_attention_slice.device)) + saved_mask = mask + this_mask = 1 - mask + attention_slice = remapped_saved_attention_slice * saved_mask + \ + this_attention_slice * this_mask + else: + # just use everything + attention_slice = saved_attention_slice + + return attention_slice + + cross_attention_modules = get_cross_attention_modules(unet, CrossAttentionType.TOKENS) + get_cross_attention_modules(unet, CrossAttentionType.SELF) + for identifier, module in cross_attention_modules: + module.identifier = identifier + try: + module.set_attention_slice_wrangler(attention_slice_wrangler) + module.set_slicing_strategy_getter( + lambda module: context.get_slicing_strategy(identifier) + ) + except AttributeError as e: + if is_attribute_error_about(e, 'set_attention_slice_wrangler'): + print(f"TODO: implement set_attention_slice_wrangler for {type(module)}") # TODO + else: + raise + + +def remove_attention_function(unet): + cross_attention_modules = get_cross_attention_modules(unet, CrossAttentionType.TOKENS) + get_cross_attention_modules(unet, CrossAttentionType.SELF) + for identifier, module in cross_attention_modules: + try: + # clear wrangler callback + module.set_attention_slice_wrangler(None) + module.set_slicing_strategy_getter(None) + except AttributeError as e: + if is_attribute_error_about(e, 'set_attention_slice_wrangler'): + print(f"TODO: implement set_attention_slice_wrangler for {type(module)}") + else: + raise + + +def is_attribute_error_about(error: AttributeError, attribute: str): + if hasattr(error, 'name'): # Python 3.10 + return error.name == attribute + else: # Python 3.9 + return attribute in str(error) + + + +def get_mem_free_total(device): + #only on cuda + if not torch.cuda.is_available(): + return None + stats = torch.cuda.memory_stats(device) + mem_active = stats['active_bytes.all.current'] + mem_reserved = stats['reserved_bytes.all.current'] + mem_free_cuda, _ = torch.cuda.mem_get_info(device) + mem_free_torch = mem_reserved - mem_active + mem_free_total = mem_free_cuda + mem_free_torch + return mem_free_total + + + +class InvokeAIDiffusersCrossAttention(diffusers.models.attention.CrossAttention, InvokeAICrossAttentionMixin): + + def __init__(self, **kwargs): + super().__init__(**kwargs) + InvokeAICrossAttentionMixin.__init__(self) + + def _attention(self, query, key, value, attention_mask=None): + #default_result = super()._attention(query, key, value) + if attention_mask is not None: + print(f"{type(self).__name__} ignoring passed-in attention_mask") + attention_result = self.get_invokeai_attention_mem_efficient(query, key, value) + + hidden_states = self.reshape_batch_dim_to_heads(attention_result) + return hidden_states + + + + + +## 🧨diffusers implementation follows + + +""" +# base implementation + +class CrossAttnProcessor: + def __call__(self, attn: CrossAttention, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length) + + query = attn.to_q(hidden_states) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + +""" +from dataclasses import field, dataclass + +import torch + +from diffusers.models.cross_attention import CrossAttention, CrossAttnProcessor, SlicedAttnProcessor + + +@dataclass +class SwapCrossAttnContext: + modified_text_embeddings: torch.Tensor + index_map: torch.Tensor # maps from original prompt token indices to the equivalent tokens in the modified prompt + mask: torch.Tensor # in the target space of the index_map + cross_attention_types_to_do: list[CrossAttentionType] = field(default_factory=list) + + def __int__(self, + cac_types_to_do: [CrossAttentionType], + modified_text_embeddings: torch.Tensor, + index_map: torch.Tensor, + mask: torch.Tensor): + self.cross_attention_types_to_do = cac_types_to_do + self.modified_text_embeddings = modified_text_embeddings + self.index_map = index_map + self.mask = mask + + def wants_cross_attention_control(self, attn_type: CrossAttentionType) -> bool: + return attn_type in self.cross_attention_types_to_do + + @classmethod + def make_mask_and_index_map(cls, edit_opcodes: list[tuple[str, int, int, int, int]], max_length: int) \ + -> tuple[torch.Tensor, torch.Tensor]: + + # mask=1 means use original prompt attention, mask=0 means use modified prompt attention + mask = torch.zeros(max_length) + indices_target = torch.arange(max_length, dtype=torch.long) + indices = torch.arange(max_length, dtype=torch.long) + for name, a0, a1, b0, b1 in edit_opcodes: + if b0 < max_length: + if name == "equal": + # these tokens remain the same as in the original prompt + indices[b0:b1] = indices_target[a0:a1] + mask[b0:b1] = 1 + + return mask, indices + + +class SlicedSwapCrossAttnProcesser(SlicedAttnProcessor): + + # TODO: dynamically pick slice size based on memory conditions + + def __call__(self, attn: CrossAttention, hidden_states, encoder_hidden_states=None, attention_mask=None, + # kwargs + swap_cross_attn_context: SwapCrossAttnContext=None): + + attention_type = CrossAttentionType.SELF if encoder_hidden_states is None else CrossAttentionType.TOKENS + + # if cross-attention control is not in play, just call through to the base implementation. + if attention_type is CrossAttentionType.SELF or \ + swap_cross_attn_context is None or \ + not swap_cross_attn_context.wants_cross_attention_control(attention_type): + #print(f"SwapCrossAttnContext for {attention_type} not active - passing request to superclass") + return super().__call__(attn, hidden_states, encoder_hidden_states, attention_mask) + #else: + # print(f"SwapCrossAttnContext for {attention_type} active") + + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask( + attention_mask=attention_mask, target_length=sequence_length, + batch_size=batch_size) + + query = attn.to_q(hidden_states) + dim = query.shape[-1] + query = attn.head_to_batch_dim(query) + + original_text_embeddings = encoder_hidden_states + modified_text_embeddings = swap_cross_attn_context.modified_text_embeddings + original_text_key = attn.to_k(original_text_embeddings) + modified_text_key = attn.to_k(modified_text_embeddings) + original_value = attn.to_v(original_text_embeddings) + modified_value = attn.to_v(modified_text_embeddings) + + original_text_key = attn.head_to_batch_dim(original_text_key) + modified_text_key = attn.head_to_batch_dim(modified_text_key) + original_value = attn.head_to_batch_dim(original_value) + modified_value = attn.head_to_batch_dim(modified_value) + + # compute slices and prepare output tensor + batch_size_attention = query.shape[0] + hidden_states = torch.zeros( + (batch_size_attention, sequence_length, dim // attn.heads), device=query.device, dtype=query.dtype + ) + + # do slices + for i in range(max(1,hidden_states.shape[0] // self.slice_size)): + start_idx = i * self.slice_size + end_idx = (i + 1) * self.slice_size + + query_slice = query[start_idx:end_idx] + original_key_slice = original_text_key[start_idx:end_idx] + modified_key_slice = modified_text_key[start_idx:end_idx] + attn_mask_slice = attention_mask[start_idx:end_idx] if attention_mask is not None else None + + original_attn_slice = attn.get_attention_scores(query_slice, original_key_slice, attn_mask_slice) + modified_attn_slice = attn.get_attention_scores(query_slice, modified_key_slice, attn_mask_slice) + + # because the prompt modifications may result in token sequences shifted forwards or backwards, + # the original attention probabilities must be remapped to account for token index changes in the + # modified prompt + remapped_original_attn_slice = torch.index_select(original_attn_slice, -1, + swap_cross_attn_context.index_map) + + # only some tokens taken from the original attention probabilities. this is controlled by the mask. + mask = swap_cross_attn_context.mask + inverse_mask = 1 - mask + attn_slice = \ + remapped_original_attn_slice * mask + \ + modified_attn_slice * inverse_mask + + del remapped_original_attn_slice, modified_attn_slice + + attn_slice = torch.bmm(attn_slice, modified_value[start_idx:end_idx]) + hidden_states[start_idx:end_idx] = attn_slice + + + # done + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class SwapCrossAttnProcessor(SlicedSwapCrossAttnProcesser): + + def __init__(self): + super(SwapCrossAttnProcessor, self).__init__(slice_size=int(1e9)) # massive slice size = don't slice + diff --git a/invokeai/models/diffusion/cross_attention_map_saving.py b/invokeai/models/diffusion/cross_attention_map_saving.py new file mode 100644 index 0000000000..eede431d33 --- /dev/null +++ b/invokeai/models/diffusion/cross_attention_map_saving.py @@ -0,0 +1,95 @@ +import math + +import PIL +import torch +from torchvision.transforms.functional import resize as tv_resize, InterpolationMode + +from .cross_attention_control import get_cross_attention_modules, CrossAttentionType + + +class AttentionMapSaver(): + + def __init__(self, token_ids: range, latents_shape: torch.Size): + self.token_ids = token_ids + self.latents_shape = latents_shape + #self.collated_maps = #torch.zeros([len(token_ids), latents_shape[0], latents_shape[1]]) + self.collated_maps = {} + + def clear_maps(self): + self.collated_maps = {} + + def add_attention_maps(self, maps: torch.Tensor, key: str): + """ + Accumulate the given attention maps and store by summing with existing maps at the passed-in key (if any). + :param maps: Attention maps to store. Expected shape [A, (H*W), N] where A is attention heads count, H and W are the map size (fixed per-key) and N is the number of tokens (typically 77). + :param key: Storage key. If a map already exists for this key it will be summed with the incoming data. In this case the maps sizes (H and W) should match. + :return: None + """ + key_and_size = f'{key}_{maps.shape[1]}' + + # extract desired tokens + maps = maps[:, :, self.token_ids] + + # merge attention heads to a single map per token + maps = torch.sum(maps, 0) + + # store + if key_and_size not in self.collated_maps: + self.collated_maps[key_and_size] = torch.zeros_like(maps, device='cpu') + self.collated_maps[key_and_size] += maps.cpu() + + def write_maps_to_disk(self, path: str): + pil_image = self.get_stacked_maps_image() + pil_image.save(path, 'PNG') + + def get_stacked_maps_image(self) -> PIL.Image: + """ + Scale all collected attention maps to the same size, blend them together and return as an image. + :return: An image containing a vertical stack of blended attention maps, one for each requested token. + """ + num_tokens = len(self.token_ids) + if num_tokens == 0: + return None + + latents_height = self.latents_shape[0] + latents_width = self.latents_shape[1] + + merged = None + + for key, maps in self.collated_maps.items(): + + # maps has shape [(H*W), N] for N tokens + # but we want [N, H, W] + this_scale_factor = math.sqrt(maps.shape[0] / (latents_width * latents_height)) + this_maps_height = int(float(latents_height) * this_scale_factor) + this_maps_width = int(float(latents_width) * this_scale_factor) + # and we need to do some dimension juggling + maps = torch.reshape(torch.swapdims(maps, 0, 1), [num_tokens, this_maps_height, this_maps_width]) + + # scale to output size if necessary + if this_scale_factor != 1: + maps = tv_resize(maps, [latents_height, latents_width], InterpolationMode.BICUBIC) + + # normalize + maps_min = torch.min(maps) + maps_range = torch.max(maps) - maps_min + #print(f"map {key} size {[this_maps_width, this_maps_height]} range {[maps_min, maps_min + maps_range]}") + maps_normalized = (maps - maps_min) / maps_range + # expand to (-0.1, 1.1) and clamp + maps_normalized_expanded = maps_normalized * 1.1 - 0.05 + maps_normalized_expanded_clamped = torch.clamp(maps_normalized_expanded, 0, 1) + + # merge together, producing a vertical stack + maps_stacked = torch.reshape(maps_normalized_expanded_clamped, [num_tokens * latents_height, latents_width]) + + if merged is None: + merged = maps_stacked + else: + # screen blend + merged = 1 - (1 - maps_stacked)*(1 - merged) + + if merged is None: + return None + + merged_bytes = merged.mul(0xff).byte() + return PIL.Image.fromarray(merged_bytes.numpy(), mode='L') diff --git a/invokeai/models/diffusion/ddim.py b/invokeai/models/diffusion/ddim.py new file mode 100644 index 0000000000..f2c6f4c591 --- /dev/null +++ b/invokeai/models/diffusion/ddim.py @@ -0,0 +1,111 @@ +"""SAMPLING ONLY.""" + +import torch +from .shared_invokeai_diffusion import InvokeAIDiffuserComponent +from .sampler import Sampler +from ldm.modules.diffusionmodules.util import noise_like + +class DDIMSampler(Sampler): + def __init__(self, model, schedule='linear', device=None, **kwargs): + super().__init__(model,schedule,model.num_timesteps,device) + + self.invokeai_diffuser = InvokeAIDiffuserComponent(self.model, + model_forward_callback = lambda x, sigma, cond: self.model.apply_model(x, sigma, cond)) + + def prepare_to_sample(self, t_enc, **kwargs): + super().prepare_to_sample(t_enc, **kwargs) + + extra_conditioning_info = kwargs.get('extra_conditioning_info', None) + all_timesteps_count = kwargs.get('all_timesteps_count', t_enc) + + if extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control: + self.invokeai_diffuser.override_cross_attention(extra_conditioning_info, step_count = all_timesteps_count) + else: + self.invokeai_diffuser.restore_default_cross_attention() + + + # This is the central routine + @torch.no_grad() + def p_sample( + self, + x, + c, + t, + index, + repeat_noise=False, + use_original_steps=False, + quantize_denoised=False, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + step_count:int=1000, # total number of steps + **kwargs, + ): + b, *_, device = *x.shape, x.device + + if ( + unconditional_conditioning is None + or unconditional_guidance_scale == 1.0 + ): + # damian0815 would like to know when/if this code path is used + e_t = self.model.apply_model(x, t, c) + else: + # step_index counts in the opposite direction to index + step_index = step_count-(index+1) + e_t = self.invokeai_diffuser.do_diffusion_step( + x, t, + unconditional_conditioning, c, + unconditional_guidance_scale, + step_index=step_index + ) + if score_corrector is not None: + assert self.model.parameterization == 'eps' + e_t = score_corrector.modify_score( + self.model, e_t, x, t, c, **corrector_kwargs + ) + + alphas = ( + self.model.alphas_cumprod + if use_original_steps + else self.ddim_alphas + ) + alphas_prev = ( + self.model.alphas_cumprod_prev + if use_original_steps + else self.ddim_alphas_prev + ) + sqrt_one_minus_alphas = ( + self.model.sqrt_one_minus_alphas_cumprod + if use_original_steps + else self.ddim_sqrt_one_minus_alphas + ) + sigmas = ( + self.model.ddim_sigmas_for_original_num_steps + if use_original_steps + else self.ddim_sigmas + ) + # select parameters corresponding to the currently considered timestep + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full((b, 1, 1, 1), alphas_prev[index], device=device) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full( + (b, 1, 1, 1), sqrt_one_minus_alphas[index], device=device + ) + + # current prediction for x_0 + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + # direction pointing to x_t + dir_xt = (1.0 - a_prev - sigma_t**2).sqrt() * e_t + noise = ( + sigma_t * noise_like(x.shape, device, repeat_noise) * temperature + ) + if noise_dropout > 0.0: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0, None + diff --git a/invokeai/models/diffusion/ddpm.py b/invokeai/models/diffusion/ddpm.py new file mode 100644 index 0000000000..1fe059cef4 --- /dev/null +++ b/invokeai/models/diffusion/ddpm.py @@ -0,0 +1,2271 @@ +""" +wild mixture of +https://github.com/lucidrains/denoising-diffusion-pytorch/blob/7706bdfc6f527f58d33f84b7b522e61e6e3164b3/denoising_diffusion_pytorch/denoising_diffusion_pytorch.py +https://github.com/openai/improved-diffusion/blob/e94489283bb876ac1477d5dd7709bbbd2d9902ce/improved_diffusion/gaussian_diffusion.py +https://github.com/CompVis/taming-transformers +-- merci +""" + +import torch + +import torch.nn as nn +import os +import numpy as np +import pytorch_lightning as pl +from torch.optim.lr_scheduler import LambdaLR +from einops import rearrange, repeat +from contextlib import contextmanager +from functools import partial +from tqdm import tqdm +from torchvision.utils import make_grid +from pytorch_lightning.utilities.distributed import rank_zero_only +from omegaconf import ListConfig +import urllib + +from ldm.modules.textual_inversion_manager import TextualInversionManager +from ldm.util import ( + log_txt_as_img, + exists, + default, + ismap, + isimage, + mean_flat, + count_params, + instantiate_from_config, +) +from ldm.modules.ema import LitEma +from ldm.modules.distributions.distributions import ( + normal_kl, + DiagonalGaussianDistribution, +) +from ..autoencoder import ( + VQModelInterface, + IdentityFirstStage, + AutoencoderKL, +) +from ldm.modules.diffusionmodules.util import ( + make_beta_schedule, + extract_into_tensor, + noise_like, +) +from .ddim import DDIMSampler + + +__conditioning_keys__ = { + 'concat': 'c_concat', + 'crossattn': 'c_crossattn', + 'adm': 'y', +} + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +def uniform_on_device(r1, r2, shape, device): + return (r1 - r2) * torch.rand(*shape, device=device) + r2 + + +class DDPM(pl.LightningModule): + # classic DDPM with Gaussian diffusion, in image space + def __init__( + self, + unet_config, + timesteps=1000, + beta_schedule='linear', + loss_type='l2', + ckpt_path=None, + ignore_keys=[], + load_only_unet=False, + monitor='val/loss', + use_ema=True, + first_stage_key='image', + image_size=256, + channels=3, + log_every_t=100, + clip_denoised=True, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3, + given_betas=None, + original_elbo_weight=0.0, + embedding_reg_weight=0.0, + v_posterior=0.0, # weight for choosing posterior variance as sigma = (1-v) * beta_tilde + v * beta + l_simple_weight=1.0, + conditioning_key=None, + parameterization='eps', # all assuming fixed variance schedules + scheduler_config=None, + use_positional_encodings=False, + learn_logvar=False, + logvar_init=0.0, + ): + super().__init__() + assert parameterization in [ + 'eps', + 'x0', + ], 'currently only supporting "eps" and "x0"' + self.parameterization = parameterization + print( + f' | {self.__class__.__name__}: Running in {self.parameterization}-prediction mode' + ) + self.cond_stage_model = None + self.clip_denoised = clip_denoised + self.log_every_t = log_every_t + self.first_stage_key = first_stage_key + self.image_size = image_size # try conv? + self.channels = channels + self.use_positional_encodings = use_positional_encodings + self.model = DiffusionWrapper(unet_config, conditioning_key) + count_params(self.model, verbose=True) + self.use_ema = use_ema + if self.use_ema: + self.model_ema = LitEma(self.model) + print(f' | Keeping EMAs of {len(list(self.model_ema.buffers()))}.') + + self.use_scheduler = scheduler_config is not None + if self.use_scheduler: + self.scheduler_config = scheduler_config + + self.v_posterior = v_posterior + self.original_elbo_weight = original_elbo_weight + self.l_simple_weight = l_simple_weight + self.embedding_reg_weight = embedding_reg_weight + + if monitor is not None: + self.monitor = monitor + if ckpt_path is not None: + self.init_from_ckpt( + ckpt_path, ignore_keys=ignore_keys, only_model=load_only_unet + ) + + self.register_schedule( + given_betas=given_betas, + beta_schedule=beta_schedule, + timesteps=timesteps, + linear_start=linear_start, + linear_end=linear_end, + cosine_s=cosine_s, + ) + + self.loss_type = loss_type + + self.learn_logvar = learn_logvar + self.logvar = torch.full( + fill_value=logvar_init, size=(self.num_timesteps,) + ) + if self.learn_logvar: + self.logvar = nn.Parameter(self.logvar, requires_grad=True) + + def register_schedule( + self, + given_betas=None, + beta_schedule='linear', + timesteps=1000, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3, + ): + if exists(given_betas): + betas = given_betas + else: + betas = make_beta_schedule( + beta_schedule, + timesteps, + linear_start=linear_start, + linear_end=linear_end, + cosine_s=cosine_s, + ) + alphas = 1.0 - betas + alphas_cumprod = np.cumprod(alphas, axis=0) + alphas_cumprod_prev = np.append(1.0, alphas_cumprod[:-1]) + + (timesteps,) = betas.shape + self.num_timesteps = int(timesteps) + self.linear_start = linear_start + self.linear_end = linear_end + assert ( + alphas_cumprod.shape[0] == self.num_timesteps + ), 'alphas have to be defined for each timestep' + + to_torch = partial(torch.tensor, dtype=torch.float32) + + self.register_buffer('betas', to_torch(betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer( + 'alphas_cumprod_prev', to_torch(alphas_cumprod_prev) + ) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer( + 'sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod)) + ) + self.register_buffer( + 'sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1.0 - alphas_cumprod)), + ) + self.register_buffer( + 'log_one_minus_alphas_cumprod', + to_torch(np.log(1.0 - alphas_cumprod)), + ) + self.register_buffer( + 'sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1.0 / alphas_cumprod)), + ) + self.register_buffer( + 'sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1.0 / alphas_cumprod - 1)), + ) + + # calculations for posterior q(x_{t-1} | x_t, x_0) + posterior_variance = (1 - self.v_posterior) * betas * ( + 1.0 - alphas_cumprod_prev + ) / (1.0 - alphas_cumprod) + self.v_posterior * betas + # above: equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) + self.register_buffer( + 'posterior_variance', to_torch(posterior_variance) + ) + # below: log calculation clipped because the posterior variance is 0 at the beginning of the diffusion chain + self.register_buffer( + 'posterior_log_variance_clipped', + to_torch(np.log(np.maximum(posterior_variance, 1e-20))), + ) + self.register_buffer( + 'posterior_mean_coef1', + to_torch( + betas * np.sqrt(alphas_cumprod_prev) / (1.0 - alphas_cumprod) + ), + ) + self.register_buffer( + 'posterior_mean_coef2', + to_torch( + (1.0 - alphas_cumprod_prev) + * np.sqrt(alphas) + / (1.0 - alphas_cumprod) + ), + ) + + if self.parameterization == 'eps': + lvlb_weights = self.betas**2 / ( + 2 + * self.posterior_variance + * to_torch(alphas) + * (1 - self.alphas_cumprod) + ) + elif self.parameterization == 'x0': + lvlb_weights = ( + 0.5 + * np.sqrt(torch.Tensor(alphas_cumprod)) + / (2.0 * 1 - torch.Tensor(alphas_cumprod)) + ) + else: + raise NotImplementedError('mu not supported') + # TODO how to choose this term + lvlb_weights[0] = lvlb_weights[1] + self.register_buffer('lvlb_weights', lvlb_weights, persistent=False) + assert not torch.isnan(self.lvlb_weights).all() + + @contextmanager + def ema_scope(self, context=None): + if self.use_ema: + self.model_ema.store(self.model.parameters()) + self.model_ema.copy_to(self.model) + if context is not None: + print(f'{context}: Switched to EMA weights') + try: + yield None + finally: + if self.use_ema: + self.model_ema.restore(self.model.parameters()) + if context is not None: + print(f'{context}: Restored training weights') + + def init_from_ckpt(self, path, ignore_keys=list(), only_model=False): + sd = torch.load(path, map_location='cpu') + if 'state_dict' in list(sd.keys()): + sd = sd['state_dict'] + keys = list(sd.keys()) + for k in keys: + for ik in ignore_keys: + if k.startswith(ik): + print('Deleting key {} from state_dict.'.format(k)) + del sd[k] + missing, unexpected = ( + self.load_state_dict(sd, strict=False) + if not only_model + else self.model.load_state_dict(sd, strict=False) + ) + print( + f'Restored from {path} with {len(missing)} missing and {len(unexpected)} unexpected keys' + ) + if len(missing) > 0: + print(f'Missing Keys: {missing}') + if len(unexpected) > 0: + print(f'Unexpected Keys: {unexpected}') + + def q_mean_variance(self, x_start, t): + """ + Get the distribution q(x_t | x_0). + :param x_start: the [N x C x ...] tensor of noiseless inputs. + :param t: the number of diffusion steps (minus 1). Here, 0 means one step. + :return: A tuple (mean, variance, log_variance), all of x_start's shape. + """ + mean = ( + extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + ) + variance = extract_into_tensor( + 1.0 - self.alphas_cumprod, t, x_start.shape + ) + log_variance = extract_into_tensor( + self.log_one_minus_alphas_cumprod, t, x_start.shape + ) + return mean, variance, log_variance + + def predict_start_from_noise(self, x_t, t, noise): + return ( + extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t + - extract_into_tensor( + self.sqrt_recipm1_alphas_cumprod, t, x_t.shape + ) + * noise + ) + + def q_posterior(self, x_start, x_t, t): + posterior_mean = ( + extract_into_tensor(self.posterior_mean_coef1, t, x_t.shape) + * x_start + + extract_into_tensor(self.posterior_mean_coef2, t, x_t.shape) + * x_t + ) + posterior_variance = extract_into_tensor( + self.posterior_variance, t, x_t.shape + ) + posterior_log_variance_clipped = extract_into_tensor( + self.posterior_log_variance_clipped, t, x_t.shape + ) + return ( + posterior_mean, + posterior_variance, + posterior_log_variance_clipped, + ) + + def p_mean_variance(self, x, t, clip_denoised: bool): + model_out = self.model(x, t) + if self.parameterization == 'eps': + x_recon = self.predict_start_from_noise(x, t=t, noise=model_out) + elif self.parameterization == 'x0': + x_recon = model_out + if clip_denoised: + x_recon.clamp_(-1.0, 1.0) + + ( + model_mean, + posterior_variance, + posterior_log_variance, + ) = self.q_posterior(x_start=x_recon, x_t=x, t=t) + return model_mean, posterior_variance, posterior_log_variance + + @torch.no_grad() + def p_sample(self, x, t, clip_denoised=True, repeat_noise=False): + b, *_, device = *x.shape, x.device + model_mean, _, model_log_variance = self.p_mean_variance( + x=x, t=t, clip_denoised=clip_denoised + ) + noise = noise_like(x.shape, device, repeat_noise) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape( + b, *((1,) * (len(x.shape) - 1)) + ) + return ( + model_mean + + nonzero_mask * (0.5 * model_log_variance).exp() * noise + ) + + @torch.no_grad() + def p_sample_loop(self, shape, return_intermediates=False): + device = self.betas.device + b = shape[0] + img = torch.randn(shape, device=device) + intermediates = [img] + for i in tqdm( + reversed(range(0, self.num_timesteps)), + desc='Sampling t', + total=self.num_timesteps, + dynamic_ncols=True, + ): + img = self.p_sample( + img, + torch.full((b,), i, device=device, dtype=torch.long), + clip_denoised=self.clip_denoised, + ) + if i % self.log_every_t == 0 or i == self.num_timesteps - 1: + intermediates.append(img) + if return_intermediates: + return img, intermediates + return img + + @torch.no_grad() + def sample(self, batch_size=16, return_intermediates=False): + image_size = self.image_size + channels = self.channels + return self.p_sample_loop( + (batch_size, channels, image_size, image_size), + return_intermediates=return_intermediates, + ) + + def q_sample(self, x_start, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + return ( + extract_into_tensor(self.sqrt_alphas_cumprod, t, x_start.shape) + * x_start + + extract_into_tensor( + self.sqrt_one_minus_alphas_cumprod, t, x_start.shape + ) + * noise + ) + + def get_loss(self, pred, target, mean=True): + if self.loss_type == 'l1': + loss = (target - pred).abs() + if mean: + loss = loss.mean() + elif self.loss_type == 'l2': + if mean: + loss = torch.nn.functional.mse_loss(target, pred) + else: + loss = torch.nn.functional.mse_loss( + target, pred, reduction='none' + ) + else: + raise NotImplementedError("unknown loss type '{loss_type}'") + + return loss + + def p_losses(self, x_start, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + model_out = self.model(x_noisy, t) + + loss_dict = {} + if self.parameterization == 'eps': + target = noise + elif self.parameterization == 'x0': + target = x_start + else: + raise NotImplementedError( + f'Paramterization {self.parameterization} not yet supported' + ) + + loss = self.get_loss(model_out, target, mean=False).mean(dim=[1, 2, 3]) + + log_prefix = 'train' if self.training else 'val' + + loss_dict.update({f'{log_prefix}/loss_simple': loss.mean()}) + loss_simple = loss.mean() * self.l_simple_weight + + loss_vlb = (self.lvlb_weights[t] * loss).mean() + loss_dict.update({f'{log_prefix}/loss_vlb': loss_vlb}) + + loss = loss_simple + self.original_elbo_weight * loss_vlb + + loss_dict.update({f'{log_prefix}/loss': loss}) + + return loss, loss_dict + + def forward(self, x, *args, **kwargs): + # b, c, h, w, device, img_size, = *x.shape, x.device, self.image_size + # assert h == img_size and w == img_size, f'height and width of image must be {img_size}' + t = torch.randint( + 0, self.num_timesteps, (x.shape[0],), device=self.device + ).long() + return self.p_losses(x, t, *args, **kwargs) + + def get_input(self, batch, k): + x = batch[k] + if len(x.shape) == 3: + x = x[..., None] + x = rearrange(x, 'b h w c -> b c h w') + x = x.to(memory_format=torch.contiguous_format).float() + return x + + def shared_step(self, batch): + x = self.get_input(batch, self.first_stage_key) + loss, loss_dict = self(x) + return loss, loss_dict + + def training_step(self, batch, batch_idx): + loss, loss_dict = self.shared_step(batch) + + self.log_dict( + loss_dict, prog_bar=True, logger=True, on_step=True, on_epoch=True + ) + + self.log( + 'global_step', + self.global_step, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=False, + ) + + if self.use_scheduler: + lr = self.optimizers().param_groups[0]['lr'] + self.log( + 'lr_abs', + lr, + prog_bar=True, + logger=True, + on_step=True, + on_epoch=False, + ) + + return loss + + @torch.no_grad() + def validation_step(self, batch, batch_idx): + _, loss_dict_no_ema = self.shared_step(batch) + with self.ema_scope(): + _, loss_dict_ema = self.shared_step(batch) + loss_dict_ema = { + key + '_ema': loss_dict_ema[key] for key in loss_dict_ema + } + self.log_dict( + loss_dict_no_ema, + prog_bar=False, + logger=True, + on_step=False, + on_epoch=True, + ) + self.log_dict( + loss_dict_ema, + prog_bar=False, + logger=True, + on_step=False, + on_epoch=True, + ) + + def on_train_batch_end(self, *args, **kwargs): + if self.use_ema: + self.model_ema(self.model) + + def _get_rows_from_list(self, samples): + n_imgs_per_row = len(samples) + denoise_grid = rearrange(samples, 'n b c h w -> b n c h w') + denoise_grid = rearrange(denoise_grid, 'b n c h w -> (b n) c h w') + denoise_grid = make_grid(denoise_grid, nrow=n_imgs_per_row) + return denoise_grid + + @torch.no_grad() + def log_images( + self, batch, N=8, n_row=2, sample=True, return_keys=None, **kwargs + ): + log = dict() + x = self.get_input(batch, self.first_stage_key) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + x = x.to(self.device)[:N] + log['inputs'] = x + + # get diffusion row + diffusion_row = list() + x_start = x[:n_row] + + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(x_start) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + diffusion_row.append(x_noisy) + + log['diffusion_row'] = self._get_rows_from_list(diffusion_row) + + if sample: + # get denoise row + with self.ema_scope('Plotting'): + samples, denoise_row = self.sample( + batch_size=N, return_intermediates=True + ) + + log['samples'] = samples + log['denoise_row'] = self._get_rows_from_list(denoise_row) + + if return_keys: + if np.intersect1d(list(log.keys()), return_keys).shape[0] == 0: + return log + else: + return {key: log[key] for key in return_keys} + return log + + def configure_optimizers(self): + lr = self.learning_rate + params = list(self.model.parameters()) + if self.learn_logvar: + params = params + [self.logvar] + opt = torch.optim.AdamW(params, lr=lr) + return opt + + +class LatentDiffusion(DDPM): + """main class""" + + def __init__( + self, + first_stage_config, + cond_stage_config, + personalization_config, + num_timesteps_cond=None, + cond_stage_key='image', + cond_stage_trainable=False, + concat_mode=True, + cond_stage_forward=None, + conditioning_key=None, + scale_factor=1.0, + scale_by_std=False, + *args, + **kwargs, + ): + + self.num_timesteps_cond = default(num_timesteps_cond, 1) + self.scale_by_std = scale_by_std + assert self.num_timesteps_cond <= kwargs['timesteps'] + # for backwards compatibility after implementation of DiffusionWrapper + if conditioning_key is None: + conditioning_key = 'concat' if concat_mode else 'crossattn' + if cond_stage_config == '__is_unconditional__': + conditioning_key = None + ckpt_path = kwargs.pop('ckpt_path', None) + ignore_keys = kwargs.pop('ignore_keys', []) + super().__init__(conditioning_key=conditioning_key, *args, **kwargs) + self.concat_mode = concat_mode + self.cond_stage_trainable = cond_stage_trainable + self.cond_stage_key = cond_stage_key + + try: + self.num_downs = ( + len(first_stage_config.params.ddconfig.ch_mult) - 1 + ) + except: + self.num_downs = 0 + if not scale_by_std: + self.scale_factor = scale_factor + else: + self.register_buffer('scale_factor', torch.tensor(scale_factor)) + self.instantiate_first_stage(first_stage_config) + self.instantiate_cond_stage(cond_stage_config) + + self.cond_stage_forward = cond_stage_forward + self.clip_denoised = False + self.bbox_tokenizer = None + + self.restarted_from_ckpt = False + if ckpt_path is not None: + self.init_from_ckpt(ckpt_path, ignore_keys) + self.restarted_from_ckpt = True + + self.cond_stage_model.train = disabled_train + for param in self.cond_stage_model.parameters(): + param.requires_grad = False + + self.model.eval() + self.model.train = disabled_train + for param in self.model.parameters(): + param.requires_grad = False + + self.embedding_manager = self.instantiate_embedding_manager( + personalization_config, self.cond_stage_model + ) + self.textual_inversion_manager = TextualInversionManager( + tokenizer = self.cond_stage_model.tokenizer, + text_encoder = self.cond_stage_model.transformer, + full_precision = True + ) + # this circular component dependency is gross and bad, needs to be rethought + self.cond_stage_model.set_textual_inversion_manager(self.textual_inversion_manager) + + self.emb_ckpt_counter = 0 + + # if self.embedding_manager.is_clip: + # self.cond_stage_model.update_embedding_func(self.embedding_manager) + + for param in self.embedding_manager.embedding_parameters(): + param.requires_grad = True + + def make_cond_schedule( + self, + ): + self.cond_ids = torch.full( + size=(self.num_timesteps,), + fill_value=self.num_timesteps - 1, + dtype=torch.long, + ) + ids = torch.round( + torch.linspace(0, self.num_timesteps - 1, self.num_timesteps_cond) + ).long() + self.cond_ids[: self.num_timesteps_cond] = ids + + @rank_zero_only + @torch.no_grad() + def on_train_batch_start(self, batch, batch_idx, dataloader_idx=None): + # only for very first batch + if ( + self.scale_by_std + and self.current_epoch == 0 + and self.global_step == 0 + and batch_idx == 0 + and not self.restarted_from_ckpt + ): + assert ( + self.scale_factor == 1.0 + ), 'rather not use custom rescaling and std-rescaling simultaneously' + # set rescale weight to 1./std of encodings + print('### USING STD-RESCALING ###') + x = super().get_input(batch, self.first_stage_key) + x = x.to(self.device) + encoder_posterior = self.encode_first_stage(x) + z = self.get_first_stage_encoding(encoder_posterior).detach() + del self.scale_factor + self.register_buffer('scale_factor', 1.0 / z.flatten().std()) + print(f'setting self.scale_factor to {self.scale_factor}') + print('### USING STD-RESCALING ###') + + def register_schedule( + self, + given_betas=None, + beta_schedule='linear', + timesteps=1000, + linear_start=1e-4, + linear_end=2e-2, + cosine_s=8e-3, + ): + super().register_schedule( + given_betas, + beta_schedule, + timesteps, + linear_start, + linear_end, + cosine_s, + ) + + self.shorten_cond_schedule = self.num_timesteps_cond > 1 + if self.shorten_cond_schedule: + self.make_cond_schedule() + + def instantiate_first_stage(self, config): + model = instantiate_from_config(config) + self.first_stage_model = model.eval() + self.first_stage_model.train = disabled_train + for param in self.first_stage_model.parameters(): + param.requires_grad = False + + def instantiate_cond_stage(self, config): + if not self.cond_stage_trainable: + if config == '__is_first_stage__': + print('Using first stage also as cond stage.') + self.cond_stage_model = self.first_stage_model + elif config == '__is_unconditional__': + print( + f'Training {self.__class__.__name__} as an unconditional model.' + ) + self.cond_stage_model = None + # self.be_unconditional = True + else: + model = instantiate_from_config(config) + self.cond_stage_model = model.eval() + self.cond_stage_model.train = disabled_train + for param in self.cond_stage_model.parameters(): + param.requires_grad = False + else: + assert config != '__is_first_stage__' + assert config != '__is_unconditional__' + try: + model = instantiate_from_config(config) + except urllib.error.URLError: + raise SystemExit( + "* Couldn't load a dependency. Try running scripts/preload_models.py from an internet-conected machine." + ) + self.cond_stage_model = model + + def instantiate_embedding_manager(self, config, embedder): + model = instantiate_from_config(config, embedder=embedder) + + if config.params.get( + 'embedding_manager_ckpt', None + ): # do not load if missing OR empty string + model.load(config.params.embedding_manager_ckpt) + + return model + + def _get_denoise_row_from_list( + self, samples, desc='', force_no_decoder_quantization=False + ): + denoise_row = [] + for zd in tqdm(samples, desc=desc): + denoise_row.append( + self.decode_first_stage( + zd.to(self.device), + force_not_quantize=force_no_decoder_quantization, + ) + ) + n_imgs_per_row = len(denoise_row) + denoise_row = torch.stack(denoise_row) # n_log_step, n_row, C, H, W + denoise_grid = rearrange(denoise_row, 'n b c h w -> b n c h w') + denoise_grid = rearrange(denoise_grid, 'b n c h w -> (b n) c h w') + denoise_grid = make_grid(denoise_grid, nrow=n_imgs_per_row) + return denoise_grid + + def get_first_stage_encoding(self, encoder_posterior): + if isinstance(encoder_posterior, DiagonalGaussianDistribution): + z = encoder_posterior.sample() + elif isinstance(encoder_posterior, torch.Tensor): + z = encoder_posterior + else: + raise NotImplementedError( + f"encoder_posterior of type '{type(encoder_posterior)}' not yet implemented" + ) + return self.scale_factor * z + + def get_learned_conditioning(self, c, **kwargs): + if self.cond_stage_forward is None: + if hasattr(self.cond_stage_model, 'encode') and callable( + self.cond_stage_model.encode + ): + c = self.cond_stage_model.encode( + c, embedding_manager=self.embedding_manager,**kwargs + ) + if isinstance(c, DiagonalGaussianDistribution): + c = c.mode() + else: + c = self.cond_stage_model(c, **kwargs) + else: + assert hasattr(self.cond_stage_model, self.cond_stage_forward) + c = getattr(self.cond_stage_model, self.cond_stage_forward)(c, **kwargs) + return c + + def meshgrid(self, h, w): + y = torch.arange(0, h).view(h, 1, 1).repeat(1, w, 1) + x = torch.arange(0, w).view(1, w, 1).repeat(h, 1, 1) + + arr = torch.cat([y, x], dim=-1) + return arr + + def delta_border(self, h, w): + """ + :param h: height + :param w: width + :return: normalized distance to image border, + wtith min distance = 0 at border and max dist = 0.5 at image center + """ + lower_right_corner = torch.tensor([h - 1, w - 1]).view(1, 1, 2) + arr = self.meshgrid(h, w) / lower_right_corner + dist_left_up = torch.min(arr, dim=-1, keepdims=True)[0] + dist_right_down = torch.min(1 - arr, dim=-1, keepdims=True)[0] + edge_dist = torch.min( + torch.cat([dist_left_up, dist_right_down], dim=-1), dim=-1 + )[0] + return edge_dist + + def get_weighting(self, h, w, Ly, Lx, device): + weighting = self.delta_border(h, w) + weighting = torch.clip( + weighting, + self.split_input_params['clip_min_weight'], + self.split_input_params['clip_max_weight'], + ) + weighting = ( + weighting.view(1, h * w, 1).repeat(1, 1, Ly * Lx).to(device) + ) + + if self.split_input_params['tie_braker']: + L_weighting = self.delta_border(Ly, Lx) + L_weighting = torch.clip( + L_weighting, + self.split_input_params['clip_min_tie_weight'], + self.split_input_params['clip_max_tie_weight'], + ) + + L_weighting = L_weighting.view(1, 1, Ly * Lx).to(device) + weighting = weighting * L_weighting + return weighting + + def get_fold_unfold( + self, x, kernel_size, stride, uf=1, df=1 + ): # todo load once not every time, shorten code + """ + :param x: img of size (bs, c, h, w) + :return: n img crops of size (n, bs, c, kernel_size[0], kernel_size[1]) + """ + bs, nc, h, w = x.shape + + # number of crops in image + Ly = (h - kernel_size[0]) // stride[0] + 1 + Lx = (w - kernel_size[1]) // stride[1] + 1 + + if uf == 1 and df == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride + ) + unfold = torch.nn.Unfold(**fold_params) + + fold = torch.nn.Fold(output_size=x.shape[2:], **fold_params) + + weighting = self.get_weighting( + kernel_size[0], kernel_size[1], Ly, Lx, x.device + ).to(x.dtype) + normalization = fold(weighting).view( + 1, 1, h, w + ) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0], kernel_size[1], Ly * Lx) + ) + + elif uf > 1 and df == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride + ) + unfold = torch.nn.Unfold(**fold_params) + + fold_params2 = dict( + kernel_size=(kernel_size[0] * uf, kernel_size[0] * uf), + dilation=1, + padding=0, + stride=(stride[0] * uf, stride[1] * uf), + ) + fold = torch.nn.Fold( + output_size=(x.shape[2] * uf, x.shape[3] * uf), **fold_params2 + ) + + weighting = self.get_weighting( + kernel_size[0] * uf, kernel_size[1] * uf, Ly, Lx, x.device + ).to(x.dtype) + normalization = fold(weighting).view( + 1, 1, h * uf, w * uf + ) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0] * uf, kernel_size[1] * uf, Ly * Lx) + ) + + elif df > 1 and uf == 1: + fold_params = dict( + kernel_size=kernel_size, dilation=1, padding=0, stride=stride + ) + unfold = torch.nn.Unfold(**fold_params) + + fold_params2 = dict( + kernel_size=(kernel_size[0] // df, kernel_size[0] // df), + dilation=1, + padding=0, + stride=(stride[0] // df, stride[1] // df), + ) + fold = torch.nn.Fold( + output_size=(x.shape[2] // df, x.shape[3] // df), + **fold_params2, + ) + + weighting = self.get_weighting( + kernel_size[0] // df, kernel_size[1] // df, Ly, Lx, x.device + ).to(x.dtype) + normalization = fold(weighting).view( + 1, 1, h // df, w // df + ) # normalizes the overlap + weighting = weighting.view( + (1, 1, kernel_size[0] // df, kernel_size[1] // df, Ly * Lx) + ) + + else: + raise NotImplementedError + + return fold, unfold, normalization, weighting + + @torch.no_grad() + def get_input( + self, + batch, + k, + return_first_stage_outputs=False, + force_c_encode=False, + cond_key=None, + return_original_cond=False, + bs=None, + ): + x = super().get_input(batch, k) + if bs is not None: + x = x[:bs] + x = x.to(self.device) + encoder_posterior = self.encode_first_stage(x) + z = self.get_first_stage_encoding(encoder_posterior).detach() + + if self.model.conditioning_key is not None: + if cond_key is None: + cond_key = self.cond_stage_key + if cond_key != self.first_stage_key: + if cond_key in ['caption', 'coordinates_bbox']: + xc = batch[cond_key] + elif cond_key == 'class_label': + xc = batch + else: + xc = super().get_input(batch, cond_key).to(self.device) + else: + xc = x + if not self.cond_stage_trainable or force_c_encode: + if isinstance(xc, dict) or isinstance(xc, list): + # import pudb; pudb.set_trace() + c = self.get_learned_conditioning(xc) + else: + c = self.get_learned_conditioning(xc.to(self.device)) + else: + c = xc + if bs is not None: + c = c[:bs] + + if self.use_positional_encodings: + pos_x, pos_y = self.compute_latent_shifts(batch) + ckey = __conditioning_keys__[self.model.conditioning_key] + c = {ckey: c, 'pos_x': pos_x, 'pos_y': pos_y} + + else: + c = None + xc = None + if self.use_positional_encodings: + pos_x, pos_y = self.compute_latent_shifts(batch) + c = {'pos_x': pos_x, 'pos_y': pos_y} + out = [z, c] + if return_first_stage_outputs: + xrec = self.decode_first_stage(z) + out.extend([x, xrec]) + if return_original_cond: + out.append(xc) + return out + + @torch.no_grad() + def decode_first_stage( + self, z, predict_cids=False, force_not_quantize=False + ): + if predict_cids: + if z.dim() == 4: + z = torch.argmax(z.exp(), dim=1).long() + z = self.first_stage_model.quantize.get_codebook_entry( + z, shape=None + ) + z = rearrange(z, 'b h w c -> b c h w').contiguous() + + z = 1.0 / self.scale_factor * z + + if hasattr(self, 'split_input_params'): + if self.split_input_params['patch_distributed_vq']: + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + uf = self.split_input_params['vqf'] + bs, nc, h, w = z.shape + if ks[0] > h or ks[1] > w: + ks = (min(ks[0], h), min(ks[1], w)) + print('reducing Kernel') + + if stride[0] > h or stride[1] > w: + stride = (min(stride[0], h), min(stride[1], w)) + print('reducing stride') + + fold, unfold, normalization, weighting = self.get_fold_unfold( + z, ks, stride, uf=uf + ) + + z = unfold(z) # (bn, nc * prod(**ks), L) + # 1. Reshape to img shape + z = z.view( + (z.shape[0], -1, ks[0], ks[1], z.shape[-1]) + ) # (bn, nc, ks[0], ks[1], L ) + + # 2. apply model loop over last dim + if isinstance(self.first_stage_model, VQModelInterface): + output_list = [ + self.first_stage_model.decode( + z[:, :, :, :, i], + force_not_quantize=predict_cids + or force_not_quantize, + ) + for i in range(z.shape[-1]) + ] + else: + + output_list = [ + self.first_stage_model.decode(z[:, :, :, :, i]) + for i in range(z.shape[-1]) + ] + + o = torch.stack( + output_list, axis=-1 + ) # # (bn, nc, ks[0], ks[1], L) + o = o * weighting + # Reverse 1. reshape to img shape + o = o.view( + (o.shape[0], -1, o.shape[-1]) + ) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + decoded = fold(o) + decoded = decoded / normalization # norm is shape (1, 1, h, w) + return decoded + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, + force_not_quantize=predict_cids or force_not_quantize, + ) + else: + return self.first_stage_model.decode(z) + + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, force_not_quantize=predict_cids or force_not_quantize + ) + else: + return self.first_stage_model.decode(z) + + # same as above but without decorator + def differentiable_decode_first_stage( + self, z, predict_cids=False, force_not_quantize=False + ): + if predict_cids: + if z.dim() == 4: + z = torch.argmax(z.exp(), dim=1).long() + z = self.first_stage_model.quantize.get_codebook_entry( + z, shape=None + ) + z = rearrange(z, 'b h w c -> b c h w').contiguous() + + z = 1.0 / self.scale_factor * z + + if hasattr(self, 'split_input_params'): + if self.split_input_params['patch_distributed_vq']: + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + uf = self.split_input_params['vqf'] + bs, nc, h, w = z.shape + if ks[0] > h or ks[1] > w: + ks = (min(ks[0], h), min(ks[1], w)) + print('reducing Kernel') + + if stride[0] > h or stride[1] > w: + stride = (min(stride[0], h), min(stride[1], w)) + print('reducing stride') + + fold, unfold, normalization, weighting = self.get_fold_unfold( + z, ks, stride, uf=uf + ) + + z = unfold(z) # (bn, nc * prod(**ks), L) + # 1. Reshape to img shape + z = z.view( + (z.shape[0], -1, ks[0], ks[1], z.shape[-1]) + ) # (bn, nc, ks[0], ks[1], L ) + + # 2. apply model loop over last dim + if isinstance(self.first_stage_model, VQModelInterface): + output_list = [ + self.first_stage_model.decode( + z[:, :, :, :, i], + force_not_quantize=predict_cids + or force_not_quantize, + ) + for i in range(z.shape[-1]) + ] + else: + + output_list = [ + self.first_stage_model.decode(z[:, :, :, :, i]) + for i in range(z.shape[-1]) + ] + + o = torch.stack( + output_list, axis=-1 + ) # # (bn, nc, ks[0], ks[1], L) + o = o * weighting + # Reverse 1. reshape to img shape + o = o.view( + (o.shape[0], -1, o.shape[-1]) + ) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + decoded = fold(o) + decoded = decoded / normalization # norm is shape (1, 1, h, w) + return decoded + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, + force_not_quantize=predict_cids or force_not_quantize, + ) + else: + return self.first_stage_model.decode(z) + + else: + if isinstance(self.first_stage_model, VQModelInterface): + return self.first_stage_model.decode( + z, force_not_quantize=predict_cids or force_not_quantize + ) + else: + return self.first_stage_model.decode(z) + + @torch.no_grad() + def encode_first_stage(self, x): + if hasattr(self, 'split_input_params'): + if self.split_input_params['patch_distributed_vq']: + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + df = self.split_input_params['vqf'] + self.split_input_params['original_image_size'] = x.shape[-2:] + bs, nc, h, w = x.shape + if ks[0] > h or ks[1] > w: + ks = (min(ks[0], h), min(ks[1], w)) + print('reducing Kernel') + + if stride[0] > h or stride[1] > w: + stride = (min(stride[0], h), min(stride[1], w)) + print('reducing stride') + + fold, unfold, normalization, weighting = self.get_fold_unfold( + x, ks, stride, df=df + ) + z = unfold(x) # (bn, nc * prod(**ks), L) + # Reshape to img shape + z = z.view( + (z.shape[0], -1, ks[0], ks[1], z.shape[-1]) + ) # (bn, nc, ks[0], ks[1], L ) + + output_list = [ + self.first_stage_model.encode(z[:, :, :, :, i]) + for i in range(z.shape[-1]) + ] + + o = torch.stack(output_list, axis=-1) + o = o * weighting + + # Reverse reshape to img shape + o = o.view( + (o.shape[0], -1, o.shape[-1]) + ) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + decoded = fold(o) + decoded = decoded / normalization + return decoded + + else: + return self.first_stage_model.encode(x) + else: + return self.first_stage_model.encode(x) + + def shared_step(self, batch, **kwargs): + x, c = self.get_input(batch, self.first_stage_key) + loss = self(x, c) + return loss + + def forward(self, x, c, *args, **kwargs): + t = torch.randint( + 0, self.num_timesteps, (x.shape[0],), device=self.device + ).long() + if self.model.conditioning_key is not None: + assert c is not None + if self.cond_stage_trainable: + c = self.get_learned_conditioning(c) + if self.shorten_cond_schedule: # TODO: drop this option + tc = self.cond_ids[t].to(self.device) + c = self.q_sample( + x_start=c, t=tc, noise=torch.randn_like(c.float()) + ) + + return self.p_losses(x, c, t, *args, **kwargs) + + def _rescale_annotations( + self, bboxes, crop_coordinates + ): # TODO: move to dataset + def rescale_bbox(bbox): + x0 = clamp((bbox[0] - crop_coordinates[0]) / crop_coordinates[2]) + y0 = clamp((bbox[1] - crop_coordinates[1]) / crop_coordinates[3]) + w = min(bbox[2] / crop_coordinates[2], 1 - x0) + h = min(bbox[3] / crop_coordinates[3], 1 - y0) + return x0, y0, w, h + + return [rescale_bbox(b) for b in bboxes] + + def apply_model(self, x_noisy, t, cond, return_ids=False): + + if isinstance(cond, dict): + # hybrid case, cond is exptected to be a dict + pass + else: + if not isinstance(cond, list): + cond = [cond] + key = ( + 'c_concat' + if self.model.conditioning_key == 'concat' + else 'c_crossattn' + ) + cond = {key: cond} + + if hasattr(self, 'split_input_params'): + assert ( + len(cond) == 1 + ) # todo can only deal with one conditioning atm + assert not return_ids + ks = self.split_input_params['ks'] # eg. (128, 128) + stride = self.split_input_params['stride'] # eg. (64, 64) + + h, w = x_noisy.shape[-2:] + + fold, unfold, normalization, weighting = self.get_fold_unfold( + x_noisy, ks, stride + ) + + z = unfold(x_noisy) # (bn, nc * prod(**ks), L) + # Reshape to img shape + z = z.view( + (z.shape[0], -1, ks[0], ks[1], z.shape[-1]) + ) # (bn, nc, ks[0], ks[1], L ) + z_list = [z[:, :, :, :, i] for i in range(z.shape[-1])] + + if ( + self.cond_stage_key + in ['image', 'LR_image', 'segmentation', 'bbox_img'] + and self.model.conditioning_key + ): # todo check for completeness + c_key = next(iter(cond.keys())) # get key + c = next(iter(cond.values())) # get value + assert ( + len(c) == 1 + ) # todo extend to list with more than one elem + c = c[0] # get element + + c = unfold(c) + c = c.view( + (c.shape[0], -1, ks[0], ks[1], c.shape[-1]) + ) # (bn, nc, ks[0], ks[1], L ) + + cond_list = [ + {c_key: [c[:, :, :, :, i]]} for i in range(c.shape[-1]) + ] + + elif self.cond_stage_key == 'coordinates_bbox': + assert ( + 'original_image_size' in self.split_input_params + ), 'BoudingBoxRescaling is missing original_image_size' + + # assuming padding of unfold is always 0 and its dilation is always 1 + n_patches_per_row = int((w - ks[0]) / stride[0] + 1) + full_img_h, full_img_w = self.split_input_params[ + 'original_image_size' + ] + # as we are operating on latents, we need the factor from the original image size to the + # spatial latent size to properly rescale the crops for regenerating the bbox annotations + num_downs = self.first_stage_model.encoder.num_resolutions - 1 + rescale_latent = 2 ** (num_downs) + + # get top left positions of patches as conforming for the bbbox tokenizer, therefore we + # need to rescale the tl patch coordinates to be in between (0,1) + tl_patch_coordinates = [ + ( + rescale_latent + * stride[0] + * (patch_nr % n_patches_per_row) + / full_img_w, + rescale_latent + * stride[1] + * (patch_nr // n_patches_per_row) + / full_img_h, + ) + for patch_nr in range(z.shape[-1]) + ] + + # patch_limits are tl_coord, width and height coordinates as (x_tl, y_tl, h, w) + patch_limits = [ + ( + x_tl, + y_tl, + rescale_latent * ks[0] / full_img_w, + rescale_latent * ks[1] / full_img_h, + ) + for x_tl, y_tl in tl_patch_coordinates + ] + # patch_values = [(np.arange(x_tl,min(x_tl+ks, 1.)),np.arange(y_tl,min(y_tl+ks, 1.))) for x_tl, y_tl in tl_patch_coordinates] + + # tokenize crop coordinates for the bounding boxes of the respective patches + patch_limits_tknzd = [ + torch.LongTensor(self.bbox_tokenizer._crop_encoder(bbox))[ + None + ].to(self.device) + for bbox in patch_limits + ] # list of length l with tensors of shape (1, 2) + print(patch_limits_tknzd[0].shape) + # cut tknzd crop position from conditioning + assert isinstance( + cond, dict + ), 'cond must be dict to be fed into model' + cut_cond = cond['c_crossattn'][0][..., :-2].to(self.device) + print(cut_cond.shape) + + adapted_cond = torch.stack( + [ + torch.cat([cut_cond, p], dim=1) + for p in patch_limits_tknzd + ] + ) + adapted_cond = rearrange(adapted_cond, 'l b n -> (l b) n') + print(adapted_cond.shape) + adapted_cond = self.get_learned_conditioning(adapted_cond) + print(adapted_cond.shape) + adapted_cond = rearrange( + adapted_cond, '(l b) n d -> l b n d', l=z.shape[-1] + ) + print(adapted_cond.shape) + + cond_list = [{'c_crossattn': [e]} for e in adapted_cond] + + else: + cond_list = [ + cond for i in range(z.shape[-1]) + ] # Todo make this more efficient + + # apply model by loop over crops + output_list = [ + self.model(z_list[i], t, **cond_list[i]) + for i in range(z.shape[-1]) + ] + assert not isinstance( + output_list[0], tuple + ) # todo cant deal with multiple model outputs check this never happens + + o = torch.stack(output_list, axis=-1) + o = o * weighting + # Reverse reshape to img shape + o = o.view( + (o.shape[0], -1, o.shape[-1]) + ) # (bn, nc * ks[0] * ks[1], L) + # stitch crops together + x_recon = fold(o) / normalization + + else: + x_recon = self.model(x_noisy, t, **cond) + + if isinstance(x_recon, tuple) and not return_ids: + return x_recon[0] + else: + return x_recon + + def _predict_eps_from_xstart(self, x_t, t, pred_xstart): + return ( + extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) + * x_t + - pred_xstart + ) / extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) + + def _prior_bpd(self, x_start): + """ + Get the prior KL term for the variational lower-bound, measured in + bits-per-dim. + This term can't be optimized, as it only depends on the encoder. + :param x_start: the [N x C x ...] tensor of inputs. + :return: a batch of [N] KL values (in bits), one per batch element. + """ + batch_size = x_start.shape[0] + t = torch.tensor( + [self.num_timesteps - 1] * batch_size, device=x_start.device + ) + qt_mean, _, qt_log_variance = self.q_mean_variance(x_start, t) + kl_prior = normal_kl( + mean1=qt_mean, logvar1=qt_log_variance, mean2=0.0, logvar2=0.0 + ) + return mean_flat(kl_prior) / np.log(2.0) + + def p_losses(self, x_start, cond, t, noise=None): + noise = default(noise, lambda: torch.randn_like(x_start)) + x_noisy = self.q_sample(x_start=x_start, t=t, noise=noise) + model_output = self.apply_model(x_noisy, t, cond) + + loss_dict = {} + prefix = 'train' if self.training else 'val' + + if self.parameterization == 'x0': + target = x_start + elif self.parameterization == 'eps': + target = noise + else: + raise NotImplementedError() + + loss_simple = self.get_loss(model_output, target, mean=False).mean( + [1, 2, 3] + ) + loss_dict.update({f'{prefix}/loss_simple': loss_simple.mean()}) + + logvar_t = self.logvar[t.item()].to(self.device) + loss = loss_simple / torch.exp(logvar_t) + logvar_t + # loss = loss_simple / torch.exp(self.logvar) + self.logvar + if self.learn_logvar: + loss_dict.update({f'{prefix}/loss_gamma': loss.mean()}) + loss_dict.update({'logvar': self.logvar.data.mean()}) + + loss = self.l_simple_weight * loss.mean() + + loss_vlb = self.get_loss(model_output, target, mean=False).mean( + dim=(1, 2, 3) + ) + loss_vlb = (self.lvlb_weights[t] * loss_vlb).mean() + loss_dict.update({f'{prefix}/loss_vlb': loss_vlb}) + loss += self.original_elbo_weight * loss_vlb + loss_dict.update({f'{prefix}/loss': loss}) + + if self.embedding_reg_weight > 0: + loss_embedding_reg = ( + self.embedding_manager.embedding_to_coarse_loss().mean() + ) + + loss_dict.update({f'{prefix}/loss_emb_reg': loss_embedding_reg}) + + loss += self.embedding_reg_weight * loss_embedding_reg + loss_dict.update({f'{prefix}/loss': loss}) + + return loss, loss_dict + + def p_mean_variance( + self, + x, + c, + t, + clip_denoised: bool, + return_codebook_ids=False, + quantize_denoised=False, + return_x0=False, + score_corrector=None, + corrector_kwargs=None, + ): + t_in = t + model_out = self.apply_model( + x, t_in, c, return_ids=return_codebook_ids + ) + + if score_corrector is not None: + assert self.parameterization == 'eps' + model_out = score_corrector.modify_score( + self, model_out, x, t, c, **corrector_kwargs + ) + + if return_codebook_ids: + model_out, logits = model_out + + if self.parameterization == 'eps': + x_recon = self.predict_start_from_noise(x, t=t, noise=model_out) + elif self.parameterization == 'x0': + x_recon = model_out + else: + raise NotImplementedError() + + if clip_denoised: + x_recon.clamp_(-1.0, 1.0) + if quantize_denoised: + x_recon, _, [_, _, indices] = self.first_stage_model.quantize( + x_recon + ) + ( + model_mean, + posterior_variance, + posterior_log_variance, + ) = self.q_posterior(x_start=x_recon, x_t=x, t=t) + if return_codebook_ids: + return ( + model_mean, + posterior_variance, + posterior_log_variance, + logits, + ) + elif return_x0: + return ( + model_mean, + posterior_variance, + posterior_log_variance, + x_recon, + ) + else: + return model_mean, posterior_variance, posterior_log_variance + + @torch.no_grad() + def p_sample( + self, + x, + c, + t, + clip_denoised=False, + repeat_noise=False, + return_codebook_ids=False, + quantize_denoised=False, + return_x0=False, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + ): + b, *_, device = *x.shape, x.device + outputs = self.p_mean_variance( + x=x, + c=c, + t=t, + clip_denoised=clip_denoised, + return_codebook_ids=return_codebook_ids, + quantize_denoised=quantize_denoised, + return_x0=return_x0, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + ) + if return_codebook_ids: + raise DeprecationWarning('Support dropped.') + model_mean, _, model_log_variance, logits = outputs + elif return_x0: + model_mean, _, model_log_variance, x0 = outputs + else: + model_mean, _, model_log_variance = outputs + + noise = noise_like(x.shape, device, repeat_noise) * temperature + if noise_dropout > 0.0: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + # no noise when t == 0 + nonzero_mask = (1 - (t == 0).float()).reshape( + b, *((1,) * (len(x.shape) - 1)) + ) + + if return_codebook_ids: + return model_mean + nonzero_mask * ( + 0.5 * model_log_variance + ).exp() * noise, logits.argmax(dim=1) + if return_x0: + return ( + model_mean + + nonzero_mask * (0.5 * model_log_variance).exp() * noise, + x0, + ) + else: + return ( + model_mean + + nonzero_mask * (0.5 * model_log_variance).exp() * noise + ) + + @torch.no_grad() + def progressive_denoising( + self, + cond, + shape, + verbose=True, + callback=None, + quantize_denoised=False, + img_callback=None, + mask=None, + x0=None, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + batch_size=None, + x_T=None, + start_T=None, + log_every_t=None, + ): + if not log_every_t: + log_every_t = self.log_every_t + timesteps = self.num_timesteps + if batch_size is not None: + b = batch_size if batch_size is not None else shape[0] + shape = [batch_size] + list(shape) + else: + b = batch_size = shape[0] + if x_T is None: + img = torch.randn(shape, device=self.device) + else: + img = x_T + intermediates = [] + if cond is not None: + if isinstance(cond, dict): + cond = { + key: cond[key][:batch_size] + if not isinstance(cond[key], list) + else list(map(lambda x: x[:batch_size], cond[key])) + for key in cond + } + else: + cond = ( + [c[:batch_size] for c in cond] + if isinstance(cond, list) + else cond[:batch_size] + ) + + if start_T is not None: + timesteps = min(timesteps, start_T) + iterator = ( + tqdm( + reversed(range(0, timesteps)), + desc='Progressive Generation', + total=timesteps, + ) + if verbose + else reversed(range(0, timesteps)) + ) + if type(temperature) == float: + temperature = [temperature] * timesteps + + for i in iterator: + ts = torch.full((b,), i, device=self.device, dtype=torch.long) + if self.shorten_cond_schedule: + assert self.model.conditioning_key != 'hybrid' + tc = self.cond_ids[ts].to(cond.device) + cond = self.q_sample( + x_start=cond, t=tc, noise=torch.randn_like(cond) + ) + + img, x0_partial = self.p_sample( + img, + cond, + ts, + clip_denoised=self.clip_denoised, + quantize_denoised=quantize_denoised, + return_x0=True, + temperature=temperature[i], + noise_dropout=noise_dropout, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + ) + if mask is not None: + assert x0 is not None + img_orig = self.q_sample(x0, ts) + img = img_orig * mask + (1.0 - mask) * img + + if i % log_every_t == 0 or i == timesteps - 1: + intermediates.append(x0_partial) + if callback: + callback(i) + if img_callback: + img_callback(img, i) + return img, intermediates + + @torch.no_grad() + def p_sample_loop( + self, + cond, + shape, + return_intermediates=False, + x_T=None, + verbose=True, + callback=None, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + img_callback=None, + start_T=None, + log_every_t=None, + ): + + if not log_every_t: + log_every_t = self.log_every_t + device = self.betas.device + b = shape[0] + if x_T is None: + img = torch.randn(shape, device=device) + else: + img = x_T + + intermediates = [img] + if timesteps is None: + timesteps = self.num_timesteps + + if start_T is not None: + timesteps = min(timesteps, start_T) + iterator = ( + tqdm( + reversed(range(0, timesteps)), + desc='Sampling t', + total=timesteps, + ) + if verbose + else reversed(range(0, timesteps)) + ) + + if mask is not None: + assert x0 is not None + assert ( + x0.shape[2:3] == mask.shape[2:3] + ) # spatial size has to match + + for i in iterator: + ts = torch.full((b,), i, device=device, dtype=torch.long) + if self.shorten_cond_schedule: + assert self.model.conditioning_key != 'hybrid' + tc = self.cond_ids[ts].to(cond.device) + cond = self.q_sample( + x_start=cond, t=tc, noise=torch.randn_like(cond) + ) + + img = self.p_sample( + img, + cond, + ts, + clip_denoised=self.clip_denoised, + quantize_denoised=quantize_denoised, + ) + if mask is not None: + img_orig = self.q_sample(x0, ts) + img = img_orig * mask + (1.0 - mask) * img + + if i % log_every_t == 0 or i == timesteps - 1: + intermediates.append(img) + if callback: + callback(i) + if img_callback: + img_callback(img, i) + + if return_intermediates: + return img, intermediates + return img + + @torch.no_grad() + def sample( + self, + cond, + batch_size=16, + return_intermediates=False, + x_T=None, + verbose=True, + timesteps=None, + quantize_denoised=False, + mask=None, + x0=None, + shape=None, + **kwargs, + ): + if shape is None: + shape = ( + batch_size, + self.channels, + self.image_size, + self.image_size, + ) + if cond is not None: + if isinstance(cond, dict): + cond = { + key: cond[key][:batch_size] + if not isinstance(cond[key], list) + else list(map(lambda x: x[:batch_size], cond[key])) + for key in cond + } + else: + cond = ( + [c[:batch_size] for c in cond] + if isinstance(cond, list) + else cond[:batch_size] + ) + return self.p_sample_loop( + cond, + shape, + return_intermediates=return_intermediates, + x_T=x_T, + verbose=verbose, + timesteps=timesteps, + quantize_denoised=quantize_denoised, + mask=mask, + x0=x0, + ) + + @torch.no_grad() + def sample_log(self, cond, batch_size, ddim, ddim_steps, **kwargs): + + if ddim: + ddim_sampler = DDIMSampler(self) + shape = (self.channels, self.image_size, self.image_size) + samples, intermediates = ddim_sampler.sample( + ddim_steps, batch_size, shape, cond, verbose=False, **kwargs + ) + + else: + samples, intermediates = self.sample( + cond=cond, + batch_size=batch_size, + return_intermediates=True, + **kwargs, + ) + + return samples, intermediates + + @torch.no_grad() + def get_unconditional_conditioning(self, batch_size, null_label=None): + if null_label is not None: + xc = null_label + if isinstance(xc, ListConfig): + xc = list(xc) + if isinstance(xc, dict) or isinstance(xc, list): + c = self.get_learned_conditioning(xc) + else: + if hasattr(xc, "to"): + xc = xc.to(self.device) + c = self.get_learned_conditioning(xc) + else: + # todo: get null label from cond_stage_model + raise NotImplementedError() + c = repeat(c, "1 ... -> b ...", b=batch_size).to(self.device) + return c + + @torch.no_grad() + def log_images( + self, + batch, + N=8, + n_row=4, + sample=True, + ddim_steps=50, + ddim_eta=1.0, + return_keys=None, + quantize_denoised=True, + inpaint=False, + plot_denoise_rows=False, + plot_progressive_rows=False, + plot_diffusion_rows=False, + **kwargs, + ): + + use_ddim = ddim_steps is not None + + log = dict() + z, c, x, xrec, xc = self.get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=N, + ) + N = min(x.shape[0], N) + n_row = min(x.shape[0], n_row) + log['inputs'] = x + log['reconstruction'] = xrec + if self.model.conditioning_key is not None: + if hasattr(self.cond_stage_model, 'decode'): + xc = self.cond_stage_model.decode(c) + log['conditioning'] = xc + elif self.cond_stage_key in ['caption']: + xc = log_txt_as_img((x.shape[2], x.shape[3]), batch['caption']) + log['conditioning'] = xc + elif self.cond_stage_key == 'class_label': + xc = log_txt_as_img( + (x.shape[2], x.shape[3]), batch['human_label'] + ) + log['conditioning'] = xc + elif isimage(xc): + log['conditioning'] = xc + if ismap(xc): + log['original_conditioning'] = self.to_rgb(xc) + + if plot_diffusion_rows: + # get diffusion row + diffusion_row = list() + z_start = z[:n_row] + for t in range(self.num_timesteps): + if t % self.log_every_t == 0 or t == self.num_timesteps - 1: + t = repeat(torch.tensor([t]), '1 -> b', b=n_row) + t = t.to(self.device).long() + noise = torch.randn_like(z_start) + z_noisy = self.q_sample(x_start=z_start, t=t, noise=noise) + diffusion_row.append(self.decode_first_stage(z_noisy)) + + diffusion_row = torch.stack( + diffusion_row + ) # n_log_step, n_row, C, H, W + diffusion_grid = rearrange(diffusion_row, 'n b c h w -> b n c h w') + diffusion_grid = rearrange( + diffusion_grid, 'b n c h w -> (b n) c h w' + ) + diffusion_grid = make_grid( + diffusion_grid, nrow=diffusion_row.shape[0] + ) + log['diffusion_row'] = diffusion_grid + + if sample: + # get denoise row + with self.ema_scope('Plotting'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + ) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True) + x_samples = self.decode_first_stage(samples) + log['samples'] = x_samples + if plot_denoise_rows: + denoise_grid = self._get_denoise_row_from_list(z_denoise_row) + log['denoise_row'] = denoise_grid + + uc = self.get_learned_conditioning(len(c) * ['']) + sample_scaled, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + unconditional_guidance_scale=5.0, + unconditional_conditioning=uc, + ) + log['samples_scaled'] = self.decode_first_stage(sample_scaled) + + if ( + quantize_denoised + and not isinstance(self.first_stage_model, AutoencoderKL) + and not isinstance(self.first_stage_model, IdentityFirstStage) + ): + # also display when quantizing x0 while sampling + with self.ema_scope('Plotting Quantized Denoised'): + samples, z_denoise_row = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + ddim_steps=ddim_steps, + eta=ddim_eta, + quantize_denoised=True, + ) + # samples, z_denoise_row = self.sample(cond=c, batch_size=N, return_intermediates=True, + # quantize_denoised=True) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_x0_quantized'] = x_samples + + if inpaint: + # make a simple center square + b, h, w = z.shape[0], z.shape[2], z.shape[3] + mask = torch.ones(N, h, w).to(self.device) + # zeros will be filled in + mask[:, h // 4 : 3 * h // 4, w // 4 : 3 * w // 4] = 0.0 + mask = mask[:, None, ...] + with self.ema_scope('Plotting Inpaint'): + + samples, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + eta=ddim_eta, + ddim_steps=ddim_steps, + x0=z[:N], + mask=mask, + ) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_inpainting'] = x_samples + log['mask'] = mask + + # outpaint + with self.ema_scope('Plotting Outpaint'): + samples, _ = self.sample_log( + cond=c, + batch_size=N, + ddim=use_ddim, + eta=ddim_eta, + ddim_steps=ddim_steps, + x0=z[:N], + mask=mask, + ) + x_samples = self.decode_first_stage(samples.to(self.device)) + log['samples_outpainting'] = x_samples + + if plot_progressive_rows: + with self.ema_scope('Plotting Progressives'): + img, progressives = self.progressive_denoising( + c, + shape=(self.channels, self.image_size, self.image_size), + batch_size=N, + ) + prog_row = self._get_denoise_row_from_list( + progressives, desc='Progressive Generation' + ) + log['progressive_row'] = prog_row + + if return_keys: + if np.intersect1d(list(log.keys()), return_keys).shape[0] == 0: + return log + else: + return {key: log[key] for key in return_keys} + return log + + def configure_optimizers(self): + lr = self.learning_rate + + if self.embedding_manager is not None: + params = list(self.embedding_manager.embedding_parameters()) + # params = list(self.cond_stage_model.transformer.text_model.embeddings.embedding_manager.embedding_parameters()) + else: + params = list(self.model.parameters()) + if self.cond_stage_trainable: + print( + f'{self.__class__.__name__}: Also optimizing conditioner params!' + ) + params = params + list(self.cond_stage_model.parameters()) + if self.learn_logvar: + print('Diffusion model optimizing logvar') + params.append(self.logvar) + opt = torch.optim.AdamW(params, lr=lr) + if self.use_scheduler: + assert 'target' in self.scheduler_config + scheduler = instantiate_from_config(self.scheduler_config) + + print('Setting up LambdaLR scheduler...') + scheduler = [ + { + 'scheduler': LambdaLR(opt, lr_lambda=scheduler.schedule), + 'interval': 'step', + 'frequency': 1, + } + ] + return [opt], scheduler + return opt + + @torch.no_grad() + def to_rgb(self, x): + x = x.float() + if not hasattr(self, 'colorize'): + self.colorize = torch.randn(3, x.shape[1], 1, 1).to(x) + x = nn.functional.conv2d(x, weight=self.colorize) + x = 2.0 * (x - x.min()) / (x.max() - x.min()) - 1.0 + return x + + @rank_zero_only + def on_save_checkpoint(self, checkpoint): + checkpoint.clear() + + if os.path.isdir(self.trainer.checkpoint_callback.dirpath): + self.embedding_manager.save( + os.path.join( + self.trainer.checkpoint_callback.dirpath, 'embeddings.pt' + ) + ) + + if (self.global_step - self.emb_ckpt_counter) > 500: + self.embedding_manager.save( + os.path.join( + self.trainer.checkpoint_callback.dirpath, + f'embeddings_gs-{self.global_step}.pt', + ) + ) + + self.emb_ckpt_counter += 500 + + +class DiffusionWrapper(pl.LightningModule): + def __init__(self, diff_model_config, conditioning_key): + super().__init__() + self.diffusion_model = instantiate_from_config(diff_model_config) + self.conditioning_key = conditioning_key + assert self.conditioning_key in [ + None, + 'concat', + 'crossattn', + 'hybrid', + 'adm', + ] + + def forward(self, x, t, c_concat: list = None, c_crossattn: list = None): + if self.conditioning_key is None: + out = self.diffusion_model(x, t) + elif self.conditioning_key == 'concat': + xc = torch.cat([x] + c_concat, dim=1) + out = self.diffusion_model(xc, t) + elif self.conditioning_key == 'crossattn': + cc = torch.cat(c_crossattn, 1) + out = self.diffusion_model(x, t, context=cc) + elif self.conditioning_key == 'hybrid': + cc = torch.cat(c_crossattn, 1) + xc = torch.cat([x] + c_concat, dim=1) + out = self.diffusion_model(xc, t, context=cc) + elif self.conditioning_key == 'adm': + cc = c_crossattn[0] + out = self.diffusion_model(x, t, y=cc) + else: + raise NotImplementedError() + + return out + + +class Layout2ImgDiffusion(LatentDiffusion): + # TODO: move all layout-specific hacks to this class + def __init__(self, cond_stage_key, *args, **kwargs): + assert ( + cond_stage_key == 'coordinates_bbox' + ), 'Layout2ImgDiffusion only for cond_stage_key="coordinates_bbox"' + super().__init__(cond_stage_key=cond_stage_key, *args, **kwargs) + + def log_images(self, batch, N=8, *args, **kwargs): + logs = super().log_images(batch=batch, N=N, *args, **kwargs) + + key = 'train' if self.training else 'validation' + dset = self.trainer.datamodule.datasets[key] + mapper = dset.conditional_builders[self.cond_stage_key] + + bbox_imgs = [] + map_fn = lambda catno: dset.get_textual_label( + dset.get_category_id(catno) + ) + for tknzd_bbox in batch[self.cond_stage_key][:N]: + bboximg = mapper.plot( + tknzd_bbox.detach().cpu(), map_fn, (256, 256) + ) + bbox_imgs.append(bboximg) + + cond_img = torch.stack(bbox_imgs, dim=0) + logs['bbox_image'] = cond_img + return logs + +class LatentInpaintDiffusion(LatentDiffusion): + def __init__( + self, + concat_keys=("mask", "masked_image"), + masked_image_key="masked_image", + finetune_keys=None, + *args, + **kwargs, + ): + super().__init__(*args, **kwargs) + self.masked_image_key = masked_image_key + assert self.masked_image_key in concat_keys + self.concat_keys = concat_keys + + + @torch.no_grad() + def get_input( + self, batch, k, cond_key=None, bs=None, return_first_stage_outputs=False + ): + # note: restricted to non-trainable encoders currently + assert ( + not self.cond_stage_trainable + ), "trainable cond stages not yet supported for inpainting" + z, c, x, xrec, xc = super().get_input( + batch, + self.first_stage_key, + return_first_stage_outputs=True, + force_c_encode=True, + return_original_cond=True, + bs=bs, + ) + + assert exists(self.concat_keys) + c_cat = list() + for ck in self.concat_keys: + cc = ( + rearrange(batch[ck], "b h w c -> b c h w") + .to(memory_format=torch.contiguous_format) + .float() + ) + if bs is not None: + cc = cc[:bs] + cc = cc.to(self.device) + bchw = z.shape + if ck != self.masked_image_key: + cc = torch.nn.functional.interpolate(cc, size=bchw[-2:]) + else: + cc = self.get_first_stage_encoding(self.encode_first_stage(cc)) + c_cat.append(cc) + c_cat = torch.cat(c_cat, dim=1) + all_conds = {"c_concat": [c_cat], "c_crossattn": [c]} + if return_first_stage_outputs: + return z, all_conds, x, xrec, xc + return z, all_conds diff --git a/invokeai/models/diffusion/ksampler.py b/invokeai/models/diffusion/ksampler.py new file mode 100644 index 0000000000..f98ca8de21 --- /dev/null +++ b/invokeai/models/diffusion/ksampler.py @@ -0,0 +1,312 @@ +"""wrapper around part of Katherine Crowson's k-diffusion library, making it call compatible with other Samplers""" + +import k_diffusion as K +import torch +from torch import nn + +from .cross_attention_map_saving import AttentionMapSaver +from .sampler import Sampler +from .shared_invokeai_diffusion import InvokeAIDiffuserComponent + + +# at this threshold, the scheduler will stop using the Karras +# noise schedule and start using the model's schedule +STEP_THRESHOLD = 30 + +def cfg_apply_threshold(result, threshold = 0.0, scale = 0.7): + if threshold <= 0.0: + return result + maxval = 0.0 + torch.max(result).cpu().numpy() + minval = 0.0 + torch.min(result).cpu().numpy() + if maxval < threshold and minval > -threshold: + return result + if maxval > threshold: + maxval = min(max(1, scale*maxval), threshold) + if minval < -threshold: + minval = max(min(-1, scale*minval), -threshold) + return torch.clamp(result, min=minval, max=maxval) + + +class CFGDenoiser(nn.Module): + def __init__(self, model, threshold = 0, warmup = 0): + super().__init__() + self.inner_model = model + self.threshold = threshold + self.warmup_max = warmup + self.warmup = max(warmup / 10, 1) + self.invokeai_diffuser = InvokeAIDiffuserComponent(model, + model_forward_callback=lambda x, sigma, cond: self.inner_model(x, sigma, cond=cond)) + + + def prepare_to_sample(self, t_enc, **kwargs): + + extra_conditioning_info = kwargs.get('extra_conditioning_info', None) + + if extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control: + self.invokeai_diffuser.override_cross_attention(extra_conditioning_info, step_count = t_enc) + else: + self.invokeai_diffuser.restore_default_cross_attention() + + + def forward(self, x, sigma, uncond, cond, cond_scale): + next_x = self.invokeai_diffuser.do_diffusion_step(x, sigma, uncond, cond, cond_scale) + if self.warmup < self.warmup_max: + thresh = max(1, 1 + (self.threshold - 1) * (self.warmup / self.warmup_max)) + self.warmup += 1 + else: + thresh = self.threshold + if thresh > self.threshold: + thresh = self.threshold + return cfg_apply_threshold(next_x, thresh) + +class KSampler(Sampler): + def __init__(self, model, schedule='lms', device=None, **kwargs): + denoiser = K.external.CompVisDenoiser(model) + super().__init__( + denoiser, + schedule, + steps=model.num_timesteps, + ) + self.sigmas = None + self.ds = None + self.s_in = None + self.karras_max = kwargs.get('karras_max',STEP_THRESHOLD) + if self.karras_max is None: + self.karras_max = STEP_THRESHOLD + + def make_schedule( + self, + ddim_num_steps, + ddim_discretize='uniform', + ddim_eta=0.0, + verbose=False, + ): + outer_model = self.model + self.model = outer_model.inner_model + super().make_schedule( + ddim_num_steps, + ddim_discretize='uniform', + ddim_eta=0.0, + verbose=False, + ) + self.model = outer_model + self.ddim_num_steps = ddim_num_steps + # we don't need both of these sigmas, but storing them here to make + # comparison easier later on + self.model_sigmas = self.model.get_sigmas(ddim_num_steps) + self.karras_sigmas = K.sampling.get_sigmas_karras( + n=ddim_num_steps, + sigma_min=self.model.sigmas[0].item(), + sigma_max=self.model.sigmas[-1].item(), + rho=7., + device=self.device, + ) + + if ddim_num_steps >= self.karras_max: + print(f'>> Ksampler using model noise schedule (steps >= {self.karras_max})') + self.sigmas = self.model_sigmas + else: + print(f'>> Ksampler using karras noise schedule (steps < {self.karras_max})') + self.sigmas = self.karras_sigmas + + # ALERT: We are completely overriding the sample() method in the base class, which + # means that inpainting will not work. To get this to work we need to be able to + # modify the inner loop of k_heun, k_lms, etc, as is done in an ugly way + # in the lstein/k-diffusion branch. + + @torch.no_grad() + def decode( + self, + z_enc, + cond, + t_enc, + img_callback=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + use_original_steps=False, + init_latent = None, + mask = None, + **kwargs + ): + samples,_ = self.sample( + batch_size = 1, + S = t_enc, + x_T = z_enc, + shape = z_enc.shape[1:], + conditioning = cond, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning = unconditional_conditioning, + img_callback = img_callback, + x0 = init_latent, + mask = mask, + **kwargs + ) + return samples + + # this is a no-op, provided here for compatibility with ddim and plms samplers + @torch.no_grad() + def stochastic_encode(self, x0, t, use_original_steps=False, noise=None): + return x0 + + # Most of these arguments are ignored and are only present for compatibility with + # other samples + @torch.no_grad() + def sample( + self, + S, + batch_size, + shape, + conditioning=None, + callback=None, + normals_sequence=None, + img_callback=None, + attention_maps_callback=None, + quantize_x0=False, + eta=0.0, + mask=None, + x0=None, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + verbose=True, + x_T=None, + log_every_t=100, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + extra_conditioning_info: InvokeAIDiffuserComponent.ExtraConditioningInfo=None, + threshold = 0, + perlin = 0, + # this has to come in the same format as the conditioning, # e.g. as encoded tokens, ... + **kwargs, + ): + def route_callback(k_callback_values): + if img_callback is not None: + img_callback(k_callback_values['x'],k_callback_values['i']) + + # if make_schedule() hasn't been called, we do it now + if self.sigmas is None: + self.make_schedule( + ddim_num_steps=S, + ddim_eta = eta, + verbose = False, + ) + + # sigmas are set up in make_schedule - we take the last steps items + sigmas = self.sigmas[-S-1:] + + # x_T is variation noise. When an init image is provided (in x0) we need to add + # more randomness to the starting image. + if x_T is not None: + if x0 is not None: + x = x_T + torch.randn_like(x0, device=self.device) * sigmas[0] + else: + x = x_T * sigmas[0] + else: + x = torch.randn([batch_size, *shape], device=self.device) * sigmas[0] + + model_wrap_cfg = CFGDenoiser(self.model, threshold=threshold, warmup=max(0.8*S,S-10)) + model_wrap_cfg.prepare_to_sample(S, extra_conditioning_info=extra_conditioning_info) + + # setup attention maps saving. checks for None are because there are multiple code paths to get here. + attention_map_saver = None + if attention_maps_callback is not None and extra_conditioning_info is not None: + eos_token_index = extra_conditioning_info.tokens_count_including_eos_bos - 1 + attention_map_token_ids = range(1, eos_token_index) + attention_map_saver = AttentionMapSaver(token_ids = attention_map_token_ids, latents_shape=x.shape[-2:]) + model_wrap_cfg.invokeai_diffuser.setup_attention_map_saving(attention_map_saver) + + extra_args = { + 'cond': conditioning, + 'uncond': unconditional_conditioning, + 'cond_scale': unconditional_guidance_scale, + } + print(f'>> Sampling with k_{self.schedule} starting at step {len(self.sigmas)-S-1} of {len(self.sigmas)-1} ({S} new sampling steps)') + sampling_result = ( + K.sampling.__dict__[f'sample_{self.schedule}']( + model_wrap_cfg, x, sigmas, extra_args=extra_args, + callback=route_callback + ), + None, + ) + if attention_map_saver is not None: + attention_maps_callback(attention_map_saver) + return sampling_result + + # this code will support inpainting if and when ksampler API modified or + # a workaround is found. + @torch.no_grad() + def p_sample( + self, + img, + cond, + ts, + index, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + extra_conditioning_info=None, + **kwargs, + ): + if self.model_wrap is None: + self.model_wrap = CFGDenoiser(self.model) + extra_args = { + 'cond': cond, + 'uncond': unconditional_conditioning, + 'cond_scale': unconditional_guidance_scale, + } + if self.s_in is None: + self.s_in = img.new_ones([img.shape[0]]) + if self.ds is None: + self.ds = [] + + # terrible, confusing names here + steps = self.ddim_num_steps + t_enc = self.t_enc + + # sigmas is a full steps in length, but t_enc might + # be less. We start in the middle of the sigma array + # and work our way to the end after t_enc steps. + # index starts at t_enc and works its way to zero, + # so the actual formula for indexing into sigmas: + # sigma_index = (steps-index) + s_index = t_enc - index - 1 + self.model_wrap.prepare_to_sample(s_index, extra_conditioning_info=extra_conditioning_info) + img = K.sampling.__dict__[f'_{self.schedule}']( + self.model_wrap, + img, + self.sigmas, + s_index, + s_in = self.s_in, + ds = self.ds, + extra_args=extra_args, + ) + + return img, None, None + + # REVIEW THIS METHOD: it has never been tested. In particular, + # we should not be multiplying by self.sigmas[0] if we + # are at an intermediate step in img2img. See similar in + # sample() which does work. + def get_initial_image(self,x_T,shape,steps): + print(f'WARNING: ksampler.get_initial_image(): get_initial_image needs testing') + x = (torch.randn(shape, device=self.device) * self.sigmas[0]) + if x_T is not None: + return x_T + x + else: + return x + + def prepare_to_sample(self,t_enc,**kwargs): + self.t_enc = t_enc + self.model_wrap = None + self.ds = None + self.s_in = None + + def q_sample(self,x0,ts): + ''' + Overrides parent method to return the q_sample of the inner model. + ''' + return self.model.inner_model.q_sample(x0,ts) + + def conditioning_key(self)->str: + return self.model.inner_model.model.conditioning_key + diff --git a/invokeai/models/diffusion/plms.py b/invokeai/models/diffusion/plms.py new file mode 100644 index 0000000000..4df703bed5 --- /dev/null +++ b/invokeai/models/diffusion/plms.py @@ -0,0 +1,146 @@ +"""SAMPLING ONLY.""" + +import torch +import numpy as np +from tqdm import tqdm +from functools import partial +from ldm.invoke.devices import choose_torch_device +from .shared_invokeai_diffusion import InvokeAIDiffuserComponent +from .sampler import Sampler +from ldm.modules.diffusionmodules.util import noise_like + + +class PLMSSampler(Sampler): + def __init__(self, model, schedule='linear', device=None, **kwargs): + super().__init__(model,schedule,model.num_timesteps, device) + + def prepare_to_sample(self, t_enc, **kwargs): + super().prepare_to_sample(t_enc, **kwargs) + + extra_conditioning_info = kwargs.get('extra_conditioning_info', None) + all_timesteps_count = kwargs.get('all_timesteps_count', t_enc) + + if extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control: + self.invokeai_diffuser.override_cross_attention(extra_conditioning_info, step_count = all_timesteps_count) + else: + self.invokeai_diffuser.restore_default_cross_attention() + + + # this is the essential routine + @torch.no_grad() + def p_sample( + self, + x, # image, called 'img' elsewhere + c, # conditioning, called 'cond' elsewhere + t, # timesteps, called 'ts' elsewhere + index, + repeat_noise=False, + use_original_steps=False, + quantize_denoised=False, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + old_eps=[], + t_next=None, + step_count:int=1000, # total number of steps + **kwargs, + ): + b, *_, device = *x.shape, x.device + + def get_model_output(x, t): + if ( + unconditional_conditioning is None + or unconditional_guidance_scale == 1.0 + ): + # damian0815 would like to know when/if this code path is used + e_t = self.model.apply_model(x, t, c) + else: + # step_index counts in the opposite direction to index + step_index = step_count-(index+1) + e_t = self.invokeai_diffuser.do_diffusion_step(x, t, + unconditional_conditioning, c, + unconditional_guidance_scale, + step_index=step_index) + if score_corrector is not None: + assert self.model.parameterization == 'eps' + e_t = score_corrector.modify_score( + self.model, e_t, x, t, c, **corrector_kwargs + ) + + return e_t + + alphas = ( + self.model.alphas_cumprod + if use_original_steps + else self.ddim_alphas + ) + alphas_prev = ( + self.model.alphas_cumprod_prev + if use_original_steps + else self.ddim_alphas_prev + ) + sqrt_one_minus_alphas = ( + self.model.sqrt_one_minus_alphas_cumprod + if use_original_steps + else self.ddim_sqrt_one_minus_alphas + ) + sigmas = ( + self.model.ddim_sigmas_for_original_num_steps + if use_original_steps + else self.ddim_sigmas + ) + + def get_x_prev_and_pred_x0(e_t, index): + # select parameters corresponding to the currently considered timestep + a_t = torch.full((b, 1, 1, 1), alphas[index], device=device) + a_prev = torch.full( + (b, 1, 1, 1), alphas_prev[index], device=device + ) + sigma_t = torch.full((b, 1, 1, 1), sigmas[index], device=device) + sqrt_one_minus_at = torch.full( + (b, 1, 1, 1), sqrt_one_minus_alphas[index], device=device + ) + + # current prediction for x_0 + pred_x0 = (x - sqrt_one_minus_at * e_t) / a_t.sqrt() + if quantize_denoised: + pred_x0, _, *_ = self.model.first_stage_model.quantize(pred_x0) + # direction pointing to x_t + dir_xt = (1.0 - a_prev - sigma_t**2).sqrt() * e_t + noise = ( + sigma_t + * noise_like(x.shape, device, repeat_noise) + * temperature + ) + if noise_dropout > 0.0: + noise = torch.nn.functional.dropout(noise, p=noise_dropout) + x_prev = a_prev.sqrt() * pred_x0 + dir_xt + noise + return x_prev, pred_x0 + + e_t = get_model_output(x, t) + if len(old_eps) == 0: + # Pseudo Improved Euler (2nd order) + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t, index) + e_t_next = get_model_output(x_prev, t_next) + e_t_prime = (e_t + e_t_next) / 2 + elif len(old_eps) == 1: + # 2nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (3 * e_t - old_eps[-1]) / 2 + elif len(old_eps) == 2: + # 3nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = (23 * e_t - 16 * old_eps[-1] + 5 * old_eps[-2]) / 12 + elif len(old_eps) >= 3: + # 4nd order Pseudo Linear Multistep (Adams-Bashforth) + e_t_prime = ( + 55 * e_t + - 59 * old_eps[-1] + + 37 * old_eps[-2] + - 9 * old_eps[-3] + ) / 24 + + x_prev, pred_x0 = get_x_prev_and_pred_x0(e_t_prime, index) + + return x_prev, pred_x0, e_t diff --git a/invokeai/models/diffusion/sampler.py b/invokeai/models/diffusion/sampler.py new file mode 100644 index 0000000000..29479ff15f --- /dev/null +++ b/invokeai/models/diffusion/sampler.py @@ -0,0 +1,450 @@ +''' +invokeai.models.diffusion.sampler + +Base class for invokeai.models.diffusion.ddim, invokeai.models.diffusion.ksampler, etc +''' +import torch +import numpy as np +from tqdm import tqdm +from functools import partial +from ldm.invoke.devices import choose_torch_device +from .shared_invokeai_diffusion import InvokeAIDiffuserComponent + +from ldm.modules.diffusionmodules.util import ( + make_ddim_sampling_parameters, + make_ddim_timesteps, + noise_like, + extract_into_tensor, +) + +class Sampler(object): + def __init__(self, model, schedule='linear', steps=None, device=None, **kwargs): + self.model = model + self.ddim_timesteps = None + self.ddpm_num_timesteps = steps + self.schedule = schedule + self.device = device or choose_torch_device() + self.invokeai_diffuser = InvokeAIDiffuserComponent(self.model, + model_forward_callback = lambda x, sigma, cond: self.model.apply_model(x, sigma, cond)) + + def register_buffer(self, name, attr): + if type(attr) == torch.Tensor: + if attr.device != torch.device(self.device): + attr = attr.to(torch.float32).to(torch.device(self.device)) + setattr(self, name, attr) + + # This method was copied over from ddim.py and probably does stuff that is + # ddim-specific. Disentangle at some point. + def make_schedule( + self, + ddim_num_steps, + ddim_discretize='uniform', + ddim_eta=0.0, + verbose=False, + ): + self.total_steps = ddim_num_steps + self.ddim_timesteps = make_ddim_timesteps( + ddim_discr_method=ddim_discretize, + num_ddim_timesteps=ddim_num_steps, + num_ddpm_timesteps=self.ddpm_num_timesteps, + verbose=verbose, + ) + alphas_cumprod = self.model.alphas_cumprod + assert ( + alphas_cumprod.shape[0] == self.ddpm_num_timesteps + ), 'alphas have to be defined for each timestep' + to_torch = ( + lambda x: x.clone() + .detach() + .to(torch.float32) + .to(self.model.device) + ) + + self.register_buffer('betas', to_torch(self.model.betas)) + self.register_buffer('alphas_cumprod', to_torch(alphas_cumprod)) + self.register_buffer( + 'alphas_cumprod_prev', to_torch(self.model.alphas_cumprod_prev) + ) + + # calculations for diffusion q(x_t | x_{t-1}) and others + self.register_buffer( + 'sqrt_alphas_cumprod', to_torch(np.sqrt(alphas_cumprod.cpu())) + ) + self.register_buffer( + 'sqrt_one_minus_alphas_cumprod', + to_torch(np.sqrt(1.0 - alphas_cumprod.cpu())), + ) + self.register_buffer( + 'log_one_minus_alphas_cumprod', + to_torch(np.log(1.0 - alphas_cumprod.cpu())), + ) + self.register_buffer( + 'sqrt_recip_alphas_cumprod', + to_torch(np.sqrt(1.0 / alphas_cumprod.cpu())), + ) + self.register_buffer( + 'sqrt_recipm1_alphas_cumprod', + to_torch(np.sqrt(1.0 / alphas_cumprod.cpu() - 1)), + ) + + # ddim sampling parameters + ( + ddim_sigmas, + ddim_alphas, + ddim_alphas_prev, + ) = make_ddim_sampling_parameters( + alphacums=alphas_cumprod.cpu(), + ddim_timesteps=self.ddim_timesteps, + eta=ddim_eta, + verbose=verbose, + ) + self.register_buffer('ddim_sigmas', ddim_sigmas) + self.register_buffer('ddim_alphas', ddim_alphas) + self.register_buffer('ddim_alphas_prev', ddim_alphas_prev) + self.register_buffer( + 'ddim_sqrt_one_minus_alphas', np.sqrt(1.0 - ddim_alphas) + ) + sigmas_for_original_sampling_steps = ddim_eta * torch.sqrt( + (1 - self.alphas_cumprod_prev) + / (1 - self.alphas_cumprod) + * (1 - self.alphas_cumprod / self.alphas_cumprod_prev) + ) + self.register_buffer( + 'ddim_sigmas_for_original_num_steps', + sigmas_for_original_sampling_steps, + ) + + @torch.no_grad() + def stochastic_encode(self, x0, t, use_original_steps=False, noise=None): + # fast, but does not allow for exact reconstruction + # t serves as an index to gather the correct alphas + if use_original_steps: + sqrt_alphas_cumprod = self.sqrt_alphas_cumprod + sqrt_one_minus_alphas_cumprod = self.sqrt_one_minus_alphas_cumprod + else: + sqrt_alphas_cumprod = torch.sqrt(self.ddim_alphas) + sqrt_one_minus_alphas_cumprod = self.ddim_sqrt_one_minus_alphas + + if noise is None: + noise = torch.randn_like(x0) + return ( + extract_into_tensor(sqrt_alphas_cumprod, t, x0.shape) * x0 + + extract_into_tensor(sqrt_one_minus_alphas_cumprod, t, x0.shape) + * noise + ) + + @torch.no_grad() + def sample( + self, + S, # S is steps + batch_size, + shape, + conditioning=None, + callback=None, + normals_sequence=None, + img_callback=None, # TODO: this is very confusing because it is called "step_callback" elsewhere. Change. + quantize_x0=False, + eta=0.0, + mask=None, + x0=None, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + verbose=False, + x_T=None, + log_every_t=100, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + # this has to come in the same format as the conditioning, # e.g. as encoded tokens, ... + **kwargs, + ): + + if conditioning is not None: + if isinstance(conditioning, dict): + ctmp = conditioning[list(conditioning.keys())[0]] + while isinstance(ctmp, list): + ctmp = ctmp[0] + cbs = ctmp.shape[0] + if cbs != batch_size: + print(f"Warning: Got {cbs} conditionings but batch-size is {batch_size}") + else: + if conditioning.shape[0] != batch_size: + print(f"Warning: Got {conditioning.shape[0]} conditionings but batch-size is {batch_size}") + + # check to see if make_schedule() has run, and if not, run it + if self.ddim_timesteps is None: + self.make_schedule( + ddim_num_steps=S, + ddim_eta = eta, + verbose = False, + ) + + ts = self.get_timesteps(S) + + # sampling + C, H, W = shape + shape = (batch_size, C, H, W) + samples, intermediates = self.do_sampling( + conditioning, + shape, + timesteps=ts, + callback=callback, + img_callback=img_callback, + quantize_denoised=quantize_x0, + mask=mask, + x0=x0, + ddim_use_original_steps=False, + noise_dropout=noise_dropout, + temperature=temperature, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + x_T=x_T, + log_every_t=log_every_t, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + steps=S, + **kwargs + ) + return samples, intermediates + + @torch.no_grad() + def do_sampling( + self, + cond, + shape, + timesteps=None, + x_T=None, + ddim_use_original_steps=False, + callback=None, + quantize_denoised=False, + mask=None, + x0=None, + img_callback=None, + log_every_t=100, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + steps=None, + **kwargs + ): + b = shape[0] + time_range = ( + list(reversed(range(0, timesteps))) + if ddim_use_original_steps + else np.flip(timesteps) + ) + + total_steps=steps + + iterator = tqdm( + time_range, + desc=f'{self.__class__.__name__}', + total=total_steps, + dynamic_ncols=True, + ) + old_eps = [] + self.prepare_to_sample(t_enc=total_steps,all_timesteps_count=steps,**kwargs) + img = self.get_initial_image(x_T,shape,total_steps) + + # probably don't need this at all + intermediates = {'x_inter': [img], 'pred_x0': [img]} + + for i, step in enumerate(iterator): + index = total_steps - i - 1 + ts = torch.full( + (b,), + step, + device=self.device, + dtype=torch.long + ) + ts_next = torch.full( + (b,), + time_range[min(i + 1, len(time_range) - 1)], + device=self.device, + dtype=torch.long, + ) + + if mask is not None: + assert x0 is not None + img_orig = self.model.q_sample( + x0, ts + ) # TODO: deterministic forward pass? + img = img_orig * mask + (1.0 - mask) * img + + outs = self.p_sample( + img, + cond, + ts, + index=index, + use_original_steps=ddim_use_original_steps, + quantize_denoised=quantize_denoised, + temperature=temperature, + noise_dropout=noise_dropout, + score_corrector=score_corrector, + corrector_kwargs=corrector_kwargs, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + old_eps=old_eps, + t_next=ts_next, + step_count=steps + ) + img, pred_x0, e_t = outs + + old_eps.append(e_t) + if len(old_eps) >= 4: + old_eps.pop(0) + if callback: + callback(i) + if img_callback: + img_callback(img,i) + + if index % log_every_t == 0 or index == total_steps - 1: + intermediates['x_inter'].append(img) + intermediates['pred_x0'].append(pred_x0) + + return img, intermediates + + # NOTE that decode() and sample() are almost the same code, and do the same thing. + # The variable names are changed in order to be confusing. + @torch.no_grad() + def decode( + self, + x_latent, + cond, + t_start, + img_callback=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + use_original_steps=False, + init_latent = None, + mask = None, + all_timesteps_count = None, + **kwargs + ): + timesteps = ( + np.arange(self.ddpm_num_timesteps) + if use_original_steps + else self.ddim_timesteps + ) + timesteps = timesteps[:t_start] + + time_range = np.flip(timesteps) + total_steps = timesteps.shape[0] + print(f'>> Running {self.__class__.__name__} sampling starting at step {self.total_steps - t_start} of {self.total_steps} ({total_steps} new sampling steps)') + + iterator = tqdm(time_range, desc='Decoding image', total=total_steps) + x_dec = x_latent + x0 = init_latent + self.prepare_to_sample(t_enc=total_steps, all_timesteps_count=all_timesteps_count, **kwargs) + + for i, step in enumerate(iterator): + index = total_steps - i - 1 + ts = torch.full( + (x_latent.shape[0],), + step, + device=x_latent.device, + dtype=torch.long, + ) + + ts_next = torch.full( + (x_latent.shape[0],), + time_range[min(i + 1, len(time_range) - 1)], + device=self.device, + dtype=torch.long, + ) + + if mask is not None: + assert x0 is not None + xdec_orig = self.q_sample(x0, ts) # TODO: deterministic forward pass? + x_dec = xdec_orig * mask + (1.0 - mask) * x_dec + + outs = self.p_sample( + x_dec, + cond, + ts, + index=index, + use_original_steps=use_original_steps, + unconditional_guidance_scale=unconditional_guidance_scale, + unconditional_conditioning=unconditional_conditioning, + t_next = ts_next, + step_count=len(self.ddim_timesteps) + ) + + x_dec, pred_x0, e_t = outs + if img_callback: + img_callback(x_dec,i) + + return x_dec + + def get_initial_image(self,x_T,shape,timesteps=None): + if x_T is None: + return torch.randn(shape, device=self.device) + else: + return x_T + + def p_sample( + self, + img, + cond, + ts, + index, + repeat_noise=False, + use_original_steps=False, + quantize_denoised=False, + temperature=1.0, + noise_dropout=0.0, + score_corrector=None, + corrector_kwargs=None, + unconditional_guidance_scale=1.0, + unconditional_conditioning=None, + old_eps=None, + t_next=None, + steps=None, + ): + raise NotImplementedError("p_sample() must be implemented in a descendent class") + + def prepare_to_sample(self,t_enc,**kwargs): + ''' + Hook that will be called right before the very first invocation of p_sample() + to allow subclass to do additional initialization. t_enc corresponds to the actual + number of steps that will be run, and may be less than total steps if img2img is + active. + ''' + pass + + def get_timesteps(self,ddim_steps): + ''' + The ddim and plms samplers work on timesteps. This method is called after + ddim_timesteps are created in make_schedule(), and selects the portion of + timesteps that will be used for sampling, depending on the t_enc in img2img. + ''' + return self.ddim_timesteps[:ddim_steps] + + def q_sample(self,x0,ts): + ''' + Returns self.model.q_sample(x0,ts). Is overridden in the k* samplers to + return self.model.inner_model.q_sample(x0,ts) + ''' + return self.model.q_sample(x0,ts) + + def conditioning_key(self)->str: + return self.model.model.conditioning_key + + def uses_inpainting_model(self)->bool: + return self.conditioning_key() in ('hybrid','concat') + + def adjust_settings(self,**kwargs): + ''' + This is a catch-all method for adjusting any instance variables + after the sampler is instantiated. No type-checking performed + here, so use with care! + ''' + for k in kwargs.keys(): + try: + setattr(self,k,kwargs[k]) + except AttributeError: + print(f'** Warning: attempt to set unknown attribute {k} in sampler of type {type(self)}') diff --git a/invokeai/models/diffusion/shared_invokeai_diffusion.py b/invokeai/models/diffusion/shared_invokeai_diffusion.py new file mode 100644 index 0000000000..32b978f704 --- /dev/null +++ b/invokeai/models/diffusion/shared_invokeai_diffusion.py @@ -0,0 +1,491 @@ +from contextlib import contextmanager +from dataclasses import dataclass +from math import ceil +from typing import Callable, Optional, Union, Any, Dict + +import numpy as np +import torch +from diffusers.models.cross_attention import AttnProcessor +from typing_extensions import TypeAlias + +from ldm.invoke.globals import Globals +from .cross_attention_control import Arguments, \ + restore_default_cross_attention, override_cross_attention, Context, get_cross_attention_modules, \ + CrossAttentionType, SwapCrossAttnContext +from .cross_attention_map_saving import AttentionMapSaver + +ModelForwardCallback: TypeAlias = Union[ + # x, t, conditioning, Optional[cross-attention kwargs] + Callable[[torch.Tensor, torch.Tensor, torch.Tensor, Optional[dict[str, Any]]], torch.Tensor], + Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor] +] + +@dataclass(frozen=True) +class PostprocessingSettings: + threshold: float + warmup: float + h_symmetry_time_pct: Optional[float] + v_symmetry_time_pct: Optional[float] + + +class InvokeAIDiffuserComponent: + ''' + The aim of this component is to provide a single place for code that can be applied identically to + all InvokeAI diffusion procedures. + + At the moment it includes the following features: + * Cross attention control ("prompt2prompt") + * Hybrid conditioning (used for inpainting) + ''' + debug_thresholding = False + sequential_guidance = False + + @dataclass + class ExtraConditioningInfo: + + tokens_count_including_eos_bos: int + cross_attention_control_args: Optional[Arguments] = None + + @property + def wants_cross_attention_control(self): + return self.cross_attention_control_args is not None + + + def __init__(self, model, model_forward_callback: ModelForwardCallback, + is_running_diffusers: bool=False, + ): + """ + :param model: the unet model to pass through to cross attention control + :param model_forward_callback: a lambda with arguments (x, sigma, conditioning_to_apply). will be called repeatedly. most likely, this should simply call model.forward(x, sigma, conditioning) + """ + self.conditioning = None + self.model = model + self.is_running_diffusers = is_running_diffusers + self.model_forward_callback = model_forward_callback + self.cross_attention_control_context = None + self.sequential_guidance = Globals.sequential_guidance + + @contextmanager + def custom_attention_context(self, + extra_conditioning_info: Optional[ExtraConditioningInfo], + step_count: int): + do_swap = extra_conditioning_info is not None and extra_conditioning_info.wants_cross_attention_control + old_attn_processor = None + if do_swap: + old_attn_processor = self.override_cross_attention(extra_conditioning_info, + step_count=step_count) + try: + yield None + finally: + if old_attn_processor is not None: + self.restore_default_cross_attention(old_attn_processor) + # TODO resuscitate attention map saving + #self.remove_attention_map_saving() + + def override_cross_attention(self, conditioning: ExtraConditioningInfo, step_count: int) -> Dict[str, AttnProcessor]: + """ + setup cross attention .swap control. for diffusers this replaces the attention processor, so + the previous attention processor is returned so that the caller can restore it later. + """ + self.conditioning = conditioning + self.cross_attention_control_context = Context( + arguments=self.conditioning.cross_attention_control_args, + step_count=step_count + ) + return override_cross_attention(self.model, + self.cross_attention_control_context, + is_running_diffusers=self.is_running_diffusers) + + def restore_default_cross_attention(self, restore_attention_processor: Optional['AttnProcessor']=None): + self.conditioning = None + self.cross_attention_control_context = None + restore_default_cross_attention(self.model, + is_running_diffusers=self.is_running_diffusers, + restore_attention_processor=restore_attention_processor) + + def setup_attention_map_saving(self, saver: AttentionMapSaver): + def callback(slice, dim, offset, slice_size, key): + if dim is not None: + # sliced tokens attention map saving is not implemented + return + saver.add_attention_maps(slice, key) + + tokens_cross_attention_modules = get_cross_attention_modules(self.model, CrossAttentionType.TOKENS) + for identifier, module in tokens_cross_attention_modules: + key = ('down' if identifier.startswith('down') else + 'up' if identifier.startswith('up') else + 'mid') + module.set_attention_slice_calculated_callback( + lambda slice, dim, offset, slice_size, key=key: callback(slice, dim, offset, slice_size, key)) + + def remove_attention_map_saving(self): + tokens_cross_attention_modules = get_cross_attention_modules(self.model, CrossAttentionType.TOKENS) + for _, module in tokens_cross_attention_modules: + module.set_attention_slice_calculated_callback(None) + + def do_diffusion_step(self, x: torch.Tensor, sigma: torch.Tensor, + unconditioning: Union[torch.Tensor,dict], + conditioning: Union[torch.Tensor,dict], + unconditional_guidance_scale: float, + step_index: Optional[int]=None, + total_step_count: Optional[int]=None, + ): + """ + :param x: current latents + :param sigma: aka t, passed to the internal model to control how much denoising will occur + :param unconditioning: embeddings for unconditioned output. for hybrid conditioning this is a dict of tensors [B x 77 x 768], otherwise a single tensor [B x 77 x 768] + :param conditioning: embeddings for conditioned output. for hybrid conditioning this is a dict of tensors [B x 77 x 768], otherwise a single tensor [B x 77 x 768] + :param unconditional_guidance_scale: aka CFG scale, controls how much effect the conditioning tensor has + :param step_index: counts upwards from 0 to (step_count-1) (as passed to setup_cross_attention_control, if using). May be called multiple times for a single step, therefore do not assume that its value will monotically increase. If None, will be estimated by comparing sigma against self.model.sigmas . + :return: the new latents after applying the model to x using unscaled unconditioning and CFG-scaled conditioning. + """ + + + cross_attention_control_types_to_do = [] + context: Context = self.cross_attention_control_context + if self.cross_attention_control_context is not None: + percent_through = self.calculate_percent_through(sigma, step_index, total_step_count) + cross_attention_control_types_to_do = context.get_active_cross_attention_control_types_for_step(percent_through) + + wants_cross_attention_control = (len(cross_attention_control_types_to_do) > 0) + wants_hybrid_conditioning = isinstance(conditioning, dict) + + if wants_hybrid_conditioning: + unconditioned_next_x, conditioned_next_x = self._apply_hybrid_conditioning(x, sigma, unconditioning, + conditioning) + elif wants_cross_attention_control: + unconditioned_next_x, conditioned_next_x = self._apply_cross_attention_controlled_conditioning(x, sigma, + unconditioning, + conditioning, + cross_attention_control_types_to_do) + elif self.sequential_guidance: + unconditioned_next_x, conditioned_next_x = self._apply_standard_conditioning_sequentially( + x, sigma, unconditioning, conditioning) + + else: + unconditioned_next_x, conditioned_next_x = self._apply_standard_conditioning( + x, sigma, unconditioning, conditioning) + + combined_next_x = self._combine(unconditioned_next_x, conditioned_next_x, unconditional_guidance_scale) + + return combined_next_x + + def do_latent_postprocessing( + self, + postprocessing_settings: PostprocessingSettings, + latents: torch.Tensor, + sigma, + step_index, + total_step_count + ) -> torch.Tensor: + if postprocessing_settings is not None: + percent_through = self.calculate_percent_through(sigma, step_index, total_step_count) + latents = self.apply_threshold(postprocessing_settings, latents, percent_through) + latents = self.apply_symmetry(postprocessing_settings, latents, percent_through) + return latents + + def calculate_percent_through(self, sigma, step_index, total_step_count): + if step_index is not None and total_step_count is not None: + # 🧨diffusers codepath + percent_through = step_index / total_step_count # will never reach 1.0 - this is deliberate + else: + # legacy compvis codepath + # TODO remove when compvis codepath support is dropped + if step_index is None and sigma is None: + raise ValueError( + f"Either step_index or sigma is required when doing cross attention control, but both are None.") + percent_through = self.estimate_percent_through(step_index, sigma) + return percent_through + + # methods below are called from do_diffusion_step and should be considered private to this class. + + def _apply_standard_conditioning(self, x, sigma, unconditioning, conditioning): + # fast batched path + x_twice = torch.cat([x] * 2) + sigma_twice = torch.cat([sigma] * 2) + both_conditionings = torch.cat([unconditioning, conditioning]) + both_results = self.model_forward_callback(x_twice, sigma_twice, both_conditionings) + unconditioned_next_x, conditioned_next_x = both_results.chunk(2) + if conditioned_next_x.device.type == 'mps': + # prevent a result filled with zeros. seems to be a torch bug. + conditioned_next_x = conditioned_next_x.clone() + return unconditioned_next_x, conditioned_next_x + + + def _apply_standard_conditioning_sequentially(self, x: torch.Tensor, sigma, unconditioning: torch.Tensor, conditioning: torch.Tensor): + # low-memory sequential path + unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning) + conditioned_next_x = self.model_forward_callback(x, sigma, conditioning) + if conditioned_next_x.device.type == 'mps': + # prevent a result filled with zeros. seems to be a torch bug. + conditioned_next_x = conditioned_next_x.clone() + return unconditioned_next_x, conditioned_next_x + + + def _apply_hybrid_conditioning(self, x, sigma, unconditioning, conditioning): + assert isinstance(conditioning, dict) + assert isinstance(unconditioning, dict) + x_twice = torch.cat([x] * 2) + sigma_twice = torch.cat([sigma] * 2) + both_conditionings = dict() + for k in conditioning: + if isinstance(conditioning[k], list): + both_conditionings[k] = [ + torch.cat([unconditioning[k][i], conditioning[k][i]]) + for i in range(len(conditioning[k])) + ] + else: + both_conditionings[k] = torch.cat([unconditioning[k], conditioning[k]]) + unconditioned_next_x, conditioned_next_x = self.model_forward_callback(x_twice, sigma_twice, both_conditionings).chunk(2) + return unconditioned_next_x, conditioned_next_x + + + def _apply_cross_attention_controlled_conditioning(self, + x: torch.Tensor, + sigma, + unconditioning, + conditioning, + cross_attention_control_types_to_do): + if self.is_running_diffusers: + return self._apply_cross_attention_controlled_conditioning__diffusers(x, sigma, unconditioning, + conditioning, + cross_attention_control_types_to_do) + else: + return self._apply_cross_attention_controlled_conditioning__compvis(x, sigma, unconditioning, conditioning, + cross_attention_control_types_to_do) + + def _apply_cross_attention_controlled_conditioning__diffusers(self, + x: torch.Tensor, + sigma, + unconditioning, + conditioning, + cross_attention_control_types_to_do): + context: Context = self.cross_attention_control_context + + cross_attn_processor_context = SwapCrossAttnContext(modified_text_embeddings=context.arguments.edited_conditioning, + index_map=context.cross_attention_index_map, + mask=context.cross_attention_mask, + cross_attention_types_to_do=[]) + # no cross attention for unconditioning (negative prompt) + unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning, + {"swap_cross_attn_context": cross_attn_processor_context}) + + # do requested cross attention types for conditioning (positive prompt) + cross_attn_processor_context.cross_attention_types_to_do = cross_attention_control_types_to_do + conditioned_next_x = self.model_forward_callback(x, sigma, conditioning, + {"swap_cross_attn_context": cross_attn_processor_context}) + return unconditioned_next_x, conditioned_next_x + + + def _apply_cross_attention_controlled_conditioning__compvis(self, x:torch.Tensor, sigma, unconditioning, conditioning, cross_attention_control_types_to_do): + # print('pct', percent_through, ': doing cross attention control on', cross_attention_control_types_to_do) + # slower non-batched path (20% slower on mac MPS) + # We are only interested in using attention maps for conditioned_next_x, but batching them with generation of + # unconditioned_next_x causes attention maps to *also* be saved for the unconditioned_next_x. + # This messes app their application later, due to mismatched shape of dim 0 (seems to be 16 for batched vs. 8) + # (For the batched invocation the `wrangler` function gets attention tensor with shape[0]=16, + # representing batched uncond + cond, but then when it comes to applying the saved attention, the + # wrangler gets an attention tensor which only has shape[0]=8, representing just self.edited_conditionings.) + # todo: give CrossAttentionControl's `wrangler` function more info so it can work with a batched call as well. + context:Context = self.cross_attention_control_context + + try: + unconditioned_next_x = self.model_forward_callback(x, sigma, unconditioning) + + # process x using the original prompt, saving the attention maps + #print("saving attention maps for", cross_attention_control_types_to_do) + for ca_type in cross_attention_control_types_to_do: + context.request_save_attention_maps(ca_type) + _ = self.model_forward_callback(x, sigma, conditioning) + context.clear_requests(cleanup=False) + + # process x again, using the saved attention maps to control where self.edited_conditioning will be applied + #print("applying saved attention maps for", cross_attention_control_types_to_do) + for ca_type in cross_attention_control_types_to_do: + context.request_apply_saved_attention_maps(ca_type) + edited_conditioning = self.conditioning.cross_attention_control_args.edited_conditioning + conditioned_next_x = self.model_forward_callback(x, sigma, edited_conditioning) + context.clear_requests(cleanup=True) + + except: + context.clear_requests(cleanup=True) + raise + + return unconditioned_next_x, conditioned_next_x + + def _combine(self, unconditioned_next_x, conditioned_next_x, guidance_scale): + # to scale how much effect conditioning has, calculate the changes it does and then scale that + scaled_delta = (conditioned_next_x - unconditioned_next_x) * guidance_scale + combined_next_x = unconditioned_next_x + scaled_delta + return combined_next_x + + def apply_threshold( + self, + postprocessing_settings: PostprocessingSettings, + latents: torch.Tensor, + percent_through: float + ) -> torch.Tensor: + + if postprocessing_settings.threshold is None or postprocessing_settings.threshold == 0.0: + return latents + + threshold = postprocessing_settings.threshold + warmup = postprocessing_settings.warmup + + if percent_through < warmup: + current_threshold = threshold + threshold * 5 * (1 - (percent_through / warmup)) + else: + current_threshold = threshold + + if current_threshold <= 0: + return latents + + maxval = latents.max().item() + minval = latents.min().item() + + scale = 0.7 # default value from #395 + + if self.debug_thresholding: + std, mean = [i.item() for i in torch.std_mean(latents)] + outside = torch.count_nonzero((latents < -current_threshold) | (latents > current_threshold)) + print(f"\nThreshold: %={percent_through} threshold={current_threshold:.3f} (of {threshold:.3f})\n" + f" | min, mean, max = {minval:.3f}, {mean:.3f}, {maxval:.3f}\tstd={std}\n" + f" | {outside / latents.numel() * 100:.2f}% values outside threshold") + + if maxval < current_threshold and minval > -current_threshold: + return latents + + num_altered = 0 + + # MPS torch.rand_like is fine because torch.rand_like is wrapped in generate.py! + + if maxval > current_threshold: + latents = torch.clone(latents) + maxval = np.clip(maxval * scale, 1, current_threshold) + num_altered += torch.count_nonzero(latents > maxval) + latents[latents > maxval] = torch.rand_like(latents[latents > maxval]) * maxval + + if minval < -current_threshold: + latents = torch.clone(latents) + minval = np.clip(minval * scale, -current_threshold, -1) + num_altered += torch.count_nonzero(latents < minval) + latents[latents < minval] = torch.rand_like(latents[latents < minval]) * minval + + if self.debug_thresholding: + print(f" | min, , max = {minval:.3f}, , {maxval:.3f}\t(scaled by {scale})\n" + f" | {num_altered / latents.numel() * 100:.2f}% values altered") + + return latents + + def apply_symmetry( + self, + postprocessing_settings: PostprocessingSettings, + latents: torch.Tensor, + percent_through: float + ) -> torch.Tensor: + + # Reset our last percent through if this is our first step. + if percent_through == 0.0: + self.last_percent_through = 0.0 + + if postprocessing_settings is None: + return latents + + # Check for out of bounds + h_symmetry_time_pct = postprocessing_settings.h_symmetry_time_pct + if (h_symmetry_time_pct is not None and (h_symmetry_time_pct <= 0.0 or h_symmetry_time_pct > 1.0)): + h_symmetry_time_pct = None + + v_symmetry_time_pct = postprocessing_settings.v_symmetry_time_pct + if (v_symmetry_time_pct is not None and (v_symmetry_time_pct <= 0.0 or v_symmetry_time_pct > 1.0)): + v_symmetry_time_pct = None + + dev = latents.device.type + + latents.to(device='cpu') + + if ( + h_symmetry_time_pct != None and + self.last_percent_through < h_symmetry_time_pct and + percent_through >= h_symmetry_time_pct + ): + # Horizontal symmetry occurs on the 3rd dimension of the latent + width = latents.shape[3] + x_flipped = torch.flip(latents, dims=[3]) + latents = torch.cat([latents[:, :, :, 0:int(width/2)], x_flipped[:, :, :, int(width/2):int(width)]], dim=3) + + if ( + v_symmetry_time_pct != None and + self.last_percent_through < v_symmetry_time_pct and + percent_through >= v_symmetry_time_pct + ): + # Vertical symmetry occurs on the 2nd dimension of the latent + height = latents.shape[2] + y_flipped = torch.flip(latents, dims=[2]) + latents = torch.cat([latents[:, :, 0:int(height / 2)], y_flipped[:, :, int(height / 2):int(height)]], dim=2) + + self.last_percent_through = percent_through + return latents.to(device=dev) + + def estimate_percent_through(self, step_index, sigma): + if step_index is not None and self.cross_attention_control_context is not None: + # percent_through will never reach 1.0 (but this is intended) + return float(step_index) / float(self.cross_attention_control_context.step_count) + # find the best possible index of the current sigma in the sigma sequence + smaller_sigmas = torch.nonzero(self.model.sigmas <= sigma) + sigma_index = smaller_sigmas[-1].item() if smaller_sigmas.shape[0] > 0 else 0 + # flip because sigmas[0] is for the fully denoised image + # percent_through must be <1 + return 1.0 - float(sigma_index + 1) / float(self.model.sigmas.shape[0]) + # print('estimated percent_through', percent_through, 'from sigma', sigma.item()) + + + # todo: make this work + @classmethod + def apply_conjunction(cls, x, t, forward_func, uc, c_or_weighted_c_list, global_guidance_scale): + x_in = torch.cat([x] * 2) + t_in = torch.cat([t] * 2) # aka sigmas + + deltas = None + uncond_latents = None + weighted_cond_list = c_or_weighted_c_list if type(c_or_weighted_c_list) is list else [(c_or_weighted_c_list, 1)] + + # below is fugly omg + num_actual_conditionings = len(c_or_weighted_c_list) + conditionings = [uc] + [c for c,weight in weighted_cond_list] + weights = [1] + [weight for c,weight in weighted_cond_list] + chunk_count = ceil(len(conditionings)/2) + deltas = None + for chunk_index in range(chunk_count): + offset = chunk_index*2 + chunk_size = min(2, len(conditionings)-offset) + + if chunk_size == 1: + c_in = conditionings[offset] + latents_a = forward_func(x_in[:-1], t_in[:-1], c_in) + latents_b = None + else: + c_in = torch.cat(conditionings[offset:offset+2]) + latents_a, latents_b = forward_func(x_in, t_in, c_in).chunk(2) + + # first chunk is guaranteed to be 2 entries: uncond_latents + first conditioining + if chunk_index == 0: + uncond_latents = latents_a + deltas = latents_b - uncond_latents + else: + deltas = torch.cat((deltas, latents_a - uncond_latents)) + if latents_b is not None: + deltas = torch.cat((deltas, latents_b - uncond_latents)) + + # merge the weighted deltas together into a single merged delta + per_delta_weights = torch.tensor(weights[1:], dtype=deltas.dtype, device=deltas.device) + normalize = False + if normalize: + per_delta_weights /= torch.sum(per_delta_weights) + reshaped_weights = per_delta_weights.reshape(per_delta_weights.shape + (1, 1, 1)) + deltas_merged = torch.sum(deltas * reshaped_weights, dim=0, keepdim=True) + + # old_return_value = super().forward(x, sigma, uncond, cond, cond_scale) + # assert(0 == len(torch.nonzero(old_return_value - (uncond_latents + deltas_merged * cond_scale)))) + + return uncond_latents + deltas_merged * global_guidance_scale diff --git a/invokeai/models/model_manager.py b/invokeai/models/model_manager.py new file mode 100644 index 0000000000..2a0a8ec933 --- /dev/null +++ b/invokeai/models/model_manager.py @@ -0,0 +1,1221 @@ +""" +Manage a cache of Stable Diffusion model files for fast switching. +They are moved between GPU and CPU as necessary. If CPU memory falls +below a preset minimum, the least recently used model will be +cleared and loaded from disk when next needed. +""" +from __future__ import annotations + +import contextlib +import gc +import hashlib +import os +import re +import sys +import textwrap +import time +import warnings +from enum import Enum +from pathlib import Path +from shutil import move, rmtree +from typing import Any, Optional, Union + +import safetensors +import safetensors.torch +import torch +import transformers +from diffusers import AutoencoderKL +from diffusers import logging as dlogging +from huggingface_hub import scan_cache_dir +from omegaconf import OmegaConf +from omegaconf.dictconfig import DictConfig +from picklescan.scanner import scan_file_path + +from ldm.invoke.devices import CPU_DEVICE +from ..generator.diffusers_pipeline import StableDiffusionGeneratorPipeline +from ldm.invoke.globals import Globals, global_cache_dir +from ldm.util import ( + ask_user, + download_with_resume, + url_attachment_name, +) + + +class SDLegacyType(Enum): + V1 = 1 + V1_INPAINT = 2 + V2 = 3 + UNKNOWN = 99 + +DEFAULT_MAX_MODELS = 2 +VAE_TO_REPO_ID = { # hack, see note in convert_and_import() + "vae-ft-mse-840000-ema-pruned": "stabilityai/sd-vae-ft-mse", +} + +class ModelManager(object): + def __init__( + self, + config: OmegaConf, + device_type: torch.device = CPU_DEVICE, + precision: str = "float16", + max_loaded_models=DEFAULT_MAX_MODELS, + sequential_offload=False, + ): + """ + Initialize with the path to the models.yaml config file, + the torch device type, and precision. The optional + min_avail_mem argument specifies how much unused system + (CPU) memory to preserve. The cache of models in RAM will + grow until this value is approached. Default is 2G. + """ + # prevent nasty-looking CLIP log message + transformers.logging.set_verbosity_error() + self.config = config + self.precision = precision + self.device = torch.device(device_type) + self.max_loaded_models = max_loaded_models + self.models = {} + self.stack = [] # this is an LRU FIFO + self.current_model = None + self.sequential_offload = sequential_offload + + def valid_model(self, model_name: str) -> bool: + """ + Given a model name, returns True if it is a valid + identifier. + """ + return model_name in self.config + + def get_model(self, model_name: str): + """ + Given a model named identified in models.yaml, return + the model object. If in RAM will load into GPU VRAM. + If on disk, will load from there. + """ + if not self.valid_model(model_name): + print( + f'** "{model_name}" is not a known model name. Please check your models.yaml file' + ) + return self.current_model + + if self.current_model != model_name: + if model_name not in self.models: # make room for a new one + self._make_cache_room() + self.offload_model(self.current_model) + + if model_name in self.models: + requested_model = self.models[model_name]["model"] + print(f">> Retrieving model {model_name} from system RAM cache") + self.models[model_name]["model"] = self._model_from_cpu(requested_model) + width = self.models[model_name]["width"] + height = self.models[model_name]["height"] + hash = self.models[model_name]["hash"] + + else: # we're about to load a new model, so potentially offload the least recently used one + requested_model, width, height, hash = self._load_model(model_name) + self.models[model_name] = { + "model": requested_model, + "width": width, + "height": height, + "hash": hash, + } + + self.current_model = model_name + self._push_newest_model(model_name) + return { + "model": requested_model, + "width": width, + "height": height, + "hash": hash, + } + + def default_model(self) -> str | None: + """ + Returns the name of the default model, or None + if none is defined. + """ + for model_name in self.config: + if self.config[model_name].get("default"): + return model_name + return list(self.config.keys())[0] # first one + + def set_default_model(self, model_name: str) -> None: + """ + Set the default model. The change will not take + effect until you call model_manager.commit() + """ + assert model_name in self.model_names(), f"unknown model '{model_name}'" + + config = self.config + for model in config: + config[model].pop("default", None) + config[model_name]["default"] = True + + def model_info(self, model_name: str) -> dict: + """ + Given a model name returns the OmegaConf (dict-like) object describing it. + """ + if model_name not in self.config: + return None + return self.config[model_name] + + def model_names(self) -> list[str]: + """ + Return a list consisting of all the names of models defined in models.yaml + """ + return list(self.config.keys()) + + def is_legacy(self, model_name: str) -> bool: + """ + Return true if this is a legacy (.ckpt) model + """ + # if we are converting legacy files automatically, then + # there are no legacy ckpts! + if Globals.ckpt_convert: + return False + info = self.model_info(model_name) + if "weights" in info and info["weights"].endswith((".ckpt", ".safetensors")): + return True + return False + + def list_models(self) -> dict: + """ + Return a dict of models in the format: + { model_name1: {'status': ('active'|'cached'|'not loaded'), + 'description': description, + 'format': ('ckpt'|'diffusers'|'vae'), + }, + model_name2: { etc } + Please use model_manager.models() to get all the model names, + model_manager.model_info('model-name') to get the stanza for the model + named 'model-name', and model_manager.config to get the full OmegaConf + object derived from models.yaml + """ + models = {} + for name in sorted(self.config, key=str.casefold): + stanza = self.config[name] + + # don't include VAEs in listing (legacy style) + if "config" in stanza and "/VAE/" in stanza["config"]: + continue + + models[name] = dict() + format = stanza.get("format", "ckpt") # Determine Format + + # Common Attribs + description = stanza.get("description", None) + if self.current_model == name: + status = "active" + elif name in self.models: + status = "cached" + else: + status = "not loaded" + models[name].update( + description=description, + format=format, + status=status, + ) + + # Checkpoint Config Parse + if format == "ckpt": + models[name].update( + config=str(stanza.get("config", None)), + weights=str(stanza.get("weights", None)), + vae=str(stanza.get("vae", None)), + width=str(stanza.get("width", 512)), + height=str(stanza.get("height", 512)), + ) + + # Diffusers Config Parse + if vae := stanza.get("vae", None): + if isinstance(vae, DictConfig): + vae = dict( + repo_id=str(vae.get("repo_id", None)), + path=str(vae.get("path", None)), + subfolder=str(vae.get("subfolder", None)), + ) + + if format == "diffusers": + models[name].update( + vae=vae, + repo_id=str(stanza.get("repo_id", None)), + path=str(stanza.get("path", None)), + ) + + return models + + def print_models(self) -> None: + """ + Print a table of models, their descriptions, and load status + """ + models = self.list_models() + for name in models: + if models[name]["format"] == "vae": + continue + line = f'{name:25s} {models[name]["status"]:>10s} {models[name]["format"]:10s} {models[name]["description"]}' + if models[name]["status"] == "active": + line = f"\033[1m{line}\033[0m" + print(line) + + def del_model(self, model_name: str, delete_files: bool = False) -> None: + """ + Delete the named model. + """ + omega = self.config + if model_name not in omega: + print(f"** Unknown model {model_name}") + return + # save these for use in deletion later + conf = omega[model_name] + repo_id = conf.get("repo_id", None) + path = self._abs_path(conf.get("path", None)) + weights = self._abs_path(conf.get("weights", None)) + + del omega[model_name] + if model_name in self.stack: + self.stack.remove(model_name) + if delete_files: + if weights: + print(f"** deleting file {weights}") + Path(weights).unlink(missing_ok=True) + elif path: + print(f"** deleting directory {path}") + rmtree(path, ignore_errors=True) + elif repo_id: + print(f"** deleting the cached model directory for {repo_id}") + self._delete_model_from_cache(repo_id) + + def add_model( + self, model_name: str, model_attributes: dict, clobber: bool = False + ) -> None: + """ + Update the named model with a dictionary of attributes. Will fail with an + assertion error if the name already exists. Pass clobber=True to overwrite. + On a successful update, the config will be changed in memory and the + method will return True. Will fail with an assertion error if provided + attributes are incorrect or the model name is missing. + """ + omega = self.config + assert "format" in model_attributes, 'missing required field "format"' + if model_attributes["format"] == "diffusers": + assert ( + "description" in model_attributes + ), 'required field "description" is missing' + assert ( + "path" in model_attributes or "repo_id" in model_attributes + ), 'model must have either the "path" or "repo_id" fields defined' + else: + for field in ("description", "weights", "height", "width", "config"): + assert field in model_attributes, f"required field {field} is missing" + + assert ( + clobber or model_name not in omega + ), f'attempt to overwrite existing model definition "{model_name}"' + + omega[model_name] = model_attributes + + if "weights" in omega[model_name]: + omega[model_name]["weights"].replace("\\", "/") + + if clobber: + self._invalidate_cached_model(model_name) + + def _load_model(self, model_name: str): + """Load and initialize the model from configuration variables passed at object creation time""" + if model_name not in self.config: + print( + f'"{model_name}" is not a known model name. Please check your models.yaml file' + ) + return + + mconfig = self.config[model_name] + + # for usage statistics + if self._has_cuda(): + torch.cuda.reset_peak_memory_stats() + torch.cuda.empty_cache() + + tic = time.time() + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + model, width, height, model_hash = self._load_diffusers_model(mconfig) + + # usage statistics + toc = time.time() + print(">> Model loaded in", "%4.2fs" % (toc - tic)) + if self._has_cuda(): + print( + ">> Max VRAM used to load the model:", + "%4.2fG" % (torch.cuda.max_memory_allocated() / 1e9), + "\n>> Current VRAM usage:" + "%4.2fG" % (torch.cuda.memory_allocated() / 1e9), + ) + return model, width, height, model_hash + + def _load_diffusers_model(self, mconfig): + name_or_path = self.model_name_or_path(mconfig) + using_fp16 = self.precision == "float16" + + print(f">> Loading diffusers model from {name_or_path}") + if using_fp16: + print(" | Using faster float16 precision") + else: + print(" | Using more accurate float32 precision") + + # TODO: scan weights maybe? + pipeline_args: dict[str, Any] = dict( + safety_checker=None, local_files_only=not Globals.internet_available + ) + if "vae" in mconfig and mconfig["vae"] is not None: + if vae := self._load_vae(mconfig["vae"]): + pipeline_args.update(vae=vae) + if not isinstance(name_or_path, Path): + pipeline_args.update(cache_dir=global_cache_dir("diffusers")) + if using_fp16: + pipeline_args.update(torch_dtype=torch.float16) + fp_args_list = [{"revision": "fp16"}, {}] + else: + fp_args_list = [{}] + + verbosity = dlogging.get_verbosity() + dlogging.set_verbosity_error() + + pipeline = None + for fp_args in fp_args_list: + try: + pipeline = StableDiffusionGeneratorPipeline.from_pretrained( + name_or_path, + **pipeline_args, + **fp_args, + ) + except OSError as e: + if str(e).startswith("fp16 is not a valid"): + pass + else: + print( + f"** An unexpected error occurred while downloading the model: {e})" + ) + if pipeline: + break + + dlogging.set_verbosity(verbosity) + assert pipeline is not None, OSError(f'"{name_or_path}" could not be loaded') + + if self.sequential_offload: + pipeline.enable_offload_submodels(self.device) + else: + pipeline.to(self.device) + + model_hash = self._diffuser_sha256(name_or_path) + + # square images??? + width = pipeline.unet.config.sample_size * pipeline.vae_scale_factor + height = width + + print(f" | Default image dimensions = {width} x {height}") + + return pipeline, width, height, model_hash + + def model_name_or_path(self, model_name: Union[str, DictConfig]) -> str | Path: + if isinstance(model_name, DictConfig) or isinstance(model_name, dict): + mconfig = model_name + elif model_name in self.config: + mconfig = self.config[model_name] + else: + raise ValueError( + f'"{model_name}" is not a known model name. Please check your models.yaml file' + ) + + if "path" in mconfig and mconfig["path"] is not None: + path = Path(mconfig["path"]) + if not path.is_absolute(): + path = Path(Globals.root, path).resolve() + return path + elif "repo_id" in mconfig: + return mconfig["repo_id"] + else: + raise ValueError("Model config must specify either repo_id or path.") + + def offload_model(self, model_name: str) -> None: + """ + Offload the indicated model to CPU. Will call + _make_cache_room() to free space if needed. + """ + if model_name not in self.models: + return + + print(f">> Offloading {model_name} to CPU") + model = self.models[model_name]["model"] + self.models[model_name]["model"] = self._model_to_cpu(model) + + gc.collect() + if self._has_cuda(): + torch.cuda.empty_cache() + + def scan_model(self, model_name, checkpoint): + """ + Apply picklescanner to the indicated checkpoint and issue a warning + and option to exit if an infected file is identified. + """ + # scan model + print(f">> Scanning Model: {model_name}") + scan_result = scan_file_path(checkpoint) + if scan_result.infected_files != 0: + if scan_result.infected_files == 1: + print(f"\n### Issues Found In Model: {scan_result.issues_count}") + print( + "### WARNING: The model you are trying to load seems to be infected." + ) + print("### For your safety, InvokeAI will not load this model.") + print("### Please use checkpoints from trusted sources.") + print("### Exiting InvokeAI") + sys.exit() + else: + print( + "\n### WARNING: InvokeAI was unable to scan the model you are using." + ) + model_safe_check_fail = ask_user( + "Do you want to to continue loading the model?", ["y", "n"] + ) + if model_safe_check_fail.lower() != "y": + print("### Exiting InvokeAI") + sys.exit() + else: + print(">> Model scanned ok") + + def import_diffuser_model( + self, + repo_or_path: Union[str, Path], + model_name: str = None, + model_description: str = None, + vae: dict = None, + commit_to_conf: Path = None, + ) -> bool: + """ + Attempts to install the indicated diffuser model and returns True if successful. + + "repo_or_path" can be either a repo-id or a path-like object corresponding to the + top of a downloaded diffusers directory. + + You can optionally provide a model name and/or description. If not provided, + then these will be derived from the repo name. If you provide a commit_to_conf + path to the configuration file, then the new entry will be committed to the + models.yaml file. + """ + model_name = model_name or Path(repo_or_path).stem + model_description = model_description or f"Imported diffusers model {model_name}" + new_config = dict( + description=model_description, + vae=vae, + format="diffusers", + ) + if isinstance(repo_or_path, Path) and repo_or_path.exists(): + new_config.update(path=str(repo_or_path)) + else: + new_config.update(repo_id=repo_or_path) + + self.add_model(model_name, new_config, True) + if commit_to_conf: + self.commit(commit_to_conf) + return model_name + + def import_ckpt_model( + self, + weights: Union[str, Path], + config: Union[str, Path] = "configs/stable-diffusion/v1-inference.yaml", + vae: Union[str, Path] = None, + model_name: str = None, + model_description: str = None, + commit_to_conf: Path = None, + ) -> str: + """ + Attempts to install the indicated ckpt file and returns True if successful. + + "weights" can be either a path-like object corresponding to a local .ckpt file + or a http/https URL pointing to a remote model. + + "vae" is a Path or str object pointing to a ckpt or safetensors file to be used + as the VAE for this model. + + "config" is the model config file to use with this ckpt file. It defaults to + v1-inference.yaml. If a URL is provided, the config will be downloaded. + + You can optionally provide a model name and/or description. If not provided, + then these will be derived from the weight file name. If you provide a commit_to_conf + path to the configuration file, then the new entry will be committed to the + models.yaml file. + + Return value is the name of the imported file, or None if an error occurred. + """ + if str(weights).startswith(("http:", "https:")): + model_name = model_name or url_attachment_name(weights) + + weights_path = self._resolve_path(weights, "models/ldm/stable-diffusion-v1") + config_path = self._resolve_path(config, "configs/stable-diffusion") + + if weights_path is None or not weights_path.exists(): + return + if config_path is None or not config_path.exists(): + return + + model_name = ( + model_name or Path(weights).stem + ) # note this gives ugly pathnames if used on a URL without a Content-Disposition header + model_description = ( + model_description or f"Imported stable diffusion weights file {model_name}" + ) + new_config = dict( + weights=str(weights_path), + config=str(config_path), + description=model_description, + format="ckpt", + width=512, + height=512, + ) + if vae: + new_config["vae"] = vae + self.add_model(model_name, new_config, True) + if commit_to_conf: + self.commit(commit_to_conf) + return model_name + + @classmethod + def probe_model_type(self, checkpoint: dict) -> SDLegacyType: + """ + Given a pickle or safetensors model object, probes contents + of the object and returns an SDLegacyType indicating its + format. Valid return values include: + SDLegacyType.V1 + SDLegacyType.V1_INPAINT + SDLegacyType.V2 + SDLegacyType.UNKNOWN + """ + key_name = "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weight" + if key_name in checkpoint and checkpoint[key_name].shape[-1] == 1024: + return SDLegacyType.V2 + + try: + state_dict = checkpoint.get("state_dict") or checkpoint + in_channels = state_dict[ + "model.diffusion_model.input_blocks.0.0.weight" + ].shape[1] + if in_channels == 9: + return SDLegacyType.V1_INPAINT + elif in_channels == 4: + return SDLegacyType.V1 + else: + return SDLegacyType.UNKNOWN + except KeyError: + return SDLegacyType.UNKNOWN + + def heuristic_import( + self, + path_url_or_repo: str, + convert: bool = True, + model_name: str = None, + description: str = None, + commit_to_conf: Path = None, + ) -> str: + """ + Accept a string which could be: + - a HF diffusers repo_id + - a URL pointing to a legacy .ckpt or .safetensors file + - a local path pointing to a legacy .ckpt or .safetensors file + - a local directory containing .ckpt and .safetensors files + - a local directory containing a diffusers model + + After determining the nature of the model and downloading it + (if necessary), the file is probed to determine the correct + configuration file (if needed) and it is imported. + + The model_name and/or description can be provided. If not, they will + be generated automatically. + + If convert is true, legacy models will be converted to diffusers + before importing. + + If commit_to_conf is provided, the newly loaded model will be written + to the `models.yaml` file at the indicated path. Otherwise, the changes + will only remain in memory. + + The (potentially derived) name of the model is returned on success, or None + on failure. When multiple models are added from a directory, only the last + imported one is returned. + """ + model_path: Path = None + thing = path_url_or_repo # to save typing + + print(f">> Probing {thing} for import") + + if thing.startswith(("http:", "https:", "ftp:")): + print(f" | {thing} appears to be a URL") + model_path = self._resolve_path( + thing, "models/ldm/stable-diffusion-v1" + ) # _resolve_path does a download if needed + + elif Path(thing).is_file() and thing.endswith((".ckpt", ".safetensors")): + if Path(thing).stem in ["model", "diffusion_pytorch_model"]: + print( + f" | {Path(thing).name} appears to be part of a diffusers model. Skipping import" + ) + return + else: + print(f" | {thing} appears to be a checkpoint file on disk") + model_path = self._resolve_path(thing, "models/ldm/stable-diffusion-v1") + + elif Path(thing).is_dir() and Path(thing, "model_index.json").exists(): + print(f" | {thing} appears to be a diffusers file on disk") + model_name = self.import_diffuser_model( + thing, + vae=dict(repo_id="stabilityai/sd-vae-ft-mse"), + model_name=model_name, + description=description, + commit_to_conf=commit_to_conf, + ) + + elif Path(thing).is_dir(): + if (Path(thing) / "model_index.json").exists(): + print(f" | {thing} appears to be a diffusers model.") + model_name = self.import_diffuser_model( + thing, commit_to_conf=commit_to_conf + ) + else: + print( + f" |{thing} appears to be a directory. Will scan for models to import" + ) + for m in list(Path(thing).rglob("*.ckpt")) + list( + Path(thing).rglob("*.safetensors") + ): + if model_name := self.heuristic_import( + str(m), convert, commit_to_conf=commit_to_conf + ): + print(f" >> {model_name} successfully imported") + return model_name + + elif re.match(r"^[\w.+-]+/[\w.+-]+$", thing): + print(f" | {thing} appears to be a HuggingFace diffusers repo_id") + model_name = self.import_diffuser_model( + thing, commit_to_conf=commit_to_conf + ) + pipeline, _, _, _ = self._load_diffusers_model(self.config[model_name]) + return model_name + else: + print( + f"** {thing}: Unknown thing. Please provide a URL, file path, directory or HuggingFace repo_id" + ) + + # Model_path is set in the event of a legacy checkpoint file. + # If not set, we're all done + if not model_path: + return + + if model_path.stem in self.config: # already imported + print(" | Already imported. Skipping") + return + + # another round of heuristics to guess the correct config file. + checkpoint = ( + safetensors.torch.load_file(model_path) + if model_path.suffix == ".safetensors" + else torch.load(model_path) + ) + model_type = self.probe_model_type(checkpoint) + + model_config_file = None + if model_type == SDLegacyType.V1: + print(" | SD-v1 model detected") + model_config_file = Path( + Globals.root, "configs/stable-diffusion/v1-inference.yaml" + ) + elif model_type == SDLegacyType.V1_INPAINT: + print(" | SD-v1 inpainting model detected") + model_config_file = Path( + Globals.root, "configs/stable-diffusion/v1-inpainting-inference.yaml" + ) + elif model_type == SDLegacyType.V2: + print( + " | SD-v2 model detected; model will be converted to diffusers format" + ) + model_config_file = Path( + Globals.root, "configs/stable-diffusion/v2-inference-v.yaml" + ) + convert = True + else: + print( + f"** {thing} is a legacy checkpoint file but not in a known Stable Diffusion model. Skipping import" + ) + return + + diffuser_path = Path( + Globals.root, "models", Globals.converted_ckpts_dir, model_path.stem + ) + model_name = self.convert_and_import( + model_path, + diffusers_path=diffuser_path, + vae=dict(repo_id="stabilityai/sd-vae-ft-mse"), + model_name=model_name, + model_description=description, + original_config_file=model_config_file, + commit_to_conf=commit_to_conf, + ) + if commit_to_conf: + self.commit(commit_to_conf) + return model_name + + def convert_and_import( + self, + ckpt_path: Path, + diffusers_path: Path, + model_name=None, + model_description=None, + vae=None, + original_config_file: Path = None, + commit_to_conf: Path = None, + ) -> str: + """ + Convert a legacy ckpt weights file to diffuser model and import + into models.yaml. + """ + ckpt_path = self._resolve_path(ckpt_path, "models/ldm/stable-diffusion-v1") + if original_config_file: + original_config_file = self._resolve_path( + original_config_file, "configs/stable-diffusion" + ) + + new_config = None + + from ldm.invoke.ckpt_to_diffuser import convert_ckpt_to_diffuser + + if diffusers_path.exists(): + print( + f"ERROR: The path {str(diffusers_path)} already exists. Please move or remove it and try again." + ) + return + + model_name = model_name or diffusers_path.name + model_description = model_description or f"Optimized version of {model_name}" + print(f">> Optimizing {model_name} (30-60s)") + try: + # By passing the specified VAE to the conversion function, the autoencoder + # will be built into the model rather than tacked on afterward via the config file + vae_model = self._load_vae(vae) if vae else None + convert_ckpt_to_diffuser( + ckpt_path, + diffusers_path, + extract_ema=True, + original_config_file=original_config_file, + vae=vae_model, + ) + print( + f" | Success. Optimized model is now located at {str(diffusers_path)}" + ) + print(f" | Writing new config file entry for {model_name}") + new_config = dict( + path=str(diffusers_path), + description=model_description, + format="diffusers", + ) + if model_name in self.config: + self.del_model(model_name) + self.add_model(model_name, new_config, True) + if commit_to_conf: + self.commit(commit_to_conf) + print(">> Conversion succeeded") + except Exception as e: + print(f"** Conversion failed: {str(e)}") + print( + "** If you are trying to convert an inpainting or 2.X model, please indicate the correct config file (e.g. v1-inpainting-inference.yaml)" + ) + + return model_name + + def search_models(self, search_folder): + print(f">> Finding Models In: {search_folder}") + models_folder_ckpt = Path(search_folder).glob("**/*.ckpt") + models_folder_safetensors = Path(search_folder).glob("**/*.safetensors") + + ckpt_files = [x for x in models_folder_ckpt if x.is_file()] + safetensor_files = [x for x in models_folder_safetensors if x.is_file()] + + files = ckpt_files + safetensor_files + + found_models = [] + for file in files: + location = str(file.resolve()).replace("\\", "/") + if ( + "model.safetensors" not in location + and "diffusion_pytorch_model.safetensors" not in location + ): + found_models.append({"name": file.stem, "location": location}) + + return search_folder, found_models + + def _choose_diffusers_vae( + self, model_name: str, vae: str = None + ) -> Union[dict, str]: + # In the event that the original entry is using a custom ckpt VAE, we try to + # map that VAE onto a diffuser VAE using a hard-coded dictionary. + # I would prefer to do this differently: We load the ckpt model into memory, swap the + # VAE in memory, and then pass that to convert_ckpt_to_diffuser() so that the swapped + # VAE is built into the model. However, when I tried this I got obscure key errors. + if vae: + return vae + if model_name in self.config and ( + vae_ckpt_path := self.model_info(model_name).get("vae", None) + ): + vae_basename = Path(vae_ckpt_path).stem + diffusers_vae = None + if diffusers_vae := VAE_TO_REPO_ID.get(vae_basename, None): + print( + f">> {vae_basename} VAE corresponds to known {diffusers_vae} diffusers version" + ) + vae = {"repo_id": diffusers_vae} + else: + print( + f'** Custom VAE "{vae_basename}" found, but corresponding diffusers model unknown' + ) + print( + '** Using "stabilityai/sd-vae-ft-mse"; If this isn\'t right, please edit the model config' + ) + vae = {"repo_id": "stabilityai/sd-vae-ft-mse"} + return vae + + def _make_cache_room(self) -> None: + num_loaded_models = len(self.models) + if num_loaded_models >= self.max_loaded_models: + least_recent_model = self._pop_oldest_model() + print( + f">> Cache limit (max={self.max_loaded_models}) reached. Purging {least_recent_model}" + ) + if least_recent_model is not None: + del self.models[least_recent_model] + gc.collect() + + def print_vram_usage(self) -> None: + if self._has_cuda: + print( + ">> Current VRAM usage: ", + "%4.2fG" % (torch.cuda.memory_allocated() / 1e9), + ) + + def commit(self, config_file_path: str) -> None: + """ + Write current configuration out to the indicated file. + """ + yaml_str = OmegaConf.to_yaml(self.config) + if not os.path.isabs(config_file_path): + config_file_path = os.path.normpath( + os.path.join(Globals.root, config_file_path) + ) + tmpfile = os.path.join(os.path.dirname(config_file_path), "new_config.tmp") + with open(tmpfile, "w", encoding="utf-8") as outfile: + outfile.write(self.preamble()) + outfile.write(yaml_str) + os.replace(tmpfile, config_file_path) + + def preamble(self) -> str: + """ + Returns the preamble for the config file. + """ + return textwrap.dedent( + """\ + # This file describes the alternative machine learning models + # available to InvokeAI script. + # + # To add a new model, follow the examples below. Each + # model requires a model config file, a weights file, + # and the width and height of the images it + # was trained on. + """ + ) + + @classmethod + def migrate_models(cls): + """ + Migrate the ~/invokeai/models directory from the legacy format used through 2.2.5 + to the 2.3.0 "diffusers" version. This should be a one-time operation, called at + script startup time. + """ + # Three transformer models to check: bert, clip and safety checker + legacy_locations = [ + Path( + "CompVis/stable-diffusion-safety-checker/models--CompVis--stable-diffusion-safety-checker" + ), + Path("bert-base-uncased/models--bert-base-uncased"), + Path( + "openai/clip-vit-large-patch14/models--openai--clip-vit-large-patch14" + ), + ] + models_dir = Path(Globals.root, "models") + legacy_layout = False + for model in legacy_locations: + legacy_layout = legacy_layout or Path(models_dir, model).exists() + if not legacy_layout: + return + + print( + "** Legacy version <= 2.2.5 model directory layout detected. Reorganizing." + ) + print("** This is a quick one-time operation.") + + # transformer files get moved into the hub directory + if cls._is_huggingface_hub_directory_present(): + hub = global_cache_dir("hub") + else: + hub = models_dir / "hub" + + os.makedirs(hub, exist_ok=True) + for model in legacy_locations: + source = models_dir / model + dest = hub / model.stem + print(f"** {source} => {dest}") + if source.exists(): + if dest.exists(): + rmtree(source) + else: + move(source, dest) + + # anything else gets moved into the diffusers directory + if cls._is_huggingface_hub_directory_present(): + diffusers = global_cache_dir("diffusers") + else: + diffusers = models_dir / "diffusers" + + os.makedirs(diffusers, exist_ok=True) + for root, dirs, _ in os.walk(models_dir, topdown=False): + for dir in dirs: + full_path = Path(root, dir) + if full_path.is_relative_to(hub) or full_path.is_relative_to(diffusers): + continue + if Path(dir).match("models--*--*"): + dest = diffusers / dir + print(f"** {full_path} => {dest}") + if dest.exists(): + rmtree(full_path) + else: + move(full_path, dest) + + # now clean up by removing any empty directories + empty = [ + root + for root, dirs, files, in os.walk(models_dir) + if not len(dirs) and not len(files) + ] + for d in empty: + os.rmdir(d) + print("** Migration is done. Continuing...") + + def _resolve_path( + self, source: Union[str, Path], dest_directory: str + ) -> Optional[Path]: + resolved_path = None + if str(source).startswith(("http:", "https:", "ftp:")): + dest_directory = Path(dest_directory) + if not dest_directory.is_absolute(): + dest_directory = Globals.root / dest_directory + dest_directory.mkdir(parents=True, exist_ok=True) + resolved_path = download_with_resume(str(source), dest_directory) + else: + if not os.path.isabs(source): + source = os.path.join(Globals.root, source) + resolved_path = Path(source) + return resolved_path + + def _invalidate_cached_model(self, model_name: str) -> None: + self.offload_model(model_name) + if model_name in self.stack: + self.stack.remove(model_name) + self.models.pop(model_name, None) + + def _model_to_cpu(self, model): + if self.device == CPU_DEVICE: + return model + + if isinstance(model, StableDiffusionGeneratorPipeline): + model.offload_all() + return model + + model.cond_stage_model.device = CPU_DEVICE + model.to(CPU_DEVICE) + + for submodel in ("first_stage_model", "cond_stage_model", "model"): + try: + getattr(model, submodel).to(CPU_DEVICE) + except AttributeError: + pass + return model + + def _model_from_cpu(self, model): + if self.device == CPU_DEVICE: + return model + + if isinstance(model, StableDiffusionGeneratorPipeline): + model.ready() + return model + + model.to(self.device) + model.cond_stage_model.device = self.device + + for submodel in ("first_stage_model", "cond_stage_model", "model"): + try: + getattr(model, submodel).to(self.device) + except AttributeError: + pass + + return model + + def _pop_oldest_model(self): + """ + Remove the first element of the FIFO, which ought + to be the least recently accessed model. Do not + pop the last one, because it is in active use! + """ + return self.stack.pop(0) + + def _push_newest_model(self, model_name: str) -> None: + """ + Maintain a simple FIFO. First element is always the + least recent, and last element is always the most recent. + """ + with contextlib.suppress(ValueError): + self.stack.remove(model_name) + self.stack.append(model_name) + + def _has_cuda(self) -> bool: + return self.device.type == "cuda" + + def _diffuser_sha256( + self, name_or_path: Union[str, Path], chunksize=4096 + ) -> Union[str, bytes]: + path = None + if isinstance(name_or_path, Path): + path = name_or_path + else: + owner, repo = name_or_path.split("/") + path = Path(global_cache_dir("diffusers") / f"models--{owner}--{repo}") + if not path.exists(): + return None + hashpath = path / "checksum.sha256" + if hashpath.exists() and path.stat().st_mtime <= hashpath.stat().st_mtime: + with open(hashpath) as f: + hash = f.read() + return hash + print(" | Calculating sha256 hash of model files") + tic = time.time() + sha = hashlib.sha256() + count = 0 + for root, dirs, files in os.walk(path, followlinks=False): + for name in files: + count += 1 + with open(os.path.join(root, name), "rb") as f: + while chunk := f.read(chunksize): + sha.update(chunk) + hash = sha.hexdigest() + toc = time.time() + print(f" | sha256 = {hash} ({count} files hashed in", "%4.2fs)" % (toc - tic)) + with open(hashpath, "w") as f: + f.write(hash) + return hash + + def _cached_sha256(self, path, data) -> Union[str, bytes]: + dirname = os.path.dirname(path) + basename = os.path.basename(path) + base, _ = os.path.splitext(basename) + hashpath = os.path.join(dirname, base + ".sha256") + + if os.path.exists(hashpath) and os.path.getmtime(path) <= os.path.getmtime( + hashpath + ): + with open(hashpath) as f: + hash = f.read() + return hash + + print(" | Calculating sha256 hash of weights file") + tic = time.time() + sha = hashlib.sha256() + sha.update(data) + hash = sha.hexdigest() + toc = time.time() + print(f">> sha256 = {hash}", "(%4.2fs)" % (toc - tic)) + + with open(hashpath, "w") as f: + f.write(hash) + return hash + + def _load_vae(self, vae_config) -> AutoencoderKL: + vae_args = {} + try: + name_or_path = self.model_name_or_path(vae_config) + except Exception: + return None + if name_or_path is None: + return None + using_fp16 = self.precision == "float16" + + vae_args.update( + cache_dir=global_cache_dir("diffusers"), + local_files_only=not Globals.internet_available, + ) + + print(f" | Loading diffusers VAE from {name_or_path}") + if using_fp16: + vae_args.update(torch_dtype=torch.float16) + fp_args_list = [{"revision": "fp16"}, {}] + else: + print(" | Using more accurate float32 precision") + fp_args_list = [{}] + + vae = None + deferred_error = None + + # A VAE may be in a subfolder of a model's repository. + if "subfolder" in vae_config: + vae_args["subfolder"] = vae_config["subfolder"] + + for fp_args in fp_args_list: + # At some point we might need to be able to use different classes here? But for now I think + # all Stable Diffusion VAE are AutoencoderKL. + try: + vae = AutoencoderKL.from_pretrained(name_or_path, **vae_args, **fp_args) + except OSError as e: + if str(e).startswith("fp16 is not a valid"): + pass + else: + deferred_error = e + if vae: + break + + if not vae and deferred_error: + print(f"** Could not load VAE {name_or_path}: {str(deferred_error)}") + + return vae + + @staticmethod + def _delete_model_from_cache(repo_id): + cache_info = scan_cache_dir(global_cache_dir("diffusers")) + + # I'm sure there is a way to do this with comprehensions + # but the code quickly became incomprehensible! + hashes_to_delete = set() + for repo in cache_info.repos: + if repo.repo_id == repo_id: + for revision in repo.revisions: + hashes_to_delete.add(revision.commit_hash) + strategy = cache_info.delete_revisions(*hashes_to_delete) + print( + f"** deletion of this model is expected to free {strategy.expected_freed_size_str}" + ) + strategy.execute() + + @staticmethod + def _abs_path(path: str | Path) -> Path: + if path is None or Path(path).is_absolute(): + return path + return Path(Globals.root, path).resolve() + + @staticmethod + def _is_huggingface_hub_directory_present() -> bool: + return ( + os.getenv("HF_HOME") is not None or os.getenv("XDG_CACHE_HOME") is not None + )