Path->str in call to expand_prompts

- Previous PR to truncate long filenames won't work on windows due to
lack of support for os.pathconf(). This works around the limitation by
hardcoding the value for PC_NAME_MAX when pathconf is unavailable.
- The `multiprocessing` send() and recv() methods weren't working
properly on Windows due to issues involving `utf-8` encoding and
pickling/unpickling. Changed these calls to `send_bytes()` and
`recv_bytes()` , which seems to fix the issue.

Not fully tested on Windows since I lack a GPU machine to test on, but
is working on CPU.

ldm/invoke/CLI.py

@@ -17,6 +17,8 @@ if sys.platform == "darwin":
 
 import pyparsing  # type: ignore
 
+print(f'DEBUG: [1] All system modules imported', file=sys.stderr)
+
 import ldm.invoke
 
 from ..generate import Generate
@@ -31,13 +33,21 @@ from .pngwriter import PngWriter, retrieve_metadata, write_metadata
 from .readline import Completer, get_completer
 from ..util import url_attachment_name
 
+print(f'DEBUG: [2] All invokeai modules imported', file=sys.stderr)
+
 # global used in multiple functions (fix)
 infile = None
 
 def main():
     """Initialize command-line parsers and the diffusion model"""
     global infile
-
+    
+    print('DEBUG: [3] Entered main()', file=sys.stderr)
+    print('DEBUG: INVOKEAI ENVIRONMENT:')
+    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+    print("\n".join([f'{x}:{os.environ[x]}' for x in os.environ.keys()]))
+    print('>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>')
+    
     opt = Args()
     args = opt.parse_args()
     if not args:
@@ -66,9 +76,13 @@ def main():
     Globals.sequential_guidance = args.sequential_guidance
     Globals.ckpt_convert = True  # always true as of 2.3.4 for LoRA support
 
+    print(f'DEBUG: [4] Globals initialized', file=sys.stderr)
+
     # run any post-install patches needed
     run_patches()
 
+    print(f'DEBUG: [5] Patches run', file=sys.stderr)
+
     print(f">> Internet connectivity is {Globals.internet_available}")
 
     if not args.conf:
@@ -84,8 +98,9 @@ def main():
     # loading here to avoid long delays on startup
     # these two lines prevent a horrible warning message from appearing
     # when the frozen CLIP tokenizer is imported
+    print(f'DEBUG: [6] Importing torch modules', file=sys.stderr)
+    
     import transformers  # type: ignore
-
     from ldm.generate import Generate
 
     transformers.logging.set_verbosity_error()
@@ -93,6 +108,7 @@ def main():
 
     diffusers.logging.set_verbosity_error()
 
+    print(f'DEBUG: [7] loading restoration models', file=sys.stderr)
     # Loading Face Restoration and ESRGAN Modules
     gfpgan, codeformer, esrgan = load_face_restoration(opt)
 
@@ -114,6 +130,7 @@ def main():
         Globals.lora_models_dir = opt.lora_path
 
     # migrate legacy models
+    print(f'DEBUG: [8] migrating  models', file=sys.stderr)
     ModelManager.migrate_models()
 
     # load the infile as a list of lines
@@ -131,6 +148,7 @@ def main():
 
     model = opt.model or retrieve_last_used_model()
 
+    print(f'DEBUG: [9] Creating generate object', file=sys.stderr)
     # creating a Generate object:
     try:
         gen = Generate(
@@ -157,6 +175,7 @@ def main():
         print(">> changed to seamless tiling mode")
 
     # preload the model
+    print(f'DEBUG: [10] Loading default model', file=sys.stderr)
     try:
         gen.load_model()
     except KeyError:
@@ -204,6 +223,7 @@ def main():
 # TODO: main_loop() has gotten busy. Needs to be refactored.
 def main_loop(gen, opt, completer):
     """prompt/read/execute loop"""
+    print(f'DEBUG: [11] In main loop', file=sys.stderr)
     global infile
     done = False
     doneAfterInFile = infile is not None
@@ -1322,15 +1342,16 @@ def install_missing_config_files():
     install ckpt configuration files that may have been added to the
     distro after original root directory configuration
     """
-    import invokeai.configs as conf
-    from shutil import copyfile
+    pass
+    # import invokeai.configs as conf
+    # from shutil import copyfile
     
-    root_configs = Path(global_config_dir(), 'stable-diffusion')
-    repo_configs = Path(conf.__path__[0], 'stable-diffusion')
-    for src in repo_configs.iterdir():
-        dest = root_configs / src.name
-        if not dest.exists():
-            copyfile(src,dest)
+    # root_configs = Path(global_config_dir(), 'stable-diffusion')
+    # repo_configs = Path(conf.__path__[0], 'stable-diffusion')
+    # for src in repo_configs.iterdir():
+    #     dest = root_configs / src.name
+    #     if not dest.exists():
+    #         copyfile(src,dest)
     
 def do_version_update(root_version: version.Version, app_version: Union[str, version.Version]):
     """

ldm/invoke/conditioning.py

@@ -12,7 +12,8 @@ from typing import Union, Optional, Any
 from transformers import CLIPTokenizer
 
 from compel import Compel
-from compel.prompt_parser import FlattenedPrompt, Blend, Fragment, CrossAttentionControlSubstitute, PromptParser
+from compel.prompt_parser import FlattenedPrompt, Blend, Fragment, CrossAttentionControlSubstitute, PromptParser, \
+    Conjunction
 from .devices import torch_dtype
 from ..models.diffusion.shared_invokeai_diffusion import InvokeAIDiffuserComponent
 from ldm.invoke.globals import Globals
@@ -55,22 +56,25 @@ def get_uc_and_c_and_ec(prompt_string, model, log_tokens=False, skip_normalize_l
     # get rid of any newline characters
     prompt_string = prompt_string.replace("\n", " ")
     positive_prompt_string, negative_prompt_string = split_prompt_to_positive_and_negative(prompt_string)
+
     legacy_blend = try_parse_legacy_blend(positive_prompt_string, skip_normalize_legacy_blend)
-    positive_prompt: FlattenedPrompt|Blend
-    lora_conditions = None
+    positive_conjunction: Conjunction
     if legacy_blend is not None:
-        positive_prompt = legacy_blend
+        positive_conjunction = legacy_blend
     else:
         positive_conjunction = Compel.parse_prompt_string(positive_prompt_string)
-        positive_prompt = positive_conjunction.prompts[0]
-        should_use_lora_manager = True
-        lora_weights = positive_conjunction.lora_weights
-        if model.peft_manager:
-            should_use_lora_manager = model.peft_manager.should_use(lora_weights)
-            if not should_use_lora_manager:
-                model.peft_manager.set_loras(lora_weights)
-        if model.lora_manager and should_use_lora_manager:
-            lora_conditions = model.lora_manager.set_loras_conditions(lora_weights)
+    positive_prompt = positive_conjunction.prompts[0]
+
+    should_use_lora_manager = True
+    lora_weights = positive_conjunction.lora_weights
+    lora_conditions = None
+    if model.peft_manager:
+        should_use_lora_manager = model.peft_manager.should_use(lora_weights)
+        if not should_use_lora_manager:
+            model.peft_manager.set_loras(lora_weights)
+    if model.lora_manager and should_use_lora_manager:
+        lora_conditions = model.lora_manager.set_loras_conditions(lora_weights)
+
     negative_conjunction = Compel.parse_prompt_string(negative_prompt_string)
     negative_prompt: FlattenedPrompt | Blend = negative_conjunction.prompts[0]
 
@@ -93,12 +97,12 @@ def get_prompt_structure(prompt_string, skip_normalize_legacy_blend: bool = Fals
     Union[FlattenedPrompt, Blend], FlattenedPrompt):
     positive_prompt_string, negative_prompt_string = split_prompt_to_positive_and_negative(prompt_string)
     legacy_blend = try_parse_legacy_blend(positive_prompt_string, skip_normalize_legacy_blend)
-    positive_prompt: FlattenedPrompt|Blend
+    positive_conjunction: Conjunction
     if legacy_blend is not None:
-        positive_prompt = legacy_blend
+        positive_conjunction = legacy_blend
     else:
         positive_conjunction = Compel.parse_prompt_string(positive_prompt_string)
-        positive_prompt = positive_conjunction.prompts[0]
+    positive_prompt = positive_conjunction.prompts[0]
     negative_conjunction = Compel.parse_prompt_string(negative_prompt_string)
     negative_prompt: FlattenedPrompt|Blend = negative_conjunction.prompts[0]
 
@@ -217,18 +221,26 @@ def log_tokenization_for_text(text, tokenizer, display_label=None):
         print(f'{discarded}\x1b[0m')
 
 
-def try_parse_legacy_blend(text: str, skip_normalize: bool=False) -> Optional[Blend]:
+def try_parse_legacy_blend(text: str, skip_normalize: bool=False) -> Optional[Conjunction]:
     weighted_subprompts = split_weighted_subprompts(text, skip_normalize=skip_normalize)
     if len(weighted_subprompts) <= 1:
         return None
     strings = [x[0] for x in weighted_subprompts]
-    weights = [x[1] for x in weighted_subprompts]
 
     pp = PromptParser()
     parsed_conjunctions = [pp.parse_conjunction(x) for x in strings]
-    flattened_prompts = [x.prompts[0] for x in parsed_conjunctions]
+    flattened_prompts = []
+    weights = []
+    loras = []
+    for i, x in enumerate(parsed_conjunctions):
+        if len(x.prompts)>0:
+            flattened_prompts.append(x.prompts[0])
+            weights.append(weighted_subprompts[i][1])
+        if len(x.lora_weights)>0:
+            loras.extend(x.lora_weights)
 
-    return Blend(prompts=flattened_prompts, weights=weights, normalize_weights=not skip_normalize)
+    return Conjunction([Blend(prompts=flattened_prompts, weights=weights, normalize_weights=not skip_normalize)],
+                       lora_weights = loras)
 
 
 def split_weighted_subprompts(text, skip_normalize=False)->list:

ldm/invoke/config/invokeai_update.py

@@ -4,14 +4,13 @@ pip install <path_to_git_source>.
 '''
 import os
 import platform
+import psutil
 import requests
 from rich import box, print
-from rich.console import Console, Group, group
+from rich.console import Console, group
 from rich.panel import Panel
 from rich.prompt import Prompt
 from rich.style import Style
-from rich.syntax import Syntax
-from rich.text import Text
 
 from ldm.invoke import __version__
 
@@ -32,6 +31,19 @@ else:
 def get_versions()->dict:
     return requests.get(url=INVOKE_AI_REL).json()
 
+def invokeai_is_running()->bool:
+    for p in psutil.process_iter():
+        try:
+            cmdline = p.cmdline()
+            matches = [x for x in cmdline if x.endswith(('invokeai','invokeai.exe'))]
+            if matches:
+                print(f':exclamation: [bold red]An InvokeAI instance appears to be running as process {p.pid}[/red bold]')
+                return True
+        except psutil.AccessDenied:
+            continue
+    return False
+        
+    
 def welcome(versions: dict):
     
     @group()
@@ -62,6 +74,10 @@ def welcome(versions: dict):
 
 def main():
     versions = get_versions()
+    if invokeai_is_running():
+        print(f':exclamation: [bold red]Please terminate all running instances of InvokeAI before updating.[/red bold]')
+        return
+
     welcome(versions)
 
     tag = None

ldm/invoke/dynamic_prompts.py

@@ -32,7 +32,8 @@ def expand_prompts(
     template_file: Path,
     run_invoke: bool = False,
     invoke_model: str = None,
-    invoke_outdir: Path = None,
+    invoke_outdir: str = None,
+    invoke_root: str = None,
     processes_per_gpu: int = 1,
 ):
     """
@@ -61,6 +62,8 @@ def expand_prompts(
             invokeai_args = [shutil.which("invokeai"), "--from_file", "-"]
             if invoke_model:
                 invokeai_args.extend(("--model", invoke_model))
+            if invoke_root:
+                invokeai_args.extend(("--root", invoke_root))
             if invoke_outdir:
                 outdir = os.path.expanduser(invoke_outdir)
                 invokeai_args.extend(("--outdir", outdir))
@@ -79,6 +82,11 @@ def expand_prompts(
             )
             import ldm.invoke.CLI
 
+            print(f'DEBUG: BATCH PARENT ENVIRONMENT:')
+            print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
+            print("\n".join([f'{x}:{os.environ[x]}' for x in os.environ.keys()]))
+            print('<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<')
+            
             parent_conn, child_conn = Pipe()
             children = set()
             for i in range(processes_to_launch):
@@ -100,8 +108,8 @@ def expand_prompts(
             for command in commands:
                 sequence += 1
                 format = _get_fn_format(outdir, sequence)
-                parent_conn.send(
-                    command + f' --fnformat="{format}"'
+                parent_conn.send_bytes(
+                    (command + f' --fnformat="{format}"').encode('utf-8')
                 )
             parent_conn.close()
         else:
@@ -111,12 +119,22 @@ def expand_prompts(
         for p in children:
             p.terminate()
 
+def _dummy_cli_main():
+    counter = 0
+    while line := sys.stdin.readline():
+        print(f'[{counter}] {os.getpid()} got command {line.rstrip()}\n')
+        counter += 1
+        time.sleep(1)
+
 def _get_fn_format(directory:str, sequence:int)->str:
     """
     Get a filename that doesn't exceed filename length restrictions
     on the current platform.
     """
-    max_length = os.pathconf(directory,'PC_NAME_MAX')
+    try:
+        max_length = os.pathconf(directory,'PC_NAME_MAX')
+    except:
+        max_length = 255
     prefix = f'dp.{sequence:04}.'
     suffix = '.png'
     max_length -= len(prefix)+len(suffix)
@@ -130,7 +148,7 @@ class MessageToStdin(object):
     def readline(self) -> str:
         try:
             if len(self.linebuffer) == 0:
-                message = self.connection.recv()
+                message = self.connection.recv_bytes().decode('utf-8')
                 self.linebuffer = message.split("\n")
             result = self.linebuffer.pop(0)
             return result
@@ -176,9 +194,9 @@ def _run_invoke(
     os.environ["CUDA_VISIBLE_DEVICES"] = f"{gpu}"
     sys.argv = args
     sys.stdin = MessageToStdin(conn_in)
-    sys.stdout = FilterStream(sys.stdout, include=re.compile("^\[\d+\]"))
-    with open(logfile, "w") as stderr, redirect_stderr(stderr):
-        entry_point()
+#    sys.stdout = FilterStream(sys.stdout, include=re.compile("^\[\d+\]"))
+#    with open(logfile, "w") as stderr, redirect_stderr(stderr):
+    entry_point()
 
 
 def _filter_output(stream: TextIOBase):
@@ -235,6 +253,10 @@ def main():
         default=1,
         help="When executing invokeai, how many parallel processes to execute per CUDA GPU.",
     )
+    parser.add_argument(
+        '--root_dir',
+        default=None,
+        help='Path to directory containing "models", "outputs" and "configs". If not present will read from environment variable INVOKEAI_ROOT. Defaults to ~/invokeai'    )
     opt = parser.parse_args()
 
     if opt.example:
@@ -258,6 +280,7 @@ def main():
         run_invoke=opt.invoke,
         invoke_model=opt.model,
         invoke_outdir=opt.outdir,
+        invoke_root=opt.root,
         processes_per_gpu=opt.processes_per_gpu,
     )