From 8c15d14099d63d0265d4e12c9a606cade75f78a1 Mon Sep 17 00:00:00 2001 From: psychedelicious <4822129+psychedelicious@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:59:20 +1100 Subject: [PATCH] fix: use locale encoding We have had a few bugs with v4 related to file encodings, especially on Windows. Windows uses its own character encodings instead of `utf-8`, often `cp1252`. Some characters cannot be decoded using `utf-8`, causing `UnicodeDecodeError`. There are a couple places where this can cause problems: - In the installer bootstrap, we install or upgrade `pip` and decode the result, using `subprocess`. The input to this includes the user's home dir. In #6105, the user had one of the problematic characters in their username. `subprocess` attempts and fails to decode the username, which crashes the installer. To fix this, we need to use `locale.getpreferredencoding()` when executing the command. - Similarly, in the model install service and config class, we attempt to load a yaml config file. If a problematic character is in the path to the file (which often includes the user's home dir), we can get the same error. One example is #6129 in which the models.yaml migration fails. To fix this, we need to open the file with `locale.getpreferredencoding()`. --- installer/lib/installer.py | 5 ++++- invokeai/app/services/config/config_default.py | 3 ++- invokeai/app/services/model_install/model_install_default.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/installer/lib/installer.py b/installer/lib/installer.py index 5297366ccb..11823b413e 100644 --- a/installer/lib/installer.py +++ b/installer/lib/installer.py @@ -3,6 +3,7 @@ InvokeAI installer script """ +import locale import os import platform import re @@ -316,7 +317,9 @@ def upgrade_pip(venv_path: Path) -> str | None: python = str(venv_path.expanduser().resolve() / python) try: - result = subprocess.check_output([python, "-m", "pip", "install", "--upgrade", "pip"]).decode() + result = subprocess.check_output([python, "-m", "pip", "install", "--upgrade", "pip"]).decode( + encoding=locale.getpreferredencoding() + ) except subprocess.CalledProcessError as e: print(e) result = None diff --git a/invokeai/app/services/config/config_default.py b/invokeai/app/services/config/config_default.py index 79cbbc39af..f453a56584 100644 --- a/invokeai/app/services/config/config_default.py +++ b/invokeai/app/services/config/config_default.py @@ -3,6 +3,7 @@ from __future__ import annotations +import locale import os import re import shutil @@ -401,7 +402,7 @@ def load_and_migrate_config(config_path: Path) -> InvokeAIAppConfig: An instance of `InvokeAIAppConfig` with the loaded and migrated settings. """ assert config_path.suffix == ".yaml" - with open(config_path) as file: + with open(config_path, "rt", encoding=locale.getpreferredencoding()) as file: loaded_config_dict = yaml.safe_load(file) assert isinstance(loaded_config_dict, dict) diff --git a/invokeai/app/services/model_install/model_install_default.py b/invokeai/app/services/model_install/model_install_default.py index b5595d18c6..20cfc1c4ff 100644 --- a/invokeai/app/services/model_install/model_install_default.py +++ b/invokeai/app/services/model_install/model_install_default.py @@ -1,5 +1,6 @@ """Model installation class.""" +import locale import os import re import signal @@ -323,7 +324,8 @@ class ModelInstallService(ModelInstallServiceBase): legacy_models_yaml_path = Path(self._app_config.root_path, legacy_models_yaml_path) if legacy_models_yaml_path.exists(): - legacy_models_yaml = yaml.safe_load(legacy_models_yaml_path.read_text()) + with open(legacy_models_yaml_path, "rt", encoding=locale.getpreferredencoding()) as file: + legacy_models_yaml = yaml.safe_load(file) yaml_metadata = legacy_models_yaml.pop("__metadata__") yaml_version = yaml_metadata.get("version")