Mirrored_Repos/InvokeAI
1
0
Fork 0
mirror of https://github.com/invoke-ai/InvokeAI synced 2024-08-30 20:32:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add warning log if moving a model from cuda to cpu causes unexpected change in VRAM usage.

Browse Source
This commit is contained in:
Ryan Dick 2023-09-29 09:41:13 -04:00
parent 7fa5bae8fd
commit ace8665411
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
invokeai/backend/model_management/model_cache.py
View File

@ -484,6 +484,20 @@ class ModelCache(object):
f" {get_pretty_snapshot_diff(snapshot_before, snapshot_after)}."
)
if not math.isclose(
abs((snapshot_before.vram or 0) - (snapshot_after.vram or 0)),
cache_entry.size,
rel_tol=0.1,
abs_tol=10 * MB,
):
self.logger.warning(
f"Moving '{model_key}' from {self.execution_device} to"
f" {self.storage_device} caused an unexpected change in VRAM usage. The model's"
" estimated size may be incorrect. Estimated model size:"
f" {(cache_entry.size/GIG):.2f} GB."
f" {get_pretty_snapshot_diff(snapshot_before, snapshot_after)}."
)
vram_in_use = snapshot_after.vram or 0
self.logger.debug(f"{(vram_in_use/GIG):.2f}GB VRAM used for models; max allowed={(reserved/GIG):.2f}GB")