Address change requests in first round of PR reviews.

Pending:

- Move model install calls into model manager and create passthrus in invocation_context.
- Consider splitting load_model_from_url() into a call to get the path and a call to load the path.
This commit is contained in:
Lincoln Stein
2024-04-24 23:53:30 -04:00
parent 34cdfc61ab
commit d72f272f16
7 changed files with 64 additions and 123 deletions

View File

@ -1,6 +1,8 @@
import base64
import io
import os
import re
import unicodedata
import warnings
from pathlib import Path
@ -12,6 +14,25 @@ from transformers import logging as transformers_logging
GIG = 1073741824
def slugify(value: str, allow_unicode: bool = False) -> str:
"""
Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated
dashes to single dashes. Remove characters that aren't alphanumerics,
underscores, or hyphens. Convert to lowercase. Also strip leading and
trailing whitespace, dashes, and underscores.
Adapted from Django: https://github.com/django/django/blob/main/django/utils/text.py
"""
value = str(value)
if allow_unicode:
value = unicodedata.normalize("NFKC", value)
else:
value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii")
value = re.sub(r"[/]", "_", value.lower())
value = re.sub(r"[^\w\s-]", "", value.lower())
return re.sub(r"[-\s]+", "-", value).strip("-_")
def directory_size(directory: Path) -> int:
"""
Return the aggregate size of all files in a directory (bytes).