Fixes to make compatible with Python 3.12 (#7112)

* Remove use of deprecated imp.load_source

The entire `imp` module has been removed from Python 3.12. This patch applies the recommended replacement using `importlib`.

* Fix usage of from importlib.metadata.entry_points to work with newer importlib & Python 3.12

* Update registry.py

Fix order of imports

* Use importlib.util.module_from_spec() instead of deprecated load_module()

* auto-fixed import style (isort)

* enable py 12

* run coverage for lower and upper bound

* fix style error

* make import conditional

* fix?

* fix env

* style fix

* only use new loader on 3.12

* fix order

* fix module loading

* reimplement assertDictContainsSubset

* remove old testing alias

---------

Co-authored-by: Thea Flowers <thea@winterbloom.com>
Co-authored-by: Oliver <oliver.henry.walters@gmail.com>

.github/workflows/qc_checks.yaml

@@ -261,18 +261,22 @@ jobs:
           coverage run -m unittest discover -s test/
 
   coverage:
-    name: Tests - DB [SQLite] + Coverage
+    name: Tests - DB [SQLite] + Coverage ${{ matrix.python_version }}
     runs-on: ubuntu-20.04
 
     needs: ["pre-commit", "paths-filter"]
     if: needs.paths-filter.outputs.server == 'true' || needs.paths-filter.outputs.force == 'true'
     continue-on-error: true # continue if a step fails so that coverage gets pushed
+    strategy:
+      matrix:
+        python_version: [3.9, 3.12]
 
     env:
       INVENTREE_DB_NAME: ./inventree.sqlite
       INVENTREE_DB_ENGINE: sqlite3
       INVENTREE_PLUGINS_ENABLED: true
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      python_version: ${{ matrix.python_version }}
 
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # pin@v4.1.1

src/backend/InvenTree/InvenTree/unit_test.py

@@ -435,3 +435,7 @@ class InvenTreeAPITestCase(ExchangeRateMixin, UserMixin, APITestCase):
             data.append(entry)
 
         return data
+
+    def assertDictContainsSubset(self, a, b):
+        """Assert that dictionary 'a' is a subset of dictionary 'b'."""
+        self.assertEqual(b, b | a)

src/backend/InvenTree/plugin/helpers.py

@@ -5,14 +5,16 @@ import logging
 import os
 import pathlib
 import pkgutil
+import sys
 import sysconfig
 import traceback
 from importlib.metadata import entry_points
+from importlib.util import module_from_spec
 
 from django import template
 from django.conf import settings
 from django.core.exceptions import AppRegistryNotReady
-from django.db.utils import IntegrityError, OperationalError, ProgrammingError
+from django.db.utils import IntegrityError
 
 logger = logging.getLogger('inventree')
 
@@ -110,7 +112,10 @@ def handle_error(error, do_raise: bool = True, do_log: bool = True, log_name: st
 
 def get_entrypoints():
     """Returns list for entrypoints for InvenTree plugins."""
-    return entry_points().get('inventree_plugins', [])
+    # on python before 3.12, we need to use importlib_metadata
+    if sys.version_info < (3, 12):
+        return entry_points().get('inventree_plugins', [])
+    return entry_points(group='inventree_plugins')
 
 
 # endregion
@@ -121,7 +126,8 @@ def get_git_log(path):
     """Get dict with info of the last commit to file named in path."""
     import datetime
 
-    from dulwich.repo import NotGitRepository, Repo
+    from dulwich.errors import NotGitRepository
+    from dulwich.repo import Repo
 
     from InvenTree.ready import isInTestMode
 
@@ -175,9 +181,15 @@ def get_modules(pkg, path=None):
     elif type(path) is not list:
         path = [path]
 
-    for loader, name, _ in pkgutil.walk_packages(path):
+    for finder, name, _ in pkgutil.walk_packages(path):
         try:
-            module = loader.find_module(name).load_module(name)
+            if sys.version_info < (3, 12):
+                module = finder.find_module(name).load_module(name)
+            else:
+                spec = finder.find_spec(name)
+                module = module_from_spec(spec)
+                sys.modules[name] = module
+                spec.loader.exec_module(module)
             pkg_names = getattr(module, '__all__', None)
             for k, v in vars(module).items():
                 if not k.startswith('_') and (pkg_names is None or k in pkg_names):

src/backend/InvenTree/plugin/registry.py

@@ -4,12 +4,15 @@
 - Manages setup and teardown of plugin class instances
 """
 
-import imp
 import importlib
+import importlib.machinery
+import importlib.util
 import logging
 import os
+import sys
 import time
 from collections import OrderedDict
+from importlib.machinery import SourceFileLoader
 from pathlib import Path
 from threading import Lock
 from typing import Any
@@ -411,9 +414,15 @@ class PluginsRegistry:
 
             # Gather Modules
             if parent_path:
-                raw_module = imp.load_source(
-                    plugin, str(parent_obj.joinpath('__init__.py'))
-                )
+                # On python 3.12 use new loader method
+                if sys.version_info < (3, 12):
+                    raw_module = _load_source(
+                        plugin, str(parent_obj.joinpath('__init__.py'))
+                    )
+                else:
+                    raw_module = SourceFileLoader(
+                        plugin, str(parent_obj.joinpath('__init__.py'))
+                    ).load_module()
             else:
                 raw_module = importlib.import_module(plugin)
             modules = get_plugins(raw_module, InvenTreePlugin, path=parent_path)
@@ -830,3 +839,18 @@ registry: PluginsRegistry = PluginsRegistry()
 def call_function(plugin_name, function_name, *args, **kwargs):
     """Global helper function to call a specific member function of a plugin."""
     return registry.call_plugin_function(plugin_name, function_name, *args, **kwargs)
+
+
+def _load_source(modname, filename):
+    """Helper function to replace deprecated & removed imp.load_source.
+
+    See https://docs.python.org/3/whatsnew/3.12.html#imp
+    """
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module