From 11d3cd3c0c20c1ad6f24868f856f9366fa669ce0 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 21 Dec 2021 00:34:23 +1100 Subject: [PATCH 1/3] Change dev environment setup to use postgresql - sqlite causes too many errors - fix requirements file - fixes for docker setup - A lot of concurrency issues - Bite the bullet, time to go! --- docker/Dockerfile | 2 +- docker/dev-config.env | 17 ++++++++--------- docker/docker-compose.dev.yml | 26 +++++++++++++++++++++++++- docker/init.sh | 2 +- docker/prod-config.env | 2 +- docker/requirements.txt | 4 ---- requirements.txt | 1 + 7 files changed, 37 insertions(+), 17 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index be8479724e..0978e43075 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -101,7 +101,7 @@ RUN chown -R inventree:inventreegroup ${INVENTREE_HOME}/* USER inventree # Install InvenTree packages -RUN pip3 install --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt +RUN pip3 install --user --no-cache-dir --disable-pip-version-check -r ${INVENTREE_HOME}/requirements.txt # Need to be running from within this directory WORKDIR ${INVENTREE_MNG_DIR} diff --git a/docker/dev-config.env b/docker/dev-config.env index 8654b708d6..df65f723c8 100644 --- a/docker/dev-config.env +++ b/docker/dev-config.env @@ -2,14 +2,13 @@ # Set DEBUG to False for a production environment! INVENTREE_DEBUG=True - -# Change verbosity level for debug output INVENTREE_DEBUG_LEVEL=INFO -# Database linking options -INVENTREE_DB_ENGINE=sqlite3 -INVENTREE_DB_NAME=/home/inventree/dev/inventree_db.sqlite3 -# INVENTREE_DB_HOST=hostaddress -# INVENTREE_DB_PORT=5432 -# INVENTREE_DB_USERNAME=dbuser -# INVENTREE_DB_PASSWEORD=dbpassword +# Database configuration options +# Note: The example setup is for a PostgreSQL database (change as required) +INVENTREE_DB_ENGINE=postgresql +INVENTREE_DB_NAME=inventree +INVENTREE_DB_HOST=inventree-dev-db +INVENTREE_DB_PORT=5432 +INVENTREE_DB_USER=pguser +INVENTREE_DB_PASSWORD=pgpassword diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index c4be092189..14a9c4bbfc 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -1,8 +1,10 @@ version: "3.8" # Docker compose recipe for InvenTree development server -# - Runs sqlite3 as the database backend +# - Runs PostgreSQL as the database backend # - Uses built-in django webserver +# - Runs the InvenTree background worker process +# - Serves media and static content directly from Django webserver # IMPORANT NOTE: # The InvenTree docker image does not clone source code from git. @@ -11,10 +13,32 @@ version: "3.8" # The django server will auto-detect any code changes and reload the server. services: + + # Database service + # Use PostgreSQL as the database backend + # Note: This can be changed to a different backend if required + inventree-dev-db: + container_name: inventree-dev-db + image: postgres:13 + ports: + - 5432/tcp + environment: + - PGDATA=/var/lib/postgresql/data/dev/pgdb + # The pguser and pgpassword values must be the same in the other containers + # Ensure that these are correctly configured in your dev-config.env file + - POSTGRES_USER=pguser + - POSTGRES_PASSWORD=pgpassword + volumes: + # Map 'data' volume such that postgres database is stored externally + - src:/var/lib/postgresql/data + restart: unless-stopped + # InvenTree web server services # Uses gunicorn as the web server inventree-dev-server: container_name: inventree-dev-server + depends_on: + - inventree-dev-db build: context: . target: dev diff --git a/docker/init.sh b/docker/init.sh index 7622806d0f..793661dc9f 100644 --- a/docker/init.sh +++ b/docker/init.sh @@ -27,7 +27,7 @@ fi if [[ -n "$INVENTREE_PY_ENV" ]]; then echo "Using Python virtual environment: ${INVENTREE_PY_ENV}" # Setup a virtual environment (within the "dev" directory) - python3 -m venv ${INVENTREE_PY_ENV} + python3 -m venv ${INVENTREE_PY_ENV} --system-site-packages # Activate the virtual environment source ${INVENTREE_PY_ENV}/bin/activate diff --git a/docker/prod-config.env b/docker/prod-config.env index 34b9c6a081..55d28e2f96 100644 --- a/docker/prod-config.env +++ b/docker/prod-config.env @@ -6,7 +6,7 @@ INVENTREE_DEBUG=False INVENTREE_LOG_LEVEL=WARNING -# Database configuration +# Database configuration options # Note: The example setup is for a PostgreSQL database (change as required) INVENTREE_DB_ENGINE=postgresql INVENTREE_DB_NAME=inventree diff --git a/docker/requirements.txt b/docker/requirements.txt index 1a110d6cde..7118a0670a 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -4,13 +4,9 @@ setuptools>=57.4.0 wheel>=0.37.0 invoke>=1.4.0 # Invoke build tool -gunicorn>=20.1.0 # Gunicorn web server # Database links psycopg2>=2.9.1 mysqlclient>=2.0.3 pgcli>=3.1.0 mariadb>=1.0.7 - -# Cache -django-redis>=5.0.0 diff --git a/requirements.txt b/requirements.txt index c6cc4effff..d6405bb7bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,6 +21,7 @@ django-markdownify==0.8.0 # Markdown rendering django-markdownx==3.0.1 # Markdown form fields django-money==1.1 # Django app for currency management django-mptt==0.11.0 # Modified Preorder Tree Traversal +django-redis>=5.0.0 django-q==1.3.4 # Background task scheduling django-sql-utils==0.5.0 # Advanced query annotation / aggregation django-stdimage==5.1.1 # Advanced ImageField management From 470ad5507ac1bd89c776c9eb2ee002e9f6e4cecb Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 21 Dec 2021 00:40:41 +1100 Subject: [PATCH 2/3] Remove sqlite tests for CI --- .github/workflows/qc_checks.yaml | 54 ++++++-------------------------- 1 file changed, 9 insertions(+), 45 deletions(-) diff --git a/.github/workflows/qc_checks.yaml b/.github/workflows/qc_checks.yaml index 598f518f27..d2906b75c4 100644 --- a/.github/workflows/qc_checks.yaml +++ b/.github/workflows/qc_checks.yaml @@ -148,49 +148,6 @@ jobs: cd ${{ env.wrapper_name }} invoke test - coverage: - name: Sqlite / coverage - needs: ['javascript', 'html'] - runs-on: ubuntu-latest - - env: - INVENTREE_DB_NAME: ./inventree.sqlite - INVENTREE_DB_ENGINE: sqlite3 - - steps: - - name: Checkout Code - uses: actions/checkout@v2 - - name: Setup Python ${{ env.python_version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ env.python_version }} - cache: 'pip' - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install gettext - pip3 install invoke - invoke install - invoke static - - name: Coverage Tests - run: | - invoke coverage - - name: Data Import Export - run: | - invoke migrate - invoke import-fixtures - invoke export-records -f data.json - rm inventree.sqlite - invoke migrate - invoke import-records -f data.json - invoke import-records -f data.json - - name: Test Translations - run: invoke translate - - name: Check Migration Files - run: python3 ci/check_migration_files.py - - name: Upload Coverage Report - run: coveralls - postgres: name: Postgres needs: ['javascript', 'html'] @@ -288,8 +245,9 @@ jobs: pip3 install invoke pip3 install mysqlclient invoke install - - name: Run Tests - run: invoke test + - name: Coverage Tests + run: | + invoke coverage - name: Data Import Export run: | invoke migrate @@ -299,3 +257,9 @@ jobs: python3 ./InvenTree/manage.py flush --noinput invoke import-records -f data.json invoke import-records -f data.json + - name: Test Translations + run: invoke translate + - name: Check Migration Files + run: python3 ci/check_migration_files.py + - name: Upload Coverage Report + run: coveralls \ No newline at end of file From e49f89d8cc830991c2db6819149133e69da85242 Mon Sep 17 00:00:00 2001 From: Oliver Date: Tue, 21 Dec 2021 00:43:03 +1100 Subject: [PATCH 3/3] Revert "Remove sqlite tests for CI" This reverts commit 470ad5507ac1bd89c776c9eb2ee002e9f6e4cecb. --- .github/workflows/qc_checks.yaml | 54 ++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/.github/workflows/qc_checks.yaml b/.github/workflows/qc_checks.yaml index d2906b75c4..598f518f27 100644 --- a/.github/workflows/qc_checks.yaml +++ b/.github/workflows/qc_checks.yaml @@ -148,6 +148,49 @@ jobs: cd ${{ env.wrapper_name }} invoke test + coverage: + name: Sqlite / coverage + needs: ['javascript', 'html'] + runs-on: ubuntu-latest + + env: + INVENTREE_DB_NAME: ./inventree.sqlite + INVENTREE_DB_ENGINE: sqlite3 + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Setup Python ${{ env.python_version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.python_version }} + cache: 'pip' + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install gettext + pip3 install invoke + invoke install + invoke static + - name: Coverage Tests + run: | + invoke coverage + - name: Data Import Export + run: | + invoke migrate + invoke import-fixtures + invoke export-records -f data.json + rm inventree.sqlite + invoke migrate + invoke import-records -f data.json + invoke import-records -f data.json + - name: Test Translations + run: invoke translate + - name: Check Migration Files + run: python3 ci/check_migration_files.py + - name: Upload Coverage Report + run: coveralls + postgres: name: Postgres needs: ['javascript', 'html'] @@ -245,9 +288,8 @@ jobs: pip3 install invoke pip3 install mysqlclient invoke install - - name: Coverage Tests - run: | - invoke coverage + - name: Run Tests + run: invoke test - name: Data Import Export run: | invoke migrate @@ -257,9 +299,3 @@ jobs: python3 ./InvenTree/manage.py flush --noinput invoke import-records -f data.json invoke import-records -f data.json - - name: Test Translations - run: invoke translate - - name: Check Migration Files - run: python3 ci/check_migration_files.py - - name: Upload Coverage Report - run: coveralls \ No newline at end of file