From 2d058ea10075c3e41b1150c2edf13840d181099a Mon Sep 17 00:00:00 2001 From: appflowy Date: Wed, 1 Dec 2021 16:44:58 +0800 Subject: [PATCH] config docker ci --- .github/workflows/backend_general.yml | 80 +++++++++++++++++++++-- backend/Cargo.lock | 46 +++++++++---- backend/Cargo.toml | 3 +- backend/Dockerfile | 73 +++++++-------------- backend/rust-toolchain | 2 - backend/scripts/database/database.mk | 6 +- backend/scripts/database/init_database.sh | 32 ++++++++- backend/scripts/database/init_postgres.sh | 22 ------- 8 files changed, 160 insertions(+), 104 deletions(-) delete mode 100644 backend/rust-toolchain delete mode 100755 backend/scripts/database/init_postgres.sh diff --git a/.github/workflows/backend_general.yml b/.github/workflows/backend_general.yml index 34d519f856..a3d4093bd0 100644 --- a/.github/workflows/backend_general.yml +++ b/.github/workflows/backend_general.yml @@ -3,18 +3,84 @@ name: Backend on: push: branches: [ main ] - paths: - - 'backend' pull_request: branches: [ main ] - paths: - - 'backend' jobs: - build: + test: + name: Test runs-on: ubuntu-latest - + services: + postgres: + image: postgres:12 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: password + POSTGRES_DB: postgres + ports: + - 5432:5432 + env: + SQLX_VERSION: 0.5.7 + SQLX_FEATURES: postgres steps: - - uses: actions/checkout@v2 + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Cache dependencies + id: cache-dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Install stable toolchain + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Cache sqlx-cli + uses: actions/cache@v2 + id: cache-sqlx + with: + path: | + ~/.cargo/bin/sqlx + ~/.cargo/bin/cargo-sqlx + key: ${{ runner.os }}-sqlx-${{ env.SQLX_VERSION }}-${{ env.SQLX_FEATURES }} + + - name: Install sqlx-cli + uses: actions-rs/cargo@v1 + if: steps.cache-sqlx.outputs.cache-hit == false + with: + command: install + args: > + sqlx-cli + --force + --version=${{ env.SQLX_VERSION }} + --features=${{ env.SQLX_FEATURES }} + --no-default-features + --locked + + - name: Migrate database + working-directory: backend/ + run: | + sudo apt-get install libpq-dev -y + SKIP_DOCKER=true ./backend/scripts/init_database.sh + + - name: Check sqlx-data.json is up-to-date + working-directory: backend/ + run: | + cargo sqlx prepare --check -- --bin backend + + - name: Run cargo test + working-directory: backend/ + uses: actions-rs/cargo@v1 + with: + command: test + diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 347d05b481..93b2bfd241 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -784,7 +784,7 @@ dependencies = [ "aes-gcm", "base64 0.13.0", "hkdf", - "hmac", + "hmac 0.10.1", "percent-encoding", "rand", "sha2", @@ -926,6 +926,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "ctr" version = "0.6.0" @@ -1668,7 +1678,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ "digest", - "hmac", + "hmac 0.10.1", ] [[package]] @@ -1677,7 +1687,17 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ - "crypto-mac", + "crypto-mac 0.10.1", + "digest", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac 0.11.1", "digest", ] @@ -3058,9 +3078,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.6" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b977121ecc75cadd442a6a8c600f5ded6d7117a46d55ed417a815bd94b2af237" +checksum = "7911b0031a0247af40095838002999c7a52fba29d9739e93326e71a5a1bc9d43" dependencies = [ "sqlx-core", "sqlx-macros", @@ -3068,9 +3088,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.6" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6da11206ae9ba3ececf83d44aef533c0d8176ae430cfaa14b408ce8b6bc4ab4b" +checksum = "aec89bfaca8f7737439bad16d52b07f1ccd0730520d3bf6ae9d069fe4b641fb1" dependencies = [ "ahash", "atoi", @@ -3091,7 +3111,8 @@ dependencies = [ "futures-util", "hashlink", "hex", - "hmac", + "hmac 0.11.0", + "indexmap", "itoa", "libc", "log", @@ -3121,13 +3142,12 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.5.6" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419f9005ad3b065ab4e774361b0d56a24e8e3dc9b2c9ddeaf018b5fdc1166ec2" +checksum = "584866c833511b1a152e87a7ee20dee2739746f60c858b3c5209150bc4b466f5" dependencies = [ "dotenv", "either", - "futures", "heck", "hex", "once_cell", @@ -3144,9 +3164,9 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.5.6" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d39baa13314f936c5991429b0b656980f52d23dfb15078bbea2c51f5086eeb" +checksum = "0d1bd069de53442e7a320f525a6d4deb8bb0621ac7a55f7eccbc2b58b57f43d0" dependencies = [ "actix-rt", "once_cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index b1c4acd0c0..97553b4d24 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -68,7 +68,7 @@ backend-service = { path = "../shared-lib/backend-service", features = ["http_se ormx = { version = "0.7", features = ["postgres"]} [dependencies.sqlx] -version = "0.5.6" +version = "0.5.7" default-features = false features = [ "runtime-actix-rustls", @@ -78,7 +78,6 @@ features = [ "chrono", "migrate", "offline", - "any", ] diff --git a/backend/Dockerfile b/backend/Dockerfile index 368d090250..a068e90649 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,57 +1,28 @@ -FROM rust:1.56.1 -# Let's switch our working directory to `app` (equivalent to `cd app`) -# The `app` folder will be created for us by Docker in case it does not -# exist already. +FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as chef WORKDIR /app +FROM chef as planner COPY . . +RUN cargo chef prepare --recipe-path recipe.json -WORKDIR /app/backend +FROM chef as builder +COPY --from=planner /app/recipe.json recipe.json +# Build dependencies +RUN cargo chef cook --release --recipe-path recipe.json +COPY . . ENV SQLX_OFFLINE true +# Build our project +RUN cargo build --release --bin backend + +FROM debian:bullseye-slim AS runtime +WORKDIR /app +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends openssl \ + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/target/release/backend backend +COPY configuration configuration ENV APP_ENVIRONMENT production -RUN RUSTFLAGS="-C opt-level=2" cargo build --release - -# When `docker run` is executed, launch the binary! -ENTRYPOINT ["./target/release/backend"] - -# - - - -## We use the latest Rust stable release as base image -#FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as planner -#WORKDIR /app -#COPY . . -# -#WORKDIR /app/backend -#RUN cargo chef prepare --recipe-path recipe.json -# -#FROM lukemathwalker/cargo-chef:latest-rust-1.53.0 as cacher -#WORKDIR /app/backend -#COPY --from=planner /app/backend/recipe.json recipe.json -## Build our project dependencies, not our application! -#RUN cargo chef cook --release --recipe-path recipe.json -# -#FROM rust:1.53.0 AS builder -#WORKDIR /app/backend -## Copy over the cached dependencies -#COPY --from=cacher /app/backend/target target -#COPY --from=cacher /usr/local/cargo /usr/local/cargo -#COPY . . -# -#ENV SQLX_OFFLINE true -#RUN cargo build --release --bin backend -# -# -#FROM debian:buster-slim AS runtime -#WORKDIR /app/backend -#RUN apt-get update -y \ -# && apt-get install -y --no-install-recommends openssl \ -# # Clean up -# && apt-get autoremove -y \ -# && apt-get clean -y \ -# && rm -rf /var/lib/apt/lists/* -#COPY --from=builder /app/backend/target/release/backend backend -##COPY configuration configuration -#ENV APP_ENVIRONMENT production -#ENTRYPOINT ["./backend"] +ENTRYPOINT ["./backend"] diff --git a/backend/rust-toolchain b/backend/rust-toolchain deleted file mode 100644 index 011b8eae6c..0000000000 --- a/backend/rust-toolchain +++ /dev/null @@ -1,2 +0,0 @@ -[toolchain] -channel = "nightly-2021-04-24" diff --git a/backend/scripts/database/database.mk b/backend/scripts/database/database.mk index 5792873a49..40d616d2c3 100644 --- a/backend/scripts/database/database.mk +++ b/backend/scripts/database/database.mk @@ -5,11 +5,9 @@ export DB_NAME=flowy export DB_PORT=5433 export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@localhost:${DB_PORT}/${DB_NAME} export ROOT = "./scripts/database" +export DB_HOST = "localhost" -init_postgres: - ${ROOT}/init_postgres.sh - -init_database: init_postgres +init_database: ${ROOT}/init_database.sh reset_db: diff --git a/backend/scripts/database/init_database.sh b/backend/scripts/database/init_database.sh index 7f11c53249..06ac0c8878 100755 --- a/backend/scripts/database/init_database.sh +++ b/backend/scripts/database/init_database.sh @@ -18,11 +18,37 @@ if ! [ -x "$(command -v sqlx)" ]; then exit 1 fi -until psql -h "localhost" -U "${DB_USER}" -p "${DB_PORT}" -d "postgres" -c '\q'; -do +if [[ -z "${SKIP_DOCKER}" ]] +then + RUNNING_POSTGRES_CONTAINER=$(docker ps --filter 'name=postgres' --format '{{.ID}}') + if [[ -n $RUNNING_POSTGRES_CONTAINER ]]; then + echo >&2 "there is a postgres container already running, kill it with" + echo >&2 " docker kill ${RUNNING_POSTGRES_CONTAINER}" + exit 1 + fi + + docker run \ + --name="flowy" \ + -e POSTGRES_USER=${DB_USER} \ + -e POSTGRES_PASSWORD=${DB_PASSWORD} \ + -e POSTGRES_DB=${DB_NAME} \ + -p "${DB_PORT}":5432 \ + -d postgres \ + postgres -N 1000 +fi + + +# Keep pinging Postgres until it's ready to accept commands +until PGPASSWORD="${DB_PASSWORD}" psql -h "${DB_HOST}" -U "${DB_USER}" -p "${DB_PORT}" -d "postgres" -c '\q'; do >&2 echo "Postgres is still unavailable - sleeping" sleep 1 done ->&2 echo "Postgres is up and running on port ${DB_PORT}!" +>&2 echo "Postgres is up and running on port ${DB_PORT} - running migrations now!" + +export DATABASE_URL=postgres://${DB_USER}:${DB_PASSWORD}@localhost:${DB_PORT}/${DB_NAME} sqlx database create +sqlx migrate run + +>&2 echo "Postgres has been migrated, ready to go!" + diff --git a/backend/scripts/database/init_postgres.sh b/backend/scripts/database/init_postgres.sh deleted file mode 100755 index 60816f0959..0000000000 --- a/backend/scripts/database/init_postgres.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -x -set -eo pipefail - -#if [[ -z "${RESET}" ]] -#then -# docker stop flowy -# docker rm flowy -#fi - -if [[ -z "${SKIP_DOCKER}" ]] -then - docker run \ - --name="flowy" \ - -e POSTGRES_USER=${DB_USER} \ - -e POSTGRES_PASSWORD=${DB_PASSWORD} \ - -e POSTGRES_DB=${DB_NAME} \ - -p "${DB_PORT}":5432 \ - -d postgres \ - postgres -N 1000 -fi - # ^ Increased maximum number of connections for testing purposes \ No newline at end of file