diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e667b6c3f0..76b42217c2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,6 +13,11 @@ variables: # https://docs.gitlab.com/ee/ci/yaml/#shallow-cloning GIT_DEPTH: 3 GIT_CLEAN_FLAGS: -f + CACHE_FOLDER: cache-all + TARGET_SUBFOLDER: debug + LINK_LIB_ARGS: -sf # Arguments for linking dependencies. Set as a variable as some job (eg: tarpaulin) might need to force it. + CACHE_FALLBACK_KEY: "veloren" # the cache of master will be used if no branch cache can be found + FF_USE_FASTZIP: "true" # Enable Fastzip, a multi threaded performant zip extractor / compressor. CACHE_IMAGE_TAG: 5d0b61ad default: @@ -35,6 +40,14 @@ before_script: - cargo --version - source ./.gitlab/scripts/env.sh - rm -rf target || echo "it seems that sometimes OLD data is left over" + - ln -s /dockercache/$CACHE_FOLDER target + - echo "Cache folder => $CACHE_FOLDER, Target subfolder => $TARGET_SUBFOLDER" + - find "cache/$TARGET_SUBFOLDER" -iname "*veloren*" -type d | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; mkdir -p "target/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist." + - find "cache/$TARGET_SUBFOLDER" -iname "*veloren*" -type f | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; cp "$line" "target/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist." + +cache: + key: "veloren" # Cache is managed on a Master basis. Each branch use the master cache. Only the master branches can push + policy: pull # Disable cache creation and upload on every branch. # 8866215 is the user that is used to sync data to the collaboration repos workflow: @@ -47,6 +60,7 @@ workflow: include: - local: .gitlab/CI/recompile.yml - local: .gitlab/CI/release.yml + - local: .gitlab/CI/cache.gitlab-ci.yml - local: .gitlab/CI/check.gitlab-ci.yml - local: .gitlab/CI/build.gitlab-ci.yml - local: .gitlab/CI/publish.gitlab-ci.yml diff --git a/.gitlab/CI/build.gitlab-ci.yml b/.gitlab/CI/build.gitlab-ci.yml index 005dd3ef95..6127bb24fb 100644 --- a/.gitlab/CI/build.gitlab-ci.yml +++ b/.gitlab/CI/build.gitlab-ci.yml @@ -1,5 +1,5 @@ -unittests: - extends: .recompile-branch +.tunittests: + extends: .use-cache-quality stage: build image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG} variables: @@ -11,8 +11,8 @@ unittests: retry: max: 2 -translation: - extends: .release +.ttranslation: + extends: .use-cache-quality stage: build image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG} script: @@ -30,9 +30,11 @@ translation: retry: max: 2 -benchmarks: - extends: .release +.tbenchmarks: + extends: .use-cache-benchmark stage: build + variables: + TARGET_SUBFOLDER: release image: registry.gitlab.com/veloren/veloren-docker-ci/cache/bench:${CACHE_IMAGE_TAG} tags: - veloren-benchmark @@ -54,9 +56,11 @@ benchmarks: # Coverage is needed on master for the README.md badge to work # tmp remove simd as it doesnt work with tarpaulin: https://github.com/rust-lang/rust/issues/77529 -coverage: - extends: .release +.tcoverage: + extends: .use-cache-coverage stage: build + variables: + CACHE_FOLDER: cache-tarpaulin image: registry.gitlab.com/veloren/veloren-docker-ci/cache/tarpaulin:${CACHE_IMAGE_TAG} script: - ln -s /dockercache/target target @@ -65,8 +69,39 @@ coverage: retry: max: 2 +unittests: + extends: + - .recompile-branch + - .tunittests + +unittests-pcache: + extends: + - .release + - .persist-cache + - .tunittests + +translation-pcache: + extends: + - .release + - .persist-cache + - .ttranslation + +benchmarks-pcache: + extends: + - .release + - .persist-cache + - .tbenchmarks + +coverage-pcache: + extends: + - .release + - .persist-cache + - .tcoverage + #linux, windows, macos builds here as template .tlinux: + variables: + CACHE_FOLDER: cache-release-linux image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-linux:${CACHE_IMAGE_TAG} script: - ln -s /dockercache/target target @@ -83,6 +118,8 @@ coverage: expire_in: 1 week .twindows: + variables: + CACHE_FOLDER: cache-release-windows image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-windows:${CACHE_IMAGE_TAG} script: - ln -s /dockercache/target target @@ -106,6 +143,8 @@ coverage: .tmacos: tags: ["veloren-macos"] + variables: + CACHE_FOLDER: cache-release-macos before_script: - source $HOME/.cargo/env - cargo --version diff --git a/.gitlab/CI/cache.gitlab-ci.yml b/.gitlab/CI/cache.gitlab-ci.yml new file mode 100644 index 0000000000..495469ff72 --- /dev/null +++ b/.gitlab/CI/cache.gitlab-ci.yml @@ -0,0 +1,79 @@ +#### TODO: needs to be adjusted + +.prune-cache: + tags: + - cache #FIXME: ADJUST TO veloren-docker + extends: .master-or-release + stage: check + cache: + policy: push + paths: + - cache + script: + - rm -rf cache/* + rules: + - when: manual + allow_failure: true + +prune-cache-all: + extends: + - .prune-cache + - .master-or-release + cache: + key: "veloren" + +prune-cache-coverage: + extends: + - .prune-cache + - .master-or-release + cache: + key: "veloren-coverage" + +prune-cache-benchmark: + extends: + - .prune-cache + - .master-or-release + cache: + key: "veloren-benchmark" + +.use-cache-all: + extends: .persist-cache #FIXME: REMOVE + tags: + - cache #FIXME: REMOVE + cache: + key: "veloren" + #policy: pull #FIXME: UNCOMMENT + paths: + - cache + +.use-cache-coverage: + extends: .persist-cache #FIXME: REMOVE + tags: + - cache #FIXME: REMOVE + cache: + key: "veloren-coverage" # specific cache to prevent conflict with tarpaulin + #policy: pull #FIXME: UNCOMMENT + paths: + - cache + +.use-cache-benchmark: + extends: .persist-cache #FIXME: REMOVE + tags: + - cache #FIXME: REMOVE + cache: + key: "veloren-benchmark" + #policy: pull #FIXME: UNCOMMENT + paths: + - cache + +# TODO: Evaluate release Caching + +.persist-cache: + tags: + - cache #FIXME: REMOVE + after_script: + - rm -rf cache && mkdir -p cache/$TARGET_SUBFOLDER/{deps,build,.fingerprint} + - find target/$TARGET_SUBFOLDER -iname "*veloren*" -type d | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; mkdir -p "cache/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some folder might already exist." + - find target/$TARGET_SUBFOLDER -iname "*veloren*" -type f -not -path "*.cargo-lock" | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; cp "$CI_PROJECT_DIR/$line" "cache/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist." + cache: + policy: pull-push \ No newline at end of file diff --git a/.gitlab/CI/check.gitlab-ci.yml b/.gitlab/CI/check.gitlab-ci.yml index fb8a13171a..0657aa3363 100644 --- a/.gitlab/CI/check.gitlab-ci.yml +++ b/.gitlab/CI/check.gitlab-ci.yml @@ -1,5 +1,5 @@ # cargo clippy is a superset of cargo check, so we don't check manually -code-quality: +.tcode-quality: extends: .recompile-branch stage: check image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG} @@ -8,12 +8,24 @@ code-quality: - cat ./.gitlab/scripts/code-quality.sh - source ./.gitlab/scripts/code-quality.sh -security: +.tsecurity: extends: .recompile-branch stage: check image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG} allow_failure: true + cache: {} # Disabling cache as it is not needed in order to reduce time script: - ln -s /dockercache/target target - cat ./.gitlab/scripts/security.sh - source ./.gitlab/scripts/security.sh + +code-quality: + extends: + - .recompile-branch + - .tcode-quality + +code-quality-pcache: + extends: + - .release + - .persist-cache + - .tcode-quality