rebase cache partially on newest master: old version was: 18fce63c2af606737d312cb16adaa1698643dbdb Version72

.gitlab-ci.yml

@@ -13,6 +13,11 @@ variables:
   # https://docs.gitlab.com/ee/ci/yaml/#shallow-cloning
   GIT_DEPTH: 3
   GIT_CLEAN_FLAGS: -f
+  CACHE_FOLDER: cache-all
+  TARGET_SUBFOLDER: debug
+  LINK_LIB_ARGS: -sf # Arguments for linking dependencies. Set as a variable as some job (eg: tarpaulin) might need to force it.
+  CACHE_FALLBACK_KEY: "veloren" # the cache of master will be used if no branch cache can be found
+  FF_USE_FASTZIP: "true" # Enable Fastzip, a multi threaded performant zip extractor / compressor.
   CACHE_IMAGE_TAG: 5d0b61ad
 
 default:
@@ -35,6 +40,14 @@ before_script:
   - cargo --version
   - source ./.gitlab/scripts/env.sh
   - rm -rf target || echo "it seems that sometimes OLD data is left over"
+  - ln -s /dockercache/$CACHE_FOLDER target
+  - echo "Cache folder => $CACHE_FOLDER, Target subfolder => $TARGET_SUBFOLDER"
+  - find "cache/$TARGET_SUBFOLDER" -iname "*veloren*" -type d | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; mkdir -p "target/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist."
+  - find "cache/$TARGET_SUBFOLDER" -iname "*veloren*" -type f | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; cp "$line" "target/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist."
+
+cache:
+  key: "veloren" # Cache is managed on a Master basis. Each branch use the master cache. Only the master branches can push
+  policy: pull # Disable cache creation and upload on every branch.
 
 # 8866215 is the user that is used to sync data to the collaboration repos
 workflow:
@@ -47,6 +60,7 @@ workflow:
 include:
   - local: .gitlab/CI/recompile.yml
   - local: .gitlab/CI/release.yml
+  - local: .gitlab/CI/cache.gitlab-ci.yml
   - local: .gitlab/CI/check.gitlab-ci.yml
   - local: .gitlab/CI/build.gitlab-ci.yml
   - local: .gitlab/CI/publish.gitlab-ci.yml

.gitlab/CI/build.gitlab-ci.yml

@@ -1,5 +1,5 @@
-unittests:
+.tunittests:
-  extends: .recompile-branch
+  extends: .use-cache-quality
   stage: build
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG}
   variables:
@@ -11,8 +11,8 @@ unittests:
   retry:
     max: 2
 
-translation:
+.ttranslation:
-  extends: .release
+  extends: .use-cache-quality
   stage: build
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG}
   script:
@@ -30,9 +30,11 @@ translation:
   retry:
     max: 2
 
-benchmarks:
+.tbenchmarks:
-  extends: .release
+  extends: .use-cache-benchmark
   stage: build
+  variables:
+    TARGET_SUBFOLDER: release
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/bench:${CACHE_IMAGE_TAG}
   tags:
     - veloren-benchmark
@@ -54,9 +56,11 @@ benchmarks:
 
 # Coverage is needed on master for the README.md badge to work
 # tmp remove simd as it doesnt work with tarpaulin: https://github.com/rust-lang/rust/issues/77529
-coverage:
+.tcoverage:
-  extends: .release
+  extends: .use-cache-coverage
   stage: build
+  variables:
+    CACHE_FOLDER: cache-tarpaulin
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/tarpaulin:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
@@ -65,8 +69,39 @@ coverage:
   retry:
     max: 2
 
+unittests:
+  extends:
+    - .recompile-branch
+    - .tunittests
+
+unittests-pcache:
+  extends:
+    - .release
+    - .persist-cache
+    - .tunittests
+
+translation-pcache:
+  extends:
+    - .release
+    - .persist-cache
+    - .ttranslation
+
+benchmarks-pcache:
+  extends:
+    - .release
+    - .persist-cache
+    - .tbenchmarks
+
+coverage-pcache:
+  extends:
+    - .release
+    - .persist-cache
+    - .tcoverage
+
 #linux, windows, macos builds here as template
 .tlinux:
+  variables:
+    CACHE_FOLDER: cache-release-linux
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-linux:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
@@ -83,6 +118,8 @@ coverage:
     expire_in: 1 week
 
 .twindows:
+  variables:
+    CACHE_FOLDER: cache-release-windows
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-windows:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
@@ -106,6 +143,8 @@ coverage:
 
 .tmacos:
   tags: ["veloren-macos"]
+  variables:
+    CACHE_FOLDER: cache-release-macos
   before_script:
     - source $HOME/.cargo/env
     - cargo --version

.gitlab/CI/cache.gitlab-ci.yml

@@ -0,0 +1,79 @@
+#### TODO: needs to be adjusted
+
+.prune-cache:
+  tags:
+    - cache #FIXME: ADJUST TO veloren-docker
+  extends: .master-or-release
+  stage: check
+  cache:
+    policy: push
+    paths:
+      - cache
+  script:
+    - rm -rf cache/*
+  rules:
+    - when: manual
+      allow_failure: true
+
+prune-cache-all:
+  extends:
+    - .prune-cache
+    - .master-or-release
+  cache:
+    key: "veloren"
+
+prune-cache-coverage:
+  extends:
+    - .prune-cache
+    - .master-or-release
+  cache:
+    key: "veloren-coverage"
+
+prune-cache-benchmark:
+  extends:
+    - .prune-cache
+    - .master-or-release
+  cache:
+    key: "veloren-benchmark"
+
+.use-cache-all:
+  extends: .persist-cache #FIXME: REMOVE
+  tags:
+    - cache #FIXME: REMOVE
+  cache:
+    key: "veloren"
+    #policy: pull #FIXME: UNCOMMENT
+    paths:
+      - cache
+
+.use-cache-coverage:
+  extends: .persist-cache #FIXME: REMOVE
+  tags:
+    - cache #FIXME: REMOVE
+  cache:
+    key: "veloren-coverage" # specific cache to prevent conflict with tarpaulin
+    #policy: pull #FIXME: UNCOMMENT
+    paths:
+      - cache
+
+.use-cache-benchmark:
+  extends: .persist-cache #FIXME: REMOVE
+  tags:
+    - cache #FIXME: REMOVE
+  cache:
+    key: "veloren-benchmark"
+    #policy: pull #FIXME: UNCOMMENT
+    paths:
+      - cache
+
+# TODO: Evaluate release Caching
+
+.persist-cache:
+  tags:
+    - cache #FIXME: REMOVE
+  after_script:
+    - rm -rf cache && mkdir -p cache/$TARGET_SUBFOLDER/{deps,build,.fingerprint}
+    - find target/$TARGET_SUBFOLDER -iname "*veloren*" -type d | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; mkdir -p "cache/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some folder might already exist."
+    - find target/$TARGET_SUBFOLDER -iname "*veloren*" -type f -not -path "*.cargo-lock" | while read line ; do export name=$(echo $line | sed "s/.*$TARGET_SUBFOLDER\///g") ; cp "$CI_PROJECT_DIR/$line" "cache/$TARGET_SUBFOLDER/$name" ; done < "${1:-/dev/stdin}" || echo "Some file might already exist."
+  cache:
+    policy: pull-push

.gitlab/CI/check.gitlab-ci.yml

@@ -1,5 +1,5 @@
 # cargo clippy is a superset of cargo check, so we don't check manually
-code-quality:
+.tcode-quality:
   extends: .recompile-branch
   stage: check
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG}
@@ -8,12 +8,24 @@ code-quality:
     - cat ./.gitlab/scripts/code-quality.sh
     - source ./.gitlab/scripts/code-quality.sh
 
-security:
+.tsecurity:
   extends: .recompile-branch
   stage: check
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG}
   allow_failure: true
+  cache: {} # Disabling cache as it is not needed in order to reduce time
   script:
     - ln -s /dockercache/target target
     - cat ./.gitlab/scripts/security.sh
     - source ./.gitlab/scripts/security.sh
+
+code-quality:
+  extends:
+    - .recompile-branch
+    - .tcode-quality
+
+code-quality-pcache:
+  extends:
+    - .release
+    - .persist-cache
+    - .tcode-quality