Merge branch 'xMAC94x/experiment_ci' into 'master'

fix/cleanup ci

See merge request veloren/veloren!2595

.gitlab-ci.yml

@@ -33,11 +33,7 @@ before_script:
   - df -h /
   - free -h
   - cargo --version
-  - export DISABLE_GIT_LFS_CHECK=true
-  - export VELOREN_ASSETS="$(pwd)/assets"
-  - echo "VELOREN_ASSETS=$VELOREN_ASSETS"
-  - export RUSTFLAGS="-D warnings"
-  - export SHADERC_LIB_DIR=/shaderc/combined/
+  - source ./.gitlab/scripts/env.sh
   - rm -rf target || echo "it seems that sometimes OLD data is left over"
 
 # 8866215 is the user that is used to sync data to the collaboration repos

.gitlab/CI/build.gitlab-ci.yml

@@ -6,11 +6,8 @@ unittests:
     GIT_DEPTH: 9999999999999
   script:
     - ln -s /dockercache/target target
-    - rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored
-    - cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib
-    - rm -r target/debug/incremental* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - cargo test
+    - cat ./.gitlab/scripts/unittest.sh
+    - source ./.gitlab/scripts/unittest.sh
   retry:
     max: 2
 
@@ -23,12 +20,8 @@ benchmarks:
   script:
     - unset DISABLE_GIT_LFS_CHECK
     - ln -s /dockercache/target target
-    - ls -la target/*
-    - rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - rm -r target/release/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - rm -r target/debug/incremental/* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - rm -r target/release/incremental/* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - cargo bench || exit 0 #temp fix
+    - cat ./.gitlab/scripts/benchmark.sh
+    - source ./.gitlab/scripts/benchmark.sh
     - TAGUUID="Z$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1)" || echo "ignore this returncode, dont ask me why, it works"
     - echo $TAGUUID
     - echo 'SET veloren.timestamp = "'"$(git show --no-patch --no-notes --pretty='%cd' HEAD)"'";' > upload.sql
@@ -48,11 +41,8 @@ coverage:
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/tarpaulin:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
-    - rm -r target/debug/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - find ./* -name "Cargo.toml" -exec sed -i 's/, "simd"]/]/g' {} \;
-    - find ./* -name "Cargo.toml" -exec sed -i 's/"simd"]/]/g' {} \;
-    - sed -i 's/vek /#vek /g' ./Cargo.toml;
-    - cargo tarpaulin -v -- --test-threads=2
+    - cat ./.gitlab/scripts/coverage.sh
+    - source ./.gitlab/scripts/coverage.sh
   retry:
     max: 2
 
@@ -61,10 +51,8 @@ coverage:
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-linux:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
-    - rm -r target/release/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - export VELOREN_USERDATA_STRATEGY=executable
-    - cargo build --release -p veloren-voxygen --no-default-features --features default-publish
-    - cargo build --release -p veloren-server-cli
+    - cat ./.gitlab/scripts/linux.sh
+    - source ./.gitlab/scripts/linux.sh
     - cp -r target/release/veloren-server-cli $CI_PROJECT_DIR
     - cp -r target/release/veloren-voxygen $CI_PROJECT_DIR
   artifacts:
@@ -78,13 +66,9 @@ coverage:
 .twindows:
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/release-windows:${CACHE_IMAGE_TAG}
   script:
-    - update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix
-    - update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
     - ln -s /dockercache/target target
-    - rm -r target/release/incremental/veloren_* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - export VELOREN_USERDATA_STRATEGY=executable
-    - cargo build --target=x86_64-pc-windows-gnu --release -p veloren-voxygen --no-default-features --features default-publish
-    - cargo build --target=x86_64-pc-windows-gnu --release -p veloren-server-cli
+    - cat ./.gitlab/scripts/windows.sh
+    - source ./.gitlab/scripts/windows.sh
     - cp -r target/x86_64-pc-windows-gnu/release/veloren-server-cli.exe $CI_PROJECT_DIR
     - cp -r target/x86_64-pc-windows-gnu/release/veloren-voxygen.exe $CI_PROJECT_DIR
     - cp /usr/lib/gcc/x86_64-w64-mingw32/7.3-posix/libgcc_s_seh-1.dll $CI_PROJECT_DIR
@@ -113,8 +97,7 @@ coverage:
   script:
     - export MACOSX_DEPLOYMENT_TARGET="10.13"
     - export VELOREN_USERDATA_STRATEGY=executable
-    - cargo build --release -p veloren-voxygen --no-default-features --features default-publish
-    - cargo build --release -p veloren-server-cli
+    - cargo build --release --no-default-features --features default-publish
     - cp -r target/release/veloren-server-cli $CI_PROJECT_DIR
     - cp -r target/release/veloren-voxygen $CI_PROJECT_DIR
   artifacts:

.gitlab/CI/check.gitlab-ci.yml

@@ -5,11 +5,8 @@ code-quality:
   image: registry.gitlab.com/veloren/veloren-docker-ci/cache/quality:${CACHE_IMAGE_TAG}
   script:
     - ln -s /dockercache/target target
-    - rm -r target/debug/incremental/* || echo "all good" # TMP FIX FOR 2021-03-22-nightly
-    - cargo clippy --all-targets --locked --features="bin_compression,bin_csv,bin_graphviz,bin_bot,asset_tweak" -- -D warnings
-    # Ensure that the veloren-voxygen default-publish feature builds as it excludes some default features
-    - cargo clippy -p veloren-voxygen --locked --no-default-features --features="default-publish" -- -D warnings
-    - cargo fmt --all -- --check
+    - cat ./.gitlab/scripts/code-quality.sh
+    - source ./.gitlab/scripts/code-quality.sh
 
 security:
   extends: .recompile-branch
@@ -18,4 +15,5 @@ security:
   allow_failure: true
   script:
     - ln -s /dockercache/target target
-    - cargo audit
+    - cat ./.gitlab/scripts/security.sh
+    - source ./.gitlab/scripts/security.sh

.gitlab/CODEOWNERS

@@ -1,3 +1,4 @@
 */Cargo.toml @xMAC94x
 Cargo.toml @xMAC94x
-rust-toolchain @xMAC94x
+rust-toolchain @xMAC94x
+.gitlab/scripts/* @xMAC94x

.gitlab/scripts/benchmark.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+time cargo bench

.gitlab/scripts/ci.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+./.gitlab/scripts/security.sh
+./.gitlab/scripts/code-quality.sh
+./.gitlab/scripts/unittest.sh

.gitlab/scripts/code-quality.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+time cargo clippy --all-targets --locked --features="bin_compression,bin_csv,bin_graphviz,bin_bot,asset_tweak" -- -D warnings &&
+# Ensure that the veloren-voxygen default-publish feature builds as it excludes some default features
+time cargo clippy -p veloren-voxygen --locked --no-default-features --features="default-publish" -- -D warnings &&
+time cargo fmt --all -- --check

.gitlab/scripts/coverage.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+echo "modifying files in 5s, ctrl+c to abort" && sleep 5
+find ./* -name "Cargo.toml" -exec sed -i 's/, "simd"]/]/g' {} \;
+find ./* -name "Cargo.toml" -exec sed -i 's/"simd"]/]/g' {} \;
+sed -i 's/vek /#vek /g' ./Cargo.toml;
+time cargo tarpaulin --skip-clean -v -- --test-threads=2;

.gitlab/scripts/env.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+# exports default env variables in CI
+export DISABLE_GIT_LFS_CHECK=true
+export VELOREN_ASSETS="assets"
+export RUSTFLAGS="-D warnings"
+export SHADERC_LIB_DIR=/shaderc/combined/

.gitlab/scripts/linux.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+export VELOREN_USERDATA_STRATEGY=executable
+time cargo build --release --no-default-features --features default-publish &&
+# eveluate --bin instead, last time i checked (2021-07-14) it was 2 minutes slower on release (but faster on debug)

.gitlab/scripts/security.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+time cargo audit

.gitlab/scripts/unittest.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export VELOREN_ASSETS="$(pwd)/assets"
+time cargo test --package veloren-i18n --lib test_all_localizations -- --nocapture --ignored &&
+time cargo test --package veloren-common-assets asset_tweak::tests --features asset_tweak --lib &&
+time cargo test

.gitlab/scripts/windows.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+update-alternatives --set x86_64-w64-mingw32-gcc /usr/bin/x86_64-w64-mingw32-gcc-posix
+update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
+export VELOREN_USERDATA_STRATEGY=executable
+time cargo build --target=x86_64-pc-windows-gnu --release --no-default-features --features default-publish &&

server-cli/Cargo.toml

@@ -16,6 +16,8 @@ This package includes the official server CLI.
 
 [features]
 worldgen = ["server/worldgen"]
+# needed to stay compatible with voxygens format
+default-publish = ["default"]
 default = ["worldgen"]
 tracy = ["common-frontend/tracy"]
 plugins = ["server/plugins"]

world/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["Joshua Barretto <joshua.s.barretto@gmail.com>"]
 edition = "2018"
 
 [features]
-simd = ["vek/platform_intrinsics"]
+simd = ["vek/platform_intrinsics", "packed_simd"]
 bin_compression = ["lz-fear", "deflate", "flate2", "image/jpeg", "num-traits", "fallible-iterator", "kiddo", "clap", "rstar"]
 
 default = ["simd"]
@@ -31,7 +31,7 @@ tracing = { version = "0.1", default-features = false }
 rand = "0.8"
 rand_chacha = "0.3"
 arr_macro = "0.1.2"
-packed_simd = { package = "packed_simd_2", version = "0.3.5" }
+packed_simd = { package = "packed_simd_2", version = "0.3.5", optional = true }
 rayon = "1.5"
 serde = { version = "1.0.110", features = ["derive"] }
 ron = { version = "0.6", default-features = false }

world/src/sim/erosion.rs

@@ -13,7 +13,7 @@ use itertools::izip;
 use noise::NoiseFn;
 use num::{Float, Zero};
 use ordered_float::NotNan;
-use packed_simd::m32;
+#[cfg(feature = "simd")] use packed_simd::m32;
 use rayon::prelude::*;
 use std::{
     cmp::{Ordering, Reverse},
@@ -607,6 +607,20 @@ fn get_max_slope(
         .into_boxed_slice()
 }
 
+// simd alternative
+#[cfg(not(feature = "simd"))]
+#[derive(Copy, Clone)]
+#[allow(non_camel_case_types)]
+struct m32(u32);
+#[cfg(not(feature = "simd"))]
+impl m32 {
+    #[inline]
+    fn new(x: bool) -> Self { if x { Self(u32::MAX) } else { Self(u32::MIN) } }
+
+    #[inline]
+    fn test(&self) -> bool { self.0 != 0 }
+}
+
 /// Erode all chunks by amount.
 ///
 /// Our equation is: