ci: reduce build time (#3519)

frontend/scripts/code_generation/env/generate_env.cmd

@@ -1,17 +0,0 @@
-@echo off
-
-REM Store the current working directory
-set "original_dir=%CD%"
-
-REM Change the current working directory to the script's location
-cd /d "%~dp0"
-
-REM Navigate to the project root
-cd ..\..\..\appflowy_flutter
-
-REM Navigate to the appflowy_flutter directory and generate files
-echo Generating env files
-call flutter packages pub get >nul 2>&1 && call dart run build_runner clean && call dart run build_runner build --delete-conflicting-outputs
-echo Done generating env files
-
-cd /d "%original_dir%"

frontend/scripts/code_generation/env/generate_env.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-
-# Store the current working directory
-original_dir=$(pwd)
-
-cd "$(dirname "$0")"
-
-# Navigate to the project root
-cd ../../../appflowy_flutter
-
-# Navigate to the appflowy_flutter directory and generate files
-echo "Generating env files"
-# flutter clean >/dev/null 2>&1 && flutter packages pub get >/dev/null 2>&1 && dart run build_runner clean &&
-flutter packages pub get >/dev/null 2>&1
-dart run build_runner clean && dart run build_runner build --delete-conflicting-outputs
-echo "Done generating env files"
-
-# Return to the original directory
-cd "$original_dir"

frontend/scripts/code_generation/flowy_icons/generate_flowy_icons.cmd

@@ -15,7 +15,7 @@ REM   the appflowy_flutter/assets/translation directory
 echo Copying resources/flowy_icons to appflowy_flutter/assets/flowy_icons
 xcopy /E /Y /I ..\resources\flowy_icons assets\flowy_icons
 
-call flutter packages pub get
+REM call flutter packages pub get
 
 echo Generating FlowySvg class
 call dart run flowy_svg

frontend/scripts/code_generation/flowy_icons/generate_flowy_icons.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+no_pub_get=false
+
+while getopts 's' flag; do
+  case "${flag}" in
+    s) no_pub_get=true ;;
+  esac
+done
+
 echo "Generating flowy icon files"
 
 # Store the current working directory
@@ -14,7 +22,9 @@ rm -rf assets/flowy_icons/
 mkdir -p assets/flowy_icons/
 rsync -r ../resources/flowy_icons/ assets/flowy_icons/
 
-flutter packages pub get
+if [ "$no_pub_get" = false ]; then
+  flutter packages pub get
+fi
 
 echo "Generating FlowySvg classes"
 dart run flowy_svg

frontend/scripts/code_generation/freezed/generate_freezed.cmd

@@ -11,7 +11,8 @@ cd ..\..\..\appflowy_flutter
 
 REM Navigate to the appflowy_flutter directory and generate files
 echo Generating files for appflowy_flutter
-call flutter clean >nul 2>&1 && call flutter packages pub get >nul 2>&1 && call dart run build_runner clean && call dart run build_runner build -d
+REM call flutter packages pub get
+call dart run build_runner clean && call dart run build_runner build -d
 echo Done generating files for appflowy_flutter
 
 echo Generating files for packages
@@ -24,7 +25,7 @@ for /D %%d in (*) do (
     if exist "pubspec.yaml" (
         echo Generating freezed files in %%d...
         echo Please wait while we clean the project and fetch the dependencies.
-        call flutter clean >nul 2>&1 && call flutter packages pub get >nul 2>&1 && call dart run build_runner clean && call dart run build_runner build -d
+        call dart run build_runner clean && call dart run build_runner build -d
         echo Done running build command in %%d
     ) else (
         echo No pubspec.yaml found in %%d, it can't be a Dart project. Skipping.

frontend/scripts/code_generation/freezed/generate_freezed.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+no_pub_get=false
+
+while getopts 's' flag; do
+  case "${flag}" in
+    s) no_pub_get=true ;;
+  esac
+done
+
 # Store the current working directory
 original_dir=$(pwd)
 
@@ -10,9 +18,12 @@ cd ../../../appflowy_flutter
 
 # Navigate to the appflowy_flutter directory and generate files
 echo "Generating files for appflowy_flutter"
-# flutter clean >/dev/null 2>&1 && flutter packages pub get >/dev/null 2>&1 && dart run build_runner clean &&
-flutter packages pub get >/dev/null 2>&1
-dart run build_runner build -d
+
+if [ "$no_pub_get" = false ]; then
+  flutter packages pub get >/dev/null 2>&1
+fi
+
+dart run build_runner clean && dart run build_runner build -d
 echo "Done generating files for appflowy_flutter"
 
 echo "Generating files for packages"
@@ -25,8 +36,10 @@ for d in */; do
   if [ -f "pubspec.yaml" ]; then
     echo "Generating freezed files in $d..."
     echo "Please wait while we clean the project and fetch the dependencies."
-    flutter packages pub get >/dev/null 2>&1
-    dart run build_runner build -d
+    if [ "$no_pub_get" = false ]; then
+      flutter packages pub get >/dev/null 2>&1
+    fi
+    dart run build_runner clean && dart run build_runner build -d
     echo "Done running build command in $d"
   else
     echo "No pubspec.yaml found in $d, it can\'t be a Dart project. Skipping."

frontend/scripts/code_generation/generate.cmd

@@ -26,15 +26,6 @@ call generate_freezed.cmd %*
 REM Return to the main script directory
 cd ..
 
-echo Generating env files using build_runner
-cd env
-REM Allow execution permissions on CI
-chmod +x generate_env.cmd
-call generate_env.cmd %*
-
-REM Return to the main script directory
-cd ..
-
 echo Generating svg files using flowy_svg
 cd flowy_icons
 REM Allow execution permissions on CI

frontend/scripts/code_generation/generate.sh

@@ -26,15 +26,6 @@ chmod +x ./generate_freezed.sh
 # Return to the main script directory
 cd ..
 
-echo "Generating env files using build_runner"
-cd env
-# Allow execution permissions on CI
-chmod +x ./generate_env.sh
-./generate_env.sh "$@"
-
-# Return to the main script directory
-cd ..
-
 echo "Generating svg files using flowy_svg"
 cd flowy_icons
 # Allow execution permissions on CI

frontend/scripts/code_generation/language_files/generate_language_files.cmd

@@ -15,7 +15,7 @@ REM   the appflowy_flutter/assets/translation directory
 echo Copying resources/translations to appflowy_flutter/assets/translations
 xcopy /E /Y /I ..\resources\translations assets\translations
 
-call flutter packages pub get
+REM call flutter packages pub get
 
 echo Specifying source directory for AppFlowy Localizations.
 call dart run easy_localization:generate -S assets/translations/

frontend/scripts/code_generation/language_files/generate_language_files.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+no_pub_get=false
+
+while getopts 's' flag; do
+  case "${flag}" in
+    s) no_pub_get=true ;;
+  esac
+done
+
 echo "Generating language files"
 
 # Store the current working directory
@@ -16,7 +24,9 @@ rm -rf assets/translations/
 mkdir -p assets/translations/
 cp -f ../resources/translations/*.json assets/translations/
 
-flutter packages pub get
+if [ "$no_pub_get" = false ]; then
+  flutter packages pub get
+fi
 
 echo "Specifying source directory for AppFlowy Localizations."
 dart run easy_localization:generate -S assets/translations/

frontend/scripts/docker-buildfiles/Dockerfile

@@ -47,8 +47,10 @@ RUN flutter doctor
 RUN dart pub global activate protoc_plugin 20.0.1
 
 # Install build dependencies for AppFlowy
-RUN sudo pacman -S --noconfirm git libkeybinder3 sqlite clang rsync libnotify
-RUN source ~/.cargo/env && cargo install --force cargo-make duckscript_cli
+RUN yay -S --noconfirm jemalloc4 cargo-make cargo-binstall
+RUN sudo pacman -S --noconfirm git libkeybinder3 sqlite clang rsync libnotify rocksdb zstd
+RUN sudo ln -s /usr/bin/sha1sum /usr/bin/shasum
+RUN source ~/.cargo/env && cargo binstall duckscript_cli -y
 
 # Build AppFlowy
 COPY . /appflowy
@@ -58,7 +60,7 @@ RUN cd frontend && \
     source ~/.cargo/env && \
     cargo make appflowy-flutter-deps-tools && \
     cargo make flutter_clean && \
-    cargo make -p production-linux-x86_64 appflowy-linux
+    OPENSSL_STATIC=1 ZSTD_SYS_USE_PKG_CONFIG=1 ROCKSDB_LIB_DIR="/usr/lib/" cargo make -p production-linux-x86_64 appflowy-linux
 
 
 #================

frontend/scripts/makefile/flutter.toml

@@ -94,6 +94,14 @@ run_task = { name = [
 ] }
 script_runner = "@shell"
 
+[tasks.appflowy-make-product-dev]
+run_task = { name = [
+  "set-app-version",
+  "flutter-build",
+  "copy-to-product",
+] }
+
+
 [tasks.copy-to-product]
 mac_alias = "copy-to-product-macos"
 windows_alias = "copy-to-product-windows"
@@ -227,8 +235,9 @@ script = [
   cd appflowy_flutter
   flutter clean
   flutter pub get
+  flutter packages pub get 
   cd ../
-  sh scripts/code_generation/generate.sh
+  sh scripts/code_generation/generate.sh -s
   """
 ]
 
@@ -239,6 +248,7 @@ script = [
   cd ./appflowy_flutter/
   exec cmd.exe /c flutter clean
   exec cmd.exe /c flutter pub get
+  exec cmd.exe /c flutter packages pub get
   cd ../
   exec scripts/code_generation/generate.cmd
   """,

frontend/scripts/makefile/tests.toml

@@ -36,6 +36,31 @@ cd appflowy_flutter
 flutter test --dart-define=RUST_LOG=${RUST_LOG} -j, --concurrency=1 --coverage
 '''
 
+[tasks.dart_unit_test_no_build]
+script = '''
+cargo make --profile test-macos-$(uname -m) run_dart_unit_test_no_build
+'''
+
+[tasks.dart_unit_test_no_build.windows]
+script = '''
+cargo make --profile test-windows run_dart_unit_test_no_build
+'''
+
+[tasks.dart_unit_test_no_build.linux]
+script = '''
+cargo make --profile test-linux run_dart_unit_test_no_build
+'''
+
+[tasks.run_dart_unit_test_no_build]
+env = { RUST_LOG = "info" }
+dependencies = ["copy-from-build-to-sandbox-folder"]
+description = "Run flutter unit tests"
+script = '''
+cd appflowy_flutter
+flutter test --dart-define=RUST_LOG=${RUST_LOG} -j, --concurrency=1 --coverage
+'''
+script_runner = "@shell"
+
 [tasks.rust_unit_test]
 run_task = { name = ["rust_lib_unit_test", "shared_lib_unit_test"] }
 
@@ -323,4 +348,49 @@ script = [
     ${dest}
   """,
 ]
-script_runner = "@duckscript"
+script_runner = "@duckscript"
+
+[tasks.copy-from-build-to-sandbox-folder]
+mac_alias = "copy-from-build-to-sandbox-folder-macos"
+windows_alias = "copy-from-build-to-sandbox-folder-windows"
+linux_alias = "copy-from-build-to-sandbox-folder-linux"
+
+[tasks.copy-from-build-to-sandbox-folder-windows]
+private = true
+script = [
+  """
+    # Copy the appflowy_backend lib to system temp directory for flutter unit test.
+    lib = set ${LIB_NAME}.${TEST_LIB_EXT}
+    dest = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/appflowy_flutter/.sandbox/${lib}
+    dart_ffi_dir= set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/appflowy_flutter/windows/flutter/dart_ffi
+    rm ${dest}
+
+    cp ${dart_ffi_dir}/${lib} ${dest}
+  """,
+]
+script_runner = "@duckscript"
+
+[tasks.copy-from-build-to-sandbox-folder-macos]
+private = true
+script = [
+  """
+    echo "This is not possible for mac because the tests wont work with .a lib"
+    exit 127
+  """,
+]
+script_runner = "@duckscript"
+
+[tasks.copy-from-build-to-sandbox-folder-linux]
+private = true
+script = [
+  """
+    # Copy the appflowy_backend lib to system temp directory for flutter unit test.
+    lib = set lib${LIB_NAME}.${TEST_LIB_EXT}
+    dest = set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/appflowy_flutter/.sandbox/${lib}
+    dart_ffi_dir= set ${CARGO_MAKE_WORKSPACE_WORKING_DIRECTORY}/appflowy_flutter/linux/flutter/dart_ffi
+    rm ${dest}
+
+    cp ${dart_ffi_dir}/${lib} ${dest}
+  """,
+]
+script_runner = "@duckscript"