From 682babb1c094de069eb4ca0cc67f66c86fc6806b Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 20 Jul 2026 18:45:24 +0200 Subject: [PATCH] perf: cache per-component builds in pack workflow (only rebuild changed repos; sparse catalogus clone) --- .gitea/workflows/pack.yaml | 146 ++++++++++++++++++++++--------------- 1 file changed, 88 insertions(+), 58 deletions(-) diff --git a/.gitea/workflows/pack.yaml b/.gitea/workflows/pack.yaml index 2d15b379..1b3dfb88 100644 --- a/.gitea/workflows/pack.yaml +++ b/.gitea/workflows/pack.yaml @@ -25,13 +25,13 @@ jobs: TOKEN: ${{ secrets.GITEA_TOKEN }} PACK_VERSION: pack-$(date +%Y%m%d-%H%M) FORCE: ${{ github.event.inputs.force }} + CACHE_DIR: /var/tmp/pack-cache steps: - name: Check for upstream changes id: check run: | set -e exec 2>&1 - # Map of repo -> ref to poll for the latest commit hash. CMS_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/EpicNext-Cms.git main 2>/dev/null | awk '{print $1}')" CAT_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git Beta-3 2>/dev/null | awk '{print $1}')" NITRO_REF="$(git ls-remote https://github.com/duckietm/Nitro-V3.git main 2>/dev/null | awk '{print $1}')" @@ -39,19 +39,29 @@ jobs: EMU_REF="$(git ls-remote https://github.com/duckietm/Polaris-Emulator.git main 2>/dev/null | awk '{print $1}')" COMBINED="${CMS_REF}:${CAT_REF}:${NITRO_REF}:${RENDER_REF}:${EMU_REF}" echo "Current refs: $COMBINED" - STATE="/var/tmp/pack-refs.state" + STATE="$CACHE_DIR/refs.state" + mkdir -p "$CACHE_DIR" PREV="$(cat "$STATE" 2>/dev/null || echo "")" + CHANGED="" if [ "${FORCE:-false}" = "true" ]; then - echo "Force rebuild requested" - echo "changed=true" >> "$GITHUB_OUTPUT" + CHANGED="force" elif [ "$COMBINED" != "$PREV" ]; then - echo "Upstream change detected (was: $PREV)" + CHANGED="upstream" + fi + if [ -n "$CHANGED" ]; then + echo "Change detected ($CHANGED)" echo "changed=true" >> "$GITHUB_OUTPUT" else echo "No upstream changes since last pack" echo "changed=false" >> "$GITHUB_OUTPUT" fi echo "$COMBINED" > "$STATE" + # Expose individual refs so later steps can compare per-component. + echo "cms_ref=$CMS_REF" >> "$GITHUB_OUTPUT" + echo "cat_ref=$CAT_REF" >> "$GITHUB_OUTPUT" + echo "nitro_ref=$NITRO_REF" >> "$GITHUB_OUTPUT" + echo "render_ref=$RENDER_REF" >> "$GITHUB_OUTPUT" + echo "emu_ref=$EMU_REF" >> "$GITHUB_OUTPUT" - name: Skip if unchanged if: steps.check.outputs.changed != 'true' @@ -66,81 +76,102 @@ jobs: exec 2>&1 echo "=== Building site pack ${PACK_VERSION} ===" WORK="/tmp/pack-build" - rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out" + rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out" "$CACHE_DIR" echo "WORK=$WORK" >> "$GITHUB_ENV" - - name: Pull CMS (this repo) + # ---- Per-component: pull only if ref changed, reuse cached build otherwise ---- + - name: CMS (this repo) if: steps.check.outputs.changed == 'true' run: | set -e - git clone --depth 1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WORK/src/cms" 2>/dev/null \ - || git -C "$WORK/src/cms" pull --ff-only 2>/dev/null + S="$CACHE_DIR/cms"; mkdir -p "$S" + if [ "${{ steps.check.outputs.cms_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then + git clone --depth 1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WORK/src/cms" 2>/dev/null \ + || git -C "$WORK/src/cms" pull --ff-only 2>/dev/null + echo "${{ steps.check.outputs.cms_ref }}" > "$S/ref" + rm -rf "$S/snapshot"; cp -r "$WORK/src/cms" "$S/snapshot" + else + cp -r "$S/snapshot" "$WORK/src/cms" + echo "CMS unchanged — reused cache" + fi echo "cms: $(git -C "$WORK/src/cms" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt" - - name: Pull Catalogus + - name: Catalogus (sparse) if: steps.check.outputs.changed == 'true' run: | set -e - git clone --depth 1 https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \ - || git -C "$WORK/src/catalogus" pull --ff-only 2>/dev/null + S="$CACHE_DIR/catalogus"; mkdir -p "$S" + if [ "${{ steps.check.outputs.cat_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then + rm -rf "$WORK/src/catalogus" + git clone --depth 1 --filter=blob:none --sparse \ + https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \ + || git -C "$WORK/src/catalogus" pull --ff-only 2>/dev/null + # Keep only what the pack actually needs (adjust paths as required). + git -C "$WORK/src/catalogus" sparse-checkout set . 2>/dev/null || true + echo "${{ steps.check.outputs.cat_ref }}" > "$S/ref" + rm -rf "$S/snapshot"; cp -r "$WORK/src/catalogus" "$S/snapshot" + else + cp -r "$S/snapshot" "$WORK/src/catalogus" + echo "Catalogus unchanged — reused cache" + fi echo "catalogus: $(git -C "$WORK/src/catalogus" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt" - - name: Pull Nitro-V3 + - name: Nitro-V3 if: steps.check.outputs.changed == 'true' run: | set -e - git clone --depth 1 https://github.com/duckietm/Nitro-V3.git "$WORK/src/nitro" 2>/dev/null \ - || git -C "$WORK/src/nitro" pull --ff-only 2>/dev/null - echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt" + S="$CACHE_DIR/nitro"; mkdir -p "$S" + if [ "${{ steps.check.outputs.nitro_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then + git clone --depth 1 https://github.com/duckietm/Nitro-V3.git "$WORK/src/nitro" 2>/dev/null \ + || git -C "$WORK/src/nitro" pull --ff-only 2>/dev/null + cd "$WORK/src/nitro" + yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1 + yarn build + rm -rf "$S/dist"; mkdir -p "$S/dist"; cp -r dist/* "$S/dist/" 2>/dev/null || true + echo "${{ steps.check.outputs.nitro_ref }}" > "$S/ref" + else + echo "Nitro unchanged — reused cached dist" + fi + mkdir -p "$WORK/out/nitro"; cp -r "$S/dist/"* "$WORK/out/nitro/" 2>/dev/null || true + echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt" - - name: Pull Nitro-Render-V3 + - name: Nitro-Render-V3 if: steps.check.outputs.changed == 'true' run: | set -e - git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \ - || git -C "$WORK/src/render" pull --ff-only 2>/dev/null - echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt" + S="$CACHE_DIR/render"; mkdir -p "$S" + if [ "${{ steps.check.outputs.render_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then + git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \ + || git -C "$WORK/src/render" pull --ff-only 2>/dev/null + cd "$WORK/src/render" + yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1 + rm -rf "$S/files"; mkdir -p "$S/files"; cp -r . "$S/files/" 2>/dev/null || true + rm -rf "$S/files/node_modules" + echo "${{ steps.check.outputs.render_ref }}" > "$S/ref" + else + echo "Render unchanged — reused cache" + fi + mkdir -p "$WORK/out/render"; cp -r "$S/files/"* "$WORK/out/render/" 2>/dev/null || true + echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt" - - name: Pull Polaris Emulator + - name: Polaris Emulator if: steps.check.outputs.changed == 'true' run: | set -e - git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \ - || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null - echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt" - - - name: Build Nitro-V3 client - if: steps.check.outputs.changed == 'true' - run: | - set -e - cd "$WORK/src/nitro" - yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1 - yarn build - mkdir -p "$WORK/out/nitro" - cp -r dist/* "$WORK/out/nitro/" 2>/dev/null || true - echo "nitro built" | tee -a "$WORK/build.txt" - - - name: Build Nitro-Render-V3 - if: steps.check.outputs.changed == 'true' - run: | - set -e - cd "$WORK/src/render" - yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1 - mkdir -p "$WORK/out/render" - cp -r . "$WORK/out/render/" 2>/dev/null || true - rm -rf "$WORK/out/render/node_modules" 2>/dev/null || true - echo "render built" | tee -a "$WORK/build.txt" - - - name: Build Polaris Emulator - if: steps.check.outputs.changed == 'true' - run: | - set -e - cd "$WORK/src/emulator" - mvn package -q -T 1C -o -Dmaven.test.skip=true 2>&1 \ - || mvn package -q -T 1C -Dmaven.test.skip=true 2>&1 - mkdir -p "$WORK/out/emulator" - find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$WORK/out/emulator/" \; 2>/dev/null || true - echo "emulator built" | tee -a "$WORK/build.txt" + S="$CACHE_DIR/emulator"; mkdir -p "$S" + if [ "${{ steps.check.outputs.emu_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then + git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \ + || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null + cd "$WORK/src/emulator" + mvn package -q -T 1C -o -Dmaven.test.skip=true 2>&1 \ + || mvn package -q -T 1C -Dmaven.test.skip=true 2>&1 + rm -f "$S"/*.jar; find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$S/" \; 2>/dev/null || true + echo "${{ steps.check.outputs.emu_ref }}" > "$S/ref" + else + echo "Emulator unchanged — reused cached jar" + fi + mkdir -p "$WORK/out/emulator"; cp "$S"/*.jar "$WORK/out/emulator/" 2>/dev/null || true + echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt" - name: Assemble Pack if: steps.check.outputs.changed == 'true' @@ -151,7 +182,6 @@ jobs: rm -rf "$WORK/out/cms/.git" "$WORK/out/cms/node_modules" 2>/dev/null || true cp -r "$WORK/src/catalogus" "$WORK/out/catalogus" 2>/dev/null || true rm -rf "$WORK/out/catalogus/.git" 2>/dev/null || true - # Manifest of exact component versions cp "$WORK/components.txt" "$WORK/out/COMPONENTS.txt" echo "Built at $(date -u)" > "$WORK/out/BUILD_INFO.txt" cat "$WORK/components.txt" >> "$WORK/out/BUILD_INFO.txt"