perf: cache per-component builds in pack workflow (only rebuild changed repos; sparse catalogus clone)

.gitea/workflows/pack.yaml

@@ -25,13 +25,13 @@ jobs:
       TOKEN: ${{ secrets.GITEA_TOKEN }}
       PACK_VERSION: pack-$(date +%Y%m%d-%H%M)
       FORCE: ${{ github.event.inputs.force }}
+      CACHE_DIR: /var/tmp/pack-cache
     steps:
       - name: Check for upstream changes
         id: check
         run: |
           set -e
           exec 2>&1
-          # Map of repo -> ref to poll for the latest commit hash.
           CMS_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/EpicNext-Cms.git main 2>/dev/null | awk '{print $1}')"
           CAT_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git Beta-3 2>/dev/null | awk '{print $1}')"
           NITRO_REF="$(git ls-remote https://github.com/duckietm/Nitro-V3.git main 2>/dev/null | awk '{print $1}')"
@@ -39,19 +39,29 @@ jobs:
           EMU_REF="$(git ls-remote https://github.com/duckietm/Polaris-Emulator.git main 2>/dev/null | awk '{print $1}')"
           COMBINED="${CMS_REF}:${CAT_REF}:${NITRO_REF}:${RENDER_REF}:${EMU_REF}"
           echo "Current refs: $COMBINED"
-          STATE="/var/tmp/pack-refs.state"
+          STATE="$CACHE_DIR/refs.state"
+          mkdir -p "$CACHE_DIR"
           PREV="$(cat "$STATE" 2>/dev/null || echo "")"
+          CHANGED=""
           if [ "${FORCE:-false}" = "true" ]; then
-            echo "Force rebuild requested"
+            CHANGED="force"
-            echo "changed=true" >> "$GITHUB_OUTPUT"
           elif [ "$COMBINED" != "$PREV" ]; then
-            echo "Upstream change detected (was: $PREV)"
+            CHANGED="upstream"
+          fi
+          if [ -n "$CHANGED" ]; then
+            echo "Change detected ($CHANGED)"
             echo "changed=true" >> "$GITHUB_OUTPUT"
           else
             echo "No upstream changes since last pack"
             echo "changed=false" >> "$GITHUB_OUTPUT"
           fi
           echo "$COMBINED" > "$STATE"
+          # Expose individual refs so later steps can compare per-component.
+          echo "cms_ref=$CMS_REF" >> "$GITHUB_OUTPUT"
+          echo "cat_ref=$CAT_REF" >> "$GITHUB_OUTPUT"
+          echo "nitro_ref=$NITRO_REF" >> "$GITHUB_OUTPUT"
+          echo "render_ref=$RENDER_REF" >> "$GITHUB_OUTPUT"
+          echo "emu_ref=$EMU_REF" >> "$GITHUB_OUTPUT"
 
       - name: Skip if unchanged
         if: steps.check.outputs.changed != 'true'
@@ -66,81 +76,102 @@ jobs:
           exec 2>&1
           echo "=== Building site pack ${PACK_VERSION} ==="
           WORK="/tmp/pack-build"
-          rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out"
+          rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out" "$CACHE_DIR"
           echo "WORK=$WORK" >> "$GITHUB_ENV"
 
-      - name: Pull CMS (this repo)
+      # ---- Per-component: pull only if ref changed, reuse cached build otherwise ----
+      - name: CMS (this repo)
         if: steps.check.outputs.changed == 'true'
         run: |
           set -e
+          S="$CACHE_DIR/cms"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.cms_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
             git clone --depth 1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WORK/src/cms" 2>/dev/null \
               || git -C "$WORK/src/cms" pull --ff-only 2>/dev/null
+            echo "${{ steps.check.outputs.cms_ref }}" > "$S/ref"
+            rm -rf "$S/snapshot"; cp -r "$WORK/src/cms" "$S/snapshot"
+          else
+            cp -r "$S/snapshot" "$WORK/src/cms"
+            echo "CMS unchanged — reused cache"
+          fi
           echo "cms: $(git -C "$WORK/src/cms" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
 
-      - name: Pull Catalogus
+      - name: Catalogus (sparse)
         if: steps.check.outputs.changed == 'true'
         run: |
           set -e
-          git clone --depth 1 https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \
+          S="$CACHE_DIR/catalogus"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.cat_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            rm -rf "$WORK/src/catalogus"
+            git clone --depth 1 --filter=blob:none --sparse \
+              https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \
               || git -C "$WORK/src/catalogus" pull --ff-only 2>/dev/null
+            # Keep only what the pack actually needs (adjust paths as required).
+            git -C "$WORK/src/catalogus" sparse-checkout set . 2>/dev/null || true
+            echo "${{ steps.check.outputs.cat_ref }}" > "$S/ref"
+            rm -rf "$S/snapshot"; cp -r "$WORK/src/catalogus" "$S/snapshot"
+          else
+            cp -r "$S/snapshot" "$WORK/src/catalogus"
+            echo "Catalogus unchanged — reused cache"
+          fi
           echo "catalogus: $(git -C "$WORK/src/catalogus" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
 
-      - name: Pull Nitro-V3
+      - name: Nitro-V3
         if: steps.check.outputs.changed == 'true'
         run: |
           set -e
+          S="$CACHE_DIR/nitro"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.nitro_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
             git clone --depth 1 https://github.com/duckietm/Nitro-V3.git "$WORK/src/nitro" 2>/dev/null \
               || git -C "$WORK/src/nitro" pull --ff-only 2>/dev/null
-          echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
-
-      - name: Pull Nitro-Render-V3
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
-          git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \
-            || git -C "$WORK/src/render" pull --ff-only 2>/dev/null
-          echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
-
-      - name: Pull Polaris Emulator
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
-          git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \
-            || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null
-          echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
-
-      - name: Build Nitro-V3 client
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
             cd "$WORK/src/nitro"
             yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
             yarn build
-          mkdir -p "$WORK/out/nitro"
+            rm -rf "$S/dist"; mkdir -p "$S/dist"; cp -r dist/* "$S/dist/" 2>/dev/null || true
-          cp -r dist/* "$WORK/out/nitro/" 2>/dev/null || true
+            echo "${{ steps.check.outputs.nitro_ref }}" > "$S/ref"
-          echo "nitro built" | tee -a "$WORK/build.txt"
+          else
+            echo "Nitro unchanged — reused cached dist"
+          fi
+          mkdir -p "$WORK/out/nitro"; cp -r "$S/dist/"* "$WORK/out/nitro/" 2>/dev/null || true
+          echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"
 
-      - name: Build Nitro-Render-V3
+      - name: Nitro-Render-V3
         if: steps.check.outputs.changed == 'true'
         run: |
           set -e
+          S="$CACHE_DIR/render"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.render_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \
+              || git -C "$WORK/src/render" pull --ff-only 2>/dev/null
             cd "$WORK/src/render"
             yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
-          mkdir -p "$WORK/out/render"
+            rm -rf "$S/files"; mkdir -p "$S/files"; cp -r . "$S/files/" 2>/dev/null || true
-          cp -r . "$WORK/out/render/" 2>/dev/null || true
+            rm -rf "$S/files/node_modules"
-          rm -rf "$WORK/out/render/node_modules" 2>/dev/null || true
+            echo "${{ steps.check.outputs.render_ref }}" > "$S/ref"
-          echo "render built" | tee -a "$WORK/build.txt"
+          else
+            echo "Render unchanged — reused cache"
+          fi
+          mkdir -p "$WORK/out/render"; cp -r "$S/files/"* "$WORK/out/render/" 2>/dev/null || true
+          echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"
 
-      - name: Build Polaris Emulator
+      - name: Polaris Emulator
         if: steps.check.outputs.changed == 'true'
         run: |
           set -e
+          S="$CACHE_DIR/emulator"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.emu_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \
+              || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null
             cd "$WORK/src/emulator"
             mvn package -q -T 1C -o -Dmaven.test.skip=true 2>&1 \
               || mvn package -q -T 1C -Dmaven.test.skip=true 2>&1
-          mkdir -p "$WORK/out/emulator"
+            rm -f "$S"/*.jar; find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$S/" \; 2>/dev/null || true
-          find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$WORK/out/emulator/" \; 2>/dev/null || true
+            echo "${{ steps.check.outputs.emu_ref }}" > "$S/ref"
-          echo "emulator built" | tee -a "$WORK/build.txt"
+          else
+            echo "Emulator unchanged — reused cached jar"
+          fi
+          mkdir -p "$WORK/out/emulator"; cp "$S"/*.jar "$WORK/out/emulator/" 2>/dev/null || true
+          echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"
 
       - name: Assemble Pack
         if: steps.check.outputs.changed == 'true'
@@ -151,7 +182,6 @@ jobs:
           rm -rf "$WORK/out/cms/.git" "$WORK/out/cms/node_modules" 2>/dev/null || true
           cp -r "$WORK/src/catalogus" "$WORK/out/catalogus" 2>/dev/null || true
           rm -rf "$WORK/out/catalogus/.git" 2>/dev/null || true
-          # Manifest of exact component versions
           cp "$WORK/components.txt" "$WORK/out/COMPONENTS.txt"
           echo "Built at $(date -u)" > "$WORK/out/BUILD_INFO.txt"
           cat "$WORK/components.txt" >> "$WORK/out/BUILD_INFO.txt"