perf: cache per-component builds in pack workflow (only rebuild changed repos; sparse catalogus clone)

2026-07-20 18:45:24 +02:00
1 parent 8b5999970b
commit 682babb1c0
@@ -25,13 +25,13 @@ jobs:
      TOKEN: ${{ secrets.GITEA_TOKEN }}
      PACK_VERSION: pack-$(date +%Y%m%d-%H%M)
      FORCE: ${{ github.event.inputs.force }}
+      CACHE_DIR: /var/tmp/pack-cache
    steps:
      - name: Check for upstream changes
        id: check
        run: |
          set -e
          exec 2>&1
-          # Map of repo -> ref to poll for the latest commit hash.
          CMS_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/EpicNext-Cms.git main 2>/dev/null | awk '{print $1}')"
          CAT_REF="$(git ls-remote https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git Beta-3 2>/dev/null | awk '{print $1}')"
          NITRO_REF="$(git ls-remote https://github.com/duckietm/Nitro-V3.git main 2>/dev/null | awk '{print $1}')"
@@ -39,19 +39,29 @@ jobs:
          EMU_REF="$(git ls-remote https://github.com/duckietm/Polaris-Emulator.git main 2>/dev/null | awk '{print $1}')"
          COMBINED="${CMS_REF}:${CAT_REF}:${NITRO_REF}:${RENDER_REF}:${EMU_REF}"
          echo "Current refs: $COMBINED"
-          STATE="/var/tmp/pack-refs.state"
+          STATE="$CACHE_DIR/refs.state"
+          mkdir -p "$CACHE_DIR"
          PREV="$(cat "$STATE" 2>/dev/null || echo "")"
+          CHANGED=""
          if [ "${FORCE:-false}" = "true" ]; then
-            echo "Force rebuild requested"
-            echo "changed=true" >> "$GITHUB_OUTPUT"
+            CHANGED="force"
          elif [ "$COMBINED" != "$PREV" ]; then
-            echo "Upstream change detected (was: $PREV)"
+            CHANGED="upstream"
+          fi
+          if [ -n "$CHANGED" ]; then
+            echo "Change detected ($CHANGED)"
            echo "changed=true" >> "$GITHUB_OUTPUT"
          else
            echo "No upstream changes since last pack"
            echo "changed=false" >> "$GITHUB_OUTPUT"
          fi
          echo "$COMBINED" > "$STATE"
+          # Expose individual refs so later steps can compare per-component.
+          echo "cms_ref=$CMS_REF" >> "$GITHUB_OUTPUT"
+          echo "cat_ref=$CAT_REF" >> "$GITHUB_OUTPUT"
+          echo "nitro_ref=$NITRO_REF" >> "$GITHUB_OUTPUT"
+          echo "render_ref=$RENDER_REF" >> "$GITHUB_OUTPUT"
+          echo "emu_ref=$EMU_REF" >> "$GITHUB_OUTPUT"

      - name: Skip if unchanged
        if: steps.check.outputs.changed != 'true'
@@ -66,81 +76,102 @@ jobs:
          exec 2>&1
          echo "=== Building site pack ${PACK_VERSION} ==="
          WORK="/tmp/pack-build"
-          rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out"
+          rm -rf "$WORK" && mkdir -p "$WORK/src" "$WORK/out" "$CACHE_DIR"
          echo "WORK=$WORK" >> "$GITHUB_ENV"

-      - name: Pull CMS (this repo)
+      # ---- Per-component: pull only if ref changed, reuse cached build otherwise ----
+      - name: CMS (this repo)
        if: steps.check.outputs.changed == 'true'
        run: |
          set -e
-          git clone --depth 1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WORK/src/cms" 2>/dev/null \
-            || git -C "$WORK/src/cms" pull --ff-only 2>/dev/null
+          S="$CACHE_DIR/cms"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.cms_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 "${{ gitea.server_url }}/${{ gitea.repository }}.git" "$WORK/src/cms" 2>/dev/null \
+              || git -C "$WORK/src/cms" pull --ff-only 2>/dev/null
+            echo "${{ steps.check.outputs.cms_ref }}" > "$S/ref"
+            rm -rf "$S/snapshot"; cp -r "$WORK/src/cms" "$S/snapshot"
+          else
+            cp -r "$S/snapshot" "$WORK/src/cms"
+            echo "CMS unchanged — reused cache"
+          fi
          echo "cms: $(git -C "$WORK/src/cms" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"

-      - name: Pull Catalogus
+      - name: Catalogus (sparse)
        if: steps.check.outputs.changed == 'true'
        run: |
          set -e
-          git clone --depth 1 https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \
-            || git -C "$WORK/src/catalogus" pull --ff-only 2>/dev/null
+          S="$CACHE_DIR/catalogus"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.cat_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            rm -rf "$WORK/src/catalogus"
+            git clone --depth 1 --filter=blob:none --sparse \
+              https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git "$WORK/src/catalogus" 2>/dev/null \
+              || git -C "$WORK/src/catalogus" pull --ff-only 2>/dev/null
+            # Keep only what the pack actually needs (adjust paths as required).
+            git -C "$WORK/src/catalogus" sparse-checkout set . 2>/dev/null || true
+            echo "${{ steps.check.outputs.cat_ref }}" > "$S/ref"
+            rm -rf "$S/snapshot"; cp -r "$WORK/src/catalogus" "$S/snapshot"
+          else
+            cp -r "$S/snapshot" "$WORK/src/catalogus"
+            echo "Catalogus unchanged — reused cache"
+          fi
          echo "catalogus: $(git -C "$WORK/src/catalogus" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"

-      - name: Pull Nitro-V3
+      - name: Nitro-V3
        if: steps.check.outputs.changed == 'true'
        run: |
          set -e
-          git clone --depth 1 https://github.com/duckietm/Nitro-V3.git "$WORK/src/nitro" 2>/dev/null \
-            || git -C "$WORK/src/nitro" pull --ff-only 2>/dev/null
-          echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
+          S="$CACHE_DIR/nitro"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.nitro_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 https://github.com/duckietm/Nitro-V3.git "$WORK/src/nitro" 2>/dev/null \
+              || git -C "$WORK/src/nitro" pull --ff-only 2>/dev/null
+            cd "$WORK/src/nitro"
+            yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
+            yarn build
+            rm -rf "$S/dist"; mkdir -p "$S/dist"; cp -r dist/* "$S/dist/" 2>/dev/null || true
+            echo "${{ steps.check.outputs.nitro_ref }}" > "$S/ref"
+          else
+            echo "Nitro unchanged — reused cached dist"
+          fi
+          mkdir -p "$WORK/out/nitro"; cp -r "$S/dist/"* "$WORK/out/nitro/" 2>/dev/null || true
+          echo "nitro: $(git -C "$WORK/src/nitro" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"

-      - name: Pull Nitro-Render-V3
+      - name: Nitro-Render-V3
        if: steps.check.outputs.changed == 'true'
        run: |
          set -e
-          git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \
-            || git -C "$WORK/src/render" pull --ff-only 2>/dev/null
-          echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
+          S="$CACHE_DIR/render"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.render_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 https://github.com/duckietm/Nitro_Render_V3.git "$WORK/src/render" 2>/dev/null \
+              || git -C "$WORK/src/render" pull --ff-only 2>/dev/null
+            cd "$WORK/src/render"
+            yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
+            rm -rf "$S/files"; mkdir -p "$S/files"; cp -r . "$S/files/" 2>/dev/null || true
+            rm -rf "$S/files/node_modules"
+            echo "${{ steps.check.outputs.render_ref }}" > "$S/ref"
+          else
+            echo "Render unchanged — reused cache"
+          fi
+          mkdir -p "$WORK/out/render"; cp -r "$S/files/"* "$WORK/out/render/" 2>/dev/null || true
+          echo "render: $(git -C "$WORK/src/render" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"

-      - name: Pull Polaris Emulator
+      - name: Polaris Emulator
        if: steps.check.outputs.changed == 'true'
        run: |
          set -e
-          git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \
-            || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null
-          echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null)" | tee -a "$WORK/components.txt"
-
-      - name: Build Nitro-V3 client
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
-          cd "$WORK/src/nitro"
-          yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
-          yarn build
-          mkdir -p "$WORK/out/nitro"
-          cp -r dist/* "$WORK/out/nitro/" 2>/dev/null || true
-          echo "nitro built" | tee -a "$WORK/build.txt"
-
-      - name: Build Nitro-Render-V3
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
-          cd "$WORK/src/render"
-          yarn install --frozen-lockfile --prefer-offline 2>&1 || yarn install 2>&1
-          mkdir -p "$WORK/out/render"
-          cp -r . "$WORK/out/render/" 2>/dev/null || true
-          rm -rf "$WORK/out/render/node_modules" 2>/dev/null || true
-          echo "render built" | tee -a "$WORK/build.txt"
-
-      - name: Build Polaris Emulator
-        if: steps.check.outputs.changed == 'true'
-        run: |
-          set -e
-          cd "$WORK/src/emulator"
-          mvn package -q -T 1C -o -Dmaven.test.skip=true 2>&1 \
-            || mvn package -q -T 1C -Dmaven.test.skip=true 2>&1
-          mkdir -p "$WORK/out/emulator"
-          find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$WORK/out/emulator/" \; 2>/dev/null || true
-          echo "emulator built" | tee -a "$WORK/build.txt"
+          S="$CACHE_DIR/emulator"; mkdir -p "$S"
+          if [ "${{ steps.check.outputs.emu_ref }}" != "$(cat "$S/ref" 2>/dev/null)" ]; then
+            git clone --depth 1 https://github.com/duckietm/Polaris-Emulator.git "$WORK/src/emulator" 2>/dev/null \
+              || git -C "$WORK/src/emulator" pull --ff-only 2>/dev/null
+            cd "$WORK/src/emulator"
+            mvn package -q -T 1C -o -Dmaven.test.skip=true 2>&1 \
+              || mvn package -q -T 1C -Dmaven.test.skip=true 2>&1
+            rm -f "$S"/*.jar; find target -maxdepth 1 -name 'Polaris-*-jar-with-dependencies.jar' -exec cp {} "$S/" \; 2>/dev/null || true
+            echo "${{ steps.check.outputs.emu_ref }}" > "$S/ref"
+          else
+            echo "Emulator unchanged — reused cached jar"
+          fi
+          mkdir -p "$WORK/out/emulator"; cp "$S"/*.jar "$WORK/out/emulator/" 2>/dev/null || true
+          echo "emulator: $(git -C "$WORK/src/emulator" rev-parse --short HEAD 2>/dev/null || echo cached)" | tee -a "$WORK/components.txt"

      - name: Assemble Pack
        if: steps.check.outputs.changed == 'true'
@@ -151,7 +182,6 @@ jobs:
          rm -rf "$WORK/out/cms/.git" "$WORK/out/cms/node_modules" 2>/dev/null || true
          cp -r "$WORK/src/catalogus" "$WORK/out/catalogus" 2>/dev/null || true
          rm -rf "$WORK/out/catalogus/.git" 2>/dev/null || true
-          # Manifest of exact component versions
          cp "$WORK/components.txt" "$WORK/out/COMPONENTS.txt"
          echo "Built at $(date -u)" > "$WORK/out/BUILD_INFO.txt"
          cat "$WORK/components.txt" >> "$WORK/out/BUILD_INFO.txt"