ops(docker): prune unused cache on deploys and nightly
CI / check (push) Failing after 23s
CI / deploy (push) Skipped
CI / publish-container (push) Skipped

Add scripts/docker-prune.sh (build cache >72h capped at 4g, unreferenced images >7d, stopped containers >24h; never volumes), run it after every CI deploy and compose update, and schedule a nightly prune from the host-side jobs-worker. Tighten the deployment contract tests to assert the scoped-prune boundaries.
This commit is contained in:
openhands committed 2026-09-13 13:04:03 +02:00
1 parent fb68df3ba9
commit 47917bb63b
6 files changed
+102 -5

No files matched your search

+3 -2
View File
@@ -193,5 +193,6 @@ while IFS= read -r tag; do
if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi
fi fi
done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms) done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms)
docker builder prune -af --filter "until=72h" --max-used-space=4g || true # Reclaim build cache, unreferenced images and long-stopped containers. Never
docker image prune -f --filter "until=168h" || true # volumes; retention boundaries are enforced inside docker-prune.sh.
bash "$deploy_dir/scripts/docker-prune.sh" || true
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
# Reclaim Docker's unused cache so host storage stays bounded.
#
# Safe scopes only, by design:
# - BuildKit cache older than 72h, hard-capped at 4 GB (Debian /pnpm store is
# shared across builds; everything newer than that speeds up rebuilds).
# - Images referenced by NO running/stopped container and older than 7 days
# (covers stale epicnext-cms sha tags, old mariadb/byparr pulls, etc.).
# - Containers stopped for more than 24h.
#
# Volumes are NEVER pruned here: mariadb-turbo-data is a database. This script
# is idempotent and exits 0 when Docker is unavailable.
set -Eeuo pipefail
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
LOG_DIR="${LOG_DIR:-$DIR/logs}"
mkdir -p "$LOG_DIR"
LOG_FILE="$LOG_DIR/docker-prune.log"
now() { date '+%Y-%m-%d %H:%M:%S'; }
command -v docker >/dev/null 2>&1 || {
printf '[%s] docker CLI unavailable; nothing to prune\n' "$(now)" >>"$LOG_FILE"
exit 0
}
printf '\n[%s] === docker prune start ===\n' "$(now)" >>"$LOG_FILE"
docker system df >>"$LOG_FILE" 2>&1 || true
docker builder prune -af --filter "until=72h" --max-used-space=4g >>"$LOG_FILE" 2>&1 || true
docker image prune -af --filter "until=168h" >>"$LOG_FILE" 2>&1 || true
docker container prune -f --filter "until=24h" >>"$LOG_FILE" 2>&1 || true
printf '\n[%s] === docker prune complete ===\n' "$(now)" >>"$LOG_FILE"
docker system df >>"$LOG_FILE" 2>&1 || true
+4
View File
@@ -150,3 +150,7 @@ done
printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp" printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp"
mv "$history.tmp" "$history" mv "$history.tmp" "$history"
log "Keeping the two latest releases; in-use images and persistent volumes are preserved." log "Keeping the two latest releases; in-use images and persistent volumes are preserved."
# Reclaim build cache + unreferenced images + long-stopped containers only;
# the scoped retention is enforced inside docker-prune.sh (never volumes).
bash "$DIR/scripts/docker-prune.sh" >>"$LOG_FILE" 2>&1 || log "Docker prune reported an error (see $LOG_FILE)"
log "Pruned unused Docker cache."
+41
View File
@@ -232,6 +232,38 @@ async function cleanupOldSessions(): Promise<void> {
} }
} }
/** Host-side: reclaim Docker's unused cache (build cache, unreferenced images,
* stopped containers). Volumes and in-use images are never touched. No-op when
* docker or the prune script is unavailable. */
async function pruneDockerCache(): Promise<void> {
const { access } = await import("node:fs/promises");
const { resolve } = await import("node:path");
const { spawn } = await import("node:child_process");
const script = resolve(process.cwd(), "scripts", "docker-prune.sh");
try {
await access(script);
} catch {
return;
}
await new Promise<void>((resolvePromise) => {
const child = spawn("bash", [script], { stdio: "ignore" });
child.on("error", (err) =>
captureWorkerError(
err,
"Docker prune could not start (is bash on PATH?)",
),
);
child.on("close", (code) => {
if (code !== 0)
captureWorkerError(
new Error(`docker-prune.sh exited ${code}`),
"Docker prune failed",
);
resolvePromise();
});
});
}
async function publishScheduledArticles(): Promise<void> { async function publishScheduledArticles(): Promise<void> {
try { try {
const now = new Date(); const now = new Date();
@@ -305,6 +337,15 @@ async function main() {
}); });
logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" }); logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" });
new Cron("0 5 * * *", () => {
pruneDockerCache().catch((e) =>
captureWorkerError(e, "Docker prune error"),
);
});
logger.info("Scheduled: Docker cache prune (daily 05:00)", {
module: "jobs",
});
new Cron("*/5 * * * *", () => { new Cron("*/5 * * * *", () => {
checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error")); checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error"));
}); });
+9 -1
View File
@@ -26,10 +26,18 @@ it("preserves production runtime configuration and recent cache", () => {
expect(deploy).toContain("/var/www/Gamedata:/var/www/Gamedata"); expect(deploy).toContain("/var/www/Gamedata:/var/www/Gamedata");
expect(deploy).toContain("/app/storage"); expect(deploy).toContain("/app/storage");
expect(deploy).not.toContain("--env-file"); expect(deploy).not.toContain("--env-file");
expect(deploy).toContain( // The scoped prune lives in docker-prune.sh; deploys invoke it for both CI
// and scheduled updates. It reclaims build cache + old unreferenced images
// but never touches volumes.
expect(deploy).toContain("bash \"$deploy_dir/scripts/docker-prune.sh\"");
const prune = readFileSync("scripts/docker-prune.sh", "utf8");
expect(prune).toContain(
'docker builder prune -af --filter "until=72h" --max-used-space=4g', 'docker builder prune -af --filter "until=72h" --max-used-space=4g',
); );
expect(prune).toContain('docker image prune -af --filter "until=168h"');
expect(prune).toContain('docker container prune -f --filter "until=24h"');
expect(deploy).not.toContain("docker volume prune"); expect(deploy).not.toContain("docker volume prune");
expect(prune).not.toContain("docker volume prune");
}); });
it("builds the checked out source without fetching a moving remote branch", () => { it("builds the checked out source without fetching a moving remote branch", () => {
const dockerfile = readFileSync("Dockerfile", "utf8"); const dockerfile = readFileSync("Dockerfile", "utf8");
+12 -2
View File
@@ -46,6 +46,10 @@ function simulate(scenario: string) {
resolve(root, "scripts/docker-update.sh"), resolve(root, "scripts/docker-update.sh"),
join(dir, "scripts/docker-update.sh"), join(dir, "scripts/docker-update.sh"),
); );
copyFileSync(
resolve(root, "scripts/docker-prune.sh"),
join(dir, "scripts/docker-prune.sh"),
);
writeFileSync(join(dir, ".env"), "HOTEL_NAME=Test\n"); writeFileSync(join(dir, ".env"), "HOTEL_NAME=Test\n");
if (scenario.startsWith("saved-")) if (scenario.startsWith("saved-"))
writeFileSync( writeFileSync(
@@ -93,7 +97,7 @@ describe("Docker clone updates", () => {
expect(r.calls).toContain( expect(r.calls).toContain(
`docker pull registry.test/team/cms:${sha}-migrations`, `docker pull registry.test/team/cms:${sha}-migrations`,
); );
expect(r.calls).not.toContain("docker build"); expect(r.calls).not.toMatch(/docker build\s/);
expect(r.calls).not.toContain("docker compose build"); expect(r.calls).not.toContain("docker compose build");
expect(r.calls).toContain("target=/app/.env,readonly"); expect(r.calls).toContain("target=/app/.env,readonly");
}); });
@@ -140,7 +144,13 @@ describe("Docker clone updates", () => {
); );
expect(r.calls).toContain("https://example.test/api/health"); expect(r.calls).toContain("https://example.test/api/health");
expect(r.output).toContain(`Verified release ${sha}`); expect(r.output).toContain(`Verified release ${sha}`);
expect(r.calls).not.toContain("prune"); // A successful update runs the scoped prune (build cache + unreferenced
// images + stopped containers), never a global or volume prune.
expect(r.calls).toContain("docker builder prune");
expect(r.calls).toContain("docker image prune");
expect(r.calls).toContain("docker container prune");
expect(r.calls).not.toContain("docker volume prune");
expect(r.calls).not.toContain("docker system prune");
}); });
it.each([ it.each([
"dirty", "dirty",