ops(docker): prune unused cache on deploys and nightly
Add scripts/docker-prune.sh (build cache >72h capped at 4g, unreferenced images >7d, stopped containers >24h; never volumes), run it after every CI deploy and compose update, and schedule a nightly prune from the host-side jobs-worker. Tighten the deployment contract tests to assert the scoped-prune boundaries.
This commit is contained in:
1 parent
fb68df3ba9
commit
47917bb63b
6 files changed
+102
-5
No files matched your search
@@ -193,5 +193,6 @@ while IFS= read -r tag; do
|
|||||||
if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi
|
if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi
|
||||||
fi
|
fi
|
||||||
done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms)
|
done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms)
|
||||||
docker builder prune -af --filter "until=72h" --max-used-space=4g || true
|
# Reclaim build cache, unreferenced images and long-stopped containers. Never
|
||||||
docker image prune -f --filter "until=168h" || true
|
# volumes; retention boundaries are enforced inside docker-prune.sh.
|
||||||
|
bash "$deploy_dir/scripts/docker-prune.sh" || true
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Reclaim Docker's unused cache so host storage stays bounded.
|
||||||
|
#
|
||||||
|
# Safe scopes only, by design:
|
||||||
|
# - BuildKit cache older than 72h, hard-capped at 4 GB (Debian /pnpm store is
|
||||||
|
# shared across builds; everything newer than that speeds up rebuilds).
|
||||||
|
# - Images referenced by NO running/stopped container and older than 7 days
|
||||||
|
# (covers stale epicnext-cms sha tags, old mariadb/byparr pulls, etc.).
|
||||||
|
# - Containers stopped for more than 24h.
|
||||||
|
#
|
||||||
|
# Volumes are NEVER pruned here: mariadb-turbo-data is a database. This script
|
||||||
|
# is idempotent and exits 0 when Docker is unavailable.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||||
|
LOG_DIR="${LOG_DIR:-$DIR/logs}"
|
||||||
|
mkdir -p "$LOG_DIR"
|
||||||
|
LOG_FILE="$LOG_DIR/docker-prune.log"
|
||||||
|
now() { date '+%Y-%m-%d %H:%M:%S'; }
|
||||||
|
|
||||||
|
command -v docker >/dev/null 2>&1 || {
|
||||||
|
printf '[%s] docker CLI unavailable; nothing to prune\n' "$(now)" >>"$LOG_FILE"
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
printf '\n[%s] === docker prune start ===\n' "$(now)" >>"$LOG_FILE"
|
||||||
|
docker system df >>"$LOG_FILE" 2>&1 || true
|
||||||
|
|
||||||
|
docker builder prune -af --filter "until=72h" --max-used-space=4g >>"$LOG_FILE" 2>&1 || true
|
||||||
|
docker image prune -af --filter "until=168h" >>"$LOG_FILE" 2>&1 || true
|
||||||
|
docker container prune -f --filter "until=24h" >>"$LOG_FILE" 2>&1 || true
|
||||||
|
|
||||||
|
printf '\n[%s] === docker prune complete ===\n' "$(now)" >>"$LOG_FILE"
|
||||||
|
docker system df >>"$LOG_FILE" 2>&1 || true
|
||||||
@@ -150,3 +150,7 @@ done
|
|||||||
printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp"
|
printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp"
|
||||||
mv "$history.tmp" "$history"
|
mv "$history.tmp" "$history"
|
||||||
log "Keeping the two latest releases; in-use images and persistent volumes are preserved."
|
log "Keeping the two latest releases; in-use images and persistent volumes are preserved."
|
||||||
|
# Reclaim build cache + unreferenced images + long-stopped containers only;
|
||||||
|
# the scoped retention is enforced inside docker-prune.sh (never volumes).
|
||||||
|
bash "$DIR/scripts/docker-prune.sh" >>"$LOG_FILE" 2>&1 || log "Docker prune reported an error (see $LOG_FILE)"
|
||||||
|
log "Pruned unused Docker cache."
|
||||||
@@ -232,6 +232,38 @@ async function cleanupOldSessions(): Promise<void> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Host-side: reclaim Docker's unused cache (build cache, unreferenced images,
|
||||||
|
* stopped containers). Volumes and in-use images are never touched. No-op when
|
||||||
|
* docker or the prune script is unavailable. */
|
||||||
|
async function pruneDockerCache(): Promise<void> {
|
||||||
|
const { access } = await import("node:fs/promises");
|
||||||
|
const { resolve } = await import("node:path");
|
||||||
|
const { spawn } = await import("node:child_process");
|
||||||
|
const script = resolve(process.cwd(), "scripts", "docker-prune.sh");
|
||||||
|
try {
|
||||||
|
await access(script);
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await new Promise<void>((resolvePromise) => {
|
||||||
|
const child = spawn("bash", [script], { stdio: "ignore" });
|
||||||
|
child.on("error", (err) =>
|
||||||
|
captureWorkerError(
|
||||||
|
err,
|
||||||
|
"Docker prune could not start (is bash on PATH?)",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
child.on("close", (code) => {
|
||||||
|
if (code !== 0)
|
||||||
|
captureWorkerError(
|
||||||
|
new Error(`docker-prune.sh exited ${code}`),
|
||||||
|
"Docker prune failed",
|
||||||
|
);
|
||||||
|
resolvePromise();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function publishScheduledArticles(): Promise<void> {
|
async function publishScheduledArticles(): Promise<void> {
|
||||||
try {
|
try {
|
||||||
const now = new Date();
|
const now = new Date();
|
||||||
@@ -305,6 +337,15 @@ async function main() {
|
|||||||
});
|
});
|
||||||
logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" });
|
logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" });
|
||||||
|
|
||||||
|
new Cron("0 5 * * *", () => {
|
||||||
|
pruneDockerCache().catch((e) =>
|
||||||
|
captureWorkerError(e, "Docker prune error"),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
logger.info("Scheduled: Docker cache prune (daily 05:00)", {
|
||||||
|
module: "jobs",
|
||||||
|
});
|
||||||
|
|
||||||
new Cron("*/5 * * * *", () => {
|
new Cron("*/5 * * * *", () => {
|
||||||
checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error"));
|
checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error"));
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -26,10 +26,18 @@ it("preserves production runtime configuration and recent cache", () => {
|
|||||||
expect(deploy).toContain("/var/www/Gamedata:/var/www/Gamedata");
|
expect(deploy).toContain("/var/www/Gamedata:/var/www/Gamedata");
|
||||||
expect(deploy).toContain("/app/storage");
|
expect(deploy).toContain("/app/storage");
|
||||||
expect(deploy).not.toContain("--env-file");
|
expect(deploy).not.toContain("--env-file");
|
||||||
expect(deploy).toContain(
|
// The scoped prune lives in docker-prune.sh; deploys invoke it for both CI
|
||||||
|
// and scheduled updates. It reclaims build cache + old unreferenced images
|
||||||
|
// but never touches volumes.
|
||||||
|
expect(deploy).toContain("bash \"$deploy_dir/scripts/docker-prune.sh\"");
|
||||||
|
const prune = readFileSync("scripts/docker-prune.sh", "utf8");
|
||||||
|
expect(prune).toContain(
|
||||||
'docker builder prune -af --filter "until=72h" --max-used-space=4g',
|
'docker builder prune -af --filter "until=72h" --max-used-space=4g',
|
||||||
);
|
);
|
||||||
|
expect(prune).toContain('docker image prune -af --filter "until=168h"');
|
||||||
|
expect(prune).toContain('docker container prune -f --filter "until=24h"');
|
||||||
expect(deploy).not.toContain("docker volume prune");
|
expect(deploy).not.toContain("docker volume prune");
|
||||||
|
expect(prune).not.toContain("docker volume prune");
|
||||||
});
|
});
|
||||||
it("builds the checked out source without fetching a moving remote branch", () => {
|
it("builds the checked out source without fetching a moving remote branch", () => {
|
||||||
const dockerfile = readFileSync("Dockerfile", "utf8");
|
const dockerfile = readFileSync("Dockerfile", "utf8");
|
||||||
|
|||||||
@@ -46,6 +46,10 @@ function simulate(scenario: string) {
|
|||||||
resolve(root, "scripts/docker-update.sh"),
|
resolve(root, "scripts/docker-update.sh"),
|
||||||
join(dir, "scripts/docker-update.sh"),
|
join(dir, "scripts/docker-update.sh"),
|
||||||
);
|
);
|
||||||
|
copyFileSync(
|
||||||
|
resolve(root, "scripts/docker-prune.sh"),
|
||||||
|
join(dir, "scripts/docker-prune.sh"),
|
||||||
|
);
|
||||||
writeFileSync(join(dir, ".env"), "HOTEL_NAME=Test\n");
|
writeFileSync(join(dir, ".env"), "HOTEL_NAME=Test\n");
|
||||||
if (scenario.startsWith("saved-"))
|
if (scenario.startsWith("saved-"))
|
||||||
writeFileSync(
|
writeFileSync(
|
||||||
@@ -93,7 +97,7 @@ describe("Docker clone updates", () => {
|
|||||||
expect(r.calls).toContain(
|
expect(r.calls).toContain(
|
||||||
`docker pull registry.test/team/cms:${sha}-migrations`,
|
`docker pull registry.test/team/cms:${sha}-migrations`,
|
||||||
);
|
);
|
||||||
expect(r.calls).not.toContain("docker build");
|
expect(r.calls).not.toMatch(/docker build\s/);
|
||||||
expect(r.calls).not.toContain("docker compose build");
|
expect(r.calls).not.toContain("docker compose build");
|
||||||
expect(r.calls).toContain("target=/app/.env,readonly");
|
expect(r.calls).toContain("target=/app/.env,readonly");
|
||||||
});
|
});
|
||||||
@@ -140,7 +144,13 @@ describe("Docker clone updates", () => {
|
|||||||
);
|
);
|
||||||
expect(r.calls).toContain("https://example.test/api/health");
|
expect(r.calls).toContain("https://example.test/api/health");
|
||||||
expect(r.output).toContain(`Verified release ${sha}`);
|
expect(r.output).toContain(`Verified release ${sha}`);
|
||||||
expect(r.calls).not.toContain("prune");
|
// A successful update runs the scoped prune (build cache + unreferenced
|
||||||
|
// images + stopped containers), never a global or volume prune.
|
||||||
|
expect(r.calls).toContain("docker builder prune");
|
||||||
|
expect(r.calls).toContain("docker image prune");
|
||||||
|
expect(r.calls).toContain("docker container prune");
|
||||||
|
expect(r.calls).not.toContain("docker volume prune");
|
||||||
|
expect(r.calls).not.toContain("docker system prune");
|
||||||
});
|
});
|
||||||
it.each([
|
it.each([
|
||||||
"dirty",
|
"dirty",
|
||||||
|
|||||||
Reference in new issue
Block a user