fix(cache): true LRU, stale-while-revalidate and cross-process invalidation

The in-process cache was a FIFO of 500 entries that was never touched on a
read, so a key polled on every request could be evicted by an unrelated burst
of dynamic keys. That looked exactly like the cache being cleared at random,
and it is what made the site fall back to the database unpredictably.

- Evict least-recently-used instead, and raise the default budget to 2000
  (CACHE_MEMORY_MAX_ENTRIES). Reading a key now marks it as used, so a hot key
  only leaves when a hotter one takes its place.
- Add opt-in stale-while-revalidate (CachedOptions.staleMs). The grace window
  lives on the entry, so one call site opting in protects every reader of that
  key. A failed background refresh keeps serving the last good value instead of
  falling through to the origin, and is reported once rather than per read.
- Invalidate across processes. invalidateKey() now clears memory, deletes the
  Redis key and publishes a signal, so a value written by one process is no
  longer served stale by the others for the rest of its TTL. A failed Redis
  delete no longer skips the broadcast.
- Guard against a refresh that started before an invalidation writing its
  outdated result back into the cache.
- Read the news revision at most once a second per process instead of on every
  call, with a pub/sub signal to drop the local copy when it rotates. A Redis
  outage now degrades to the in-process cache rather than to no cache at all.
- Warm the hot public keys on boot, so the first visitors after a deploy do not
  each pay for a miss.
- Count hits, misses, stale serves, errors and evictions per key, exposed at
  GET /api/admin/devops/cache. Without it a wrong REDIS_URL, a full budget and
  a dead origin all look identical from the outside.
- Enforce the imaging cache budget for real: records are .img/.json pairs, so
  the old cap counted files and never removed anything while entries were
  fresh. Sweeps are throttled per directory and prune to a low-water mark.
- Cap the JWT version map, and stop per-test scratch roots from littering the
  runtime imaging cache.

Public read-only endpoints get grace windows; admin, account and auth data
deliberately stays fresh. Redis TTLs get a little jitter so keys written
together no longer expire together.

3209 tests pass. next build could not be verified on this host: the optimized
build is OOM-killed before prerender, so this has not run in a real Next
runtime yet.

.env.example

@@ -11,6 +11,12 @@ DATABASE_CONNECT_TIMEOUT_MS=5000
 # --- REDIS (Lightning Fast Caching & Sessions) ---
 REDIS_URL=redis://127.0.0.1:6379?connect_timeout=2
 REDIS_CACHE_TTL_DEFAULT=7200
+# In-process cache entries kept per instance, evicted least-recently-used. Raise
+# it if hot keys are evicted while memory headroom remains (default 2000).
+CACHE_MEMORY_MAX_ENTRIES=2000
+# Renders kept per imaging cache directory, counted as .img/.json pairs. A sweep
+# every 5 minutes brings an over-budget directory back to 90% of this (default 20000).
+IMAGING_CACHE_MAX_ENTRIES=20000
 
 # --- CORE RUNTIME & PERFORMANCE FLAGS ---
 NODE_ENV=production

src/app/(site)/leaderboard/page.tsx

@@ -56,16 +56,20 @@ type Row = { username: string; look: string; value: number };
 
 async function loadCreditsRows(): Promise<Row[]> {
 	try {
-		const users = await cached("lb_credits", 60_000, () =>
-			db
-				.select({
-					username: User.username,
-					look: User.look,
-					credits: User.credits,
-				})
-				.from(User)
-				.orderBy(desc(User.credits))
-				.limit(20),
+		const users = await cached(
+			"lb_credits",
+			60_000,
+			() =>
+				db
+					.select({
+						username: User.username,
+						look: User.look,
+						credits: User.credits,
+					})
+					.from(User)
+					.orderBy(desc(User.credits))
+					.limit(20),
+			{ staleMs: 120000 },
 		);
 		return users.map((u) => ({
 			username: u.username,
@@ -79,18 +83,22 @@ async function loadCreditsRows(): Promise<Row[]> {
 
 async function loadCurrencyRows(type: number): Promise<Row[]> {
 	try {
-		return await cached(`lb_currency_${type}`, 60_000, () =>
-			db
-				.select({
-					username: User.username,
-					look: User.look,
-					value: UsersCurrency.amount,
-				})
-				.from(UsersCurrency)
-				.innerJoin(User, eq(UsersCurrency.userId, User.id))
-				.where(eq(UsersCurrency.type, type))
-				.orderBy(desc(UsersCurrency.amount))
-				.limit(20),
+		return await cached(
+			`lb_currency_${type}`,
+			60_000,
+			() =>
+				db
+					.select({
+						username: User.username,
+						look: User.look,
+						value: UsersCurrency.amount,
+					})
+					.from(UsersCurrency)
+					.innerJoin(User, eq(UsersCurrency.userId, User.id))
+					.where(eq(UsersCurrency.type, type))
+					.orderBy(desc(UsersCurrency.amount))
+					.limit(20),
+			{ staleMs: 120000 },
 		);
 	} catch {
 		return [];
@@ -102,17 +110,21 @@ async function loadSettingsRows(
 ): Promise<Row[]> {
 	try {
 		const column = UsersSettings[field];
-		return await cached(`lb_settings_${field}`, 60_000, () =>
-			db
-				.select({
-					username: User.username,
-					look: User.look,
-					value: column,
-				})
-				.from(UsersSettings)
-				.innerJoin(User, eq(UsersSettings.userId, User.id))
-				.orderBy(desc(column))
-				.limit(20),
+		return await cached(
+			`lb_settings_${field}`,
+			60_000,
+			() =>
+				db
+					.select({
+						username: User.username,
+						look: User.look,
+						value: column,
+					})
+					.from(UsersSettings)
+					.innerJoin(User, eq(UsersSettings.userId, User.id))
+					.orderBy(desc(column))
+					.limit(20),
+			{ staleMs: 120000 },
 		);
 	} catch {
 		return [];

src/app/(site)/page.tsx

@@ -181,40 +181,60 @@ async function getHotelData() {
 				.where(eq(User.online, "1"))
 				.then((rows) => rows[0]?.total ?? 0),
 		).catch(publicReadFailure("home.online")),
-		cached("total_users", 300_000, () =>
-			db
-				.select({ total: count() })
-				.from(User)
-				.then((rows) => rows[0]?.total ?? 0),
+		cached(
+			"total_users",
+			300_000,
+			() =>
+				db
+					.select({ total: count() })
+					.from(User)
+					.then((rows) => rows[0]?.total ?? 0),
+			{ staleMs: 300000 },
 		).catch(publicReadFailure("home.users")),
-		cached("total_rooms", 300_000, () =>
-			db
-				.select({ total: count() })
-				.from(Rooms)
-				.then((rows) => rows[0]?.total ?? 0),
+		cached(
+			"total_rooms",
+			300_000,
+			() =>
+				db
+					.select({ total: count() })
+					.from(Rooms)
+					.then((rows) => rows[0]?.total ?? 0),
+			{ staleMs: 300000 },
 		).catch(publicReadFailure("home.rooms")),
-		cached("total_photos", 300_000, () =>
-			db
-				.select({ total: count() })
-				.from(CameraWeb)
-				.then((rows) => rows[0]?.total ?? 0),
+		cached(
+			"total_photos",
+			300_000,
+			() =>
+				db
+					.select({ total: count() })
+					.from(CameraWeb)
+					.then((rows) => rows[0]?.total ?? 0),
+			{ staleMs: 300000 },
 		).catch(publicReadFailure("home.photos-count")),
 		getNewsList(4, { throwOnError: true }).catch(
 			publicReadFailure("home.news"),
 		),
-		cached("home_online_users", 15_000, () =>
-			db
-				.select({ username: User.username, look: User.look })
-				.from(User)
-				.where(eq(User.online, "1"))
-				.limit(12),
+		cached(
+			"home_online_users",
+			15_000,
+			() =>
+				db
+					.select({ username: User.username, look: User.look })
+					.from(User)
+					.where(eq(User.online, "1"))
+					.limit(12),
+			{ staleMs: 30000 },
 		).catch(publicReadFailure("home.recent-users")),
-		cached("home_recent_photos", 60_000, () =>
-			db
-				.select({ id: CameraWeb.id, url: CameraWeb.url })
-				.from(CameraWeb)
-				.orderBy(desc(CameraWeb.timestamp))
-				.limit(4),
+		cached(
+			"home_recent_photos",
+			60_000,
+			() =>
+				db
+					.select({ id: CameraWeb.id, url: CameraWeb.url })
+					.from(CameraWeb)
+					.orderBy(desc(CameraWeb.timestamp))
+					.limit(4),
+			{ staleMs: 120000 },
 		).catch(publicReadFailure("home.photos")),
 	]);
 

src/app/(site)/photos/page.tsx

@@ -38,8 +38,16 @@ export default async function PhotosPage() {
 
 	let photos: Photo[] | null = [];
 	try {
-		photos = await cached("photos:grid", 60_000, async () =>
-			db.select().from(CameraWeb).orderBy(desc(CameraWeb.timestamp)).limit(48),
+		photos = await cached(
+			"photos:grid",
+			60_000,
+			async () =>
+				db
+					.select()
+					.from(CameraWeb)
+					.orderBy(desc(CameraWeb.timestamp))
+					.limit(48),
+			{ staleMs: 120000 },
 		);
 	} catch (error) {
 		photos = publicReadFailure("photos")(error);

src/app/(site)/rankings/page.tsx

@@ -40,18 +40,22 @@ export default async function RankingsPage() {
 	const t = await getTranslations("pages.rankings");
 	let users: TopUser[] | null = [];
 	try {
-		users = await cached("rankings:top", 60_000, async () =>
-			db
-				.select({
-					username: User.username,
-					look: User.look,
-					credits: User.credits,
-					rank: User.rank,
-					online: User.online,
-				})
-				.from(User)
-				.orderBy(desc(User.credits))
-				.limit(12),
+		users = await cached(
+			"rankings:top",
+			60_000,
+			async () =>
+				db
+					.select({
+						username: User.username,
+						look: User.look,
+						credits: User.credits,
+						rank: User.rank,
+						online: User.online,
+					})
+					.from(User)
+					.orderBy(desc(User.credits))
+					.limit(12),
+			{ staleMs: 120000 },
 		);
 	} catch (error) {
 		users = publicReadFailure("rankings")(error);

src/app/(site)/register/page.tsx

@@ -31,19 +31,27 @@ export default async function RegisterPage() {
 				.where(eq(User.online, "1"))
 				.then((rows) => rows[0]?.total ?? 0),
 		).catch(() => 0),
-		cached("register_online_users", 10_000, () =>
-			db
-				.select({ username: User.username, look: User.look })
-				.from(User)
-				.where(eq(User.online, "1"))
-				.limit(8),
+		cached(
+			"register_online_users",
+			10_000,
+			() =>
+				db
+					.select({ username: User.username, look: User.look })
+					.from(User)
+					.where(eq(User.online, "1"))
+					.limit(8),
+			{ staleMs: 30000 },
 		).catch(() => []),
-		cached("register_latest_users", 30_000, () =>
-			db
-				.select({ username: User.username, look: User.look })
-				.from(User)
-				.orderBy(desc(User.accountCreated))
-				.limit(8),
+		cached(
+			"register_latest_users",
+			30_000,
+			() =>
+				db
+					.select({ username: User.username, look: User.look })
+					.from(User)
+					.orderBy(desc(User.accountCreated))
+					.limit(8),
+			{ staleMs: 60000 },
 		).catch(() => []),
 	]);
 

src/app/api/admin/devops/cache/route.ts

@@ -0,0 +1,15 @@
+import { NextResponse } from "next/server";
+import { withAdmin } from "@/lib/api-handler";
+import { cacheBudget } from "@/lib/cache";
+import { readCacheStats } from "@/lib/cache-stats";
+import { PERMS } from "@/lib/permissions";
+
+/**
+ * Cache health report: per-key hit/miss/stale/error/eviction counters plus the
+ * in-process budget, so a cache that is silently doing nothing is visible
+ * instead of only showing up as "the site got slower".
+ */
+export const GET = withAdmin({ permission: PERMS.DEVOPS_VIEW }, async () => {
+	const stats = await readCacheStats();
+	return NextResponse.json({ ...stats, budget: cacheBudget() });
+});

src/app/api/badges/leaderboard/route.ts

@@ -362,6 +362,7 @@ export async function GET(req: Request) {
 
 				return cacheSafe({ badgeStats, totalBadges, achievementLevel, rarity });
 			},
+			{ staleMs: 120_000 },
 		);
 
 		// Per-viewer rank entries (only when a user is signed in).

src/app/api/guilds/[id]/route.ts

@@ -56,6 +56,7 @@ export async function GET(
 				const { userId, ...rest } = guild;
 				return cacheSafe({ data: { ...rest, ownerId: userId, memberCount } });
 			},
+			{ staleMs: 120_000 },
 		);
 
 		if (data === null) {

src/app/api/guilds/route.ts

@@ -50,6 +50,7 @@ export async function GET(req: Request) {
 					},
 				});
 			},
+			{ staleMs: 120_000 },
 		);
 
 		return apiJson(data);

src/app/api/leaderboard/route.ts

@@ -84,6 +84,9 @@ export async function GET(req: Request) {
 					? await loadCreditsRows()
 					: // eslint-disable-next-line security/detect-object-injection -- type validated to "diamonds"|"duckets"
 						await loadCurrencyRows(CURRENCY_TYPE[type]),
+			// Rankings are re-sorted on every read; serving the previous order for a
+			// minute keeps a busy leaderboard off the database.
+			{ staleMs: 60 },
 		);
 
 		return apiJson({ type, data: rows }, { status: 200 });

src/app/api/online/count/route.ts

@@ -10,13 +10,21 @@ import { db, User } from "@/lib/db";
 
 export async function GET(_req: Request) {
 	try {
-		const result = await cached("online_count", 10_000, async () => {
-			const [row] = await db
-				.select({ total: count() })
-				.from(User)
-				.where(eq(User.online, "1"));
-			return row?.total ?? 0;
-		});
+		// Polled by every page view and the in-game SSE stream, so a count that
+		// is one interval behind is worthless as "wrong" but a blocking COUNT(*)
+		// on each expiry is not. The grace window keeps the origin off the path.
+		const result = await cached(
+			"online_count",
+			10_000,
+			async () => {
+				const [row] = await db
+					.select({ total: count() })
+					.from(User)
+					.where(eq(User.online, "1"));
+				return row?.total ?? 0;
+			},
+			{ staleMs: 15_000 },
+		);
 		return apiJson({ count: result });
 	} catch {
 		return apiJson({ count: 0 });

src/app/api/online/count/stream/route.ts

@@ -13,13 +13,19 @@ import { rcon } from "@/lib/services/rcon";
 
 async function fetchOnlineCount(): Promise<number> {
 	try {
-		return await cached("online_count", 10_000, async () => {
-			const [row] = await db
-				.select({ total: count() })
-				.from(User)
-				.where(eq(User.online, "1"));
-			return row?.total ?? 0;
-		});
+		return await cached(
+			"online_count",
+			10_000,
+			async () => {
+				const [row] = await db
+					.select({ total: count() })
+					.from(User)
+					.where(eq(User.online, "1"));
+				return row?.total ?? 0;
+			},
+			// A subscriber must never wait on a COUNT(*) to get its next event.
+			{ staleMs: 15_000 },
+		);
 	} catch {
 		return 0;
 	}
@@ -30,8 +36,13 @@ async function fetchOnlineCount(): Promise<number> {
 // degradation on failure so the stream never errors out.
 async function fetchEmulatorStatus(): Promise<boolean> {
 	try {
-		return await cached("emulator_rcon_status", 15_000, () =>
-			rcon.send("ping", null).catch(() => false),
+		return await cached(
+			"emulator_rcon_status",
+			15_000,
+			() => rcon.send("ping", null).catch(() => false),
+			// Reachability is reported, never acted on, so a previous answer is
+			// good enough while the ping is re-run.
+			{ staleMs: 15_000 },
 		);
 	} catch {
 		return false;

src/app/api/online/route.ts

@@ -10,12 +10,18 @@ import { db, User } from "@/lib/db";
 
 export async function GET(_req: Request) {
 	try {
-		const users = await cached("online_users", 10_000, async () =>
-			db
-				.select({ username: User.username, look: User.look })
-				.from(User)
-				.where(eq(User.online, "1"))
-				.limit(100),
+		// A briefly stale online list is harmless; blocking the first request
+		// after the TTL on a COUNT + SELECT is not.
+		const users = await cached(
+			"online_users",
+			10_000,
+			async () =>
+				db
+					.select({ username: User.username, look: User.look })
+					.from(User)
+					.where(eq(User.online, "1"))
+					.limit(100),
+			{ staleMs: 15_000 },
 		);
 
 		return apiJson({ users });

src/app/api/photos/route.ts

@@ -44,6 +44,7 @@ export async function GET(req: Request) {
 					},
 				});
 			},
+			{ staleMs: 120_000 },
 		);
 
 		return apiJson(data);

src/app/api/radio/current-dj/route.ts

@@ -38,6 +38,7 @@ export async function GET(_req: Request) {
 
 				return { username: user.username, look: user.look };
 			},
+			{ staleMs: 30_000 },
 		);
 
 		return apiJson({ dj });

src/app/api/radio/points/leaderboard/route.ts

@@ -48,6 +48,7 @@ export async function GET(_req: Request) {
 					};
 				});
 			},
+			{ staleMs: 120_000 },
 		);
 
 		return apiJson({ data });

src/app/api/radio/shouts/route.ts

@@ -58,6 +58,7 @@ export async function GET(_req: Request) {
 					};
 				});
 			},
+			{ staleMs: 10_000 },
 		);
 
 		return apiJson({ shouts: data });

src/app/api/shop/categories/route.ts

@@ -26,6 +26,7 @@ export async function GET(_req: Request) {
 							asc(WebsiteShopCategories.name),
 						),
 				),
+			{ staleMs: 120_000 },
 		);
 
 		return apiJson({ data });

src/app/api/shop/route.ts

@@ -71,6 +71,7 @@ export async function GET(req: Request) {
 					},
 				});
 			},
+			{ staleMs: 120_000 },
 		);
 
 		return apiJson(data);

src/app/api/staff/route.ts

@@ -12,18 +12,24 @@ export async function GET(_req: Request) {
 		const minStaffRank =
 			Number(await siteSettings.get("min_staff_rank", "7")) || 7;
 
-		const staff = await redisCache(apiCacheKey("staff"), 300, () =>
-			db
-				.select({
-					username: User.username,
-					look: User.look,
-					rank: User.rank,
-					motto: User.motto,
-				})
-				.from(User)
-				.where(gte(User.rank, minStaffRank))
-				.orderBy(desc(User.rank), asc(User.username))
-				.limit(100),
+		const staff = await redisCache(
+			apiCacheKey("staff"),
+			300,
+			() =>
+				db
+					.select({
+						username: User.username,
+						look: User.look,
+						rank: User.rank,
+						motto: User.motto,
+					})
+					.from(User)
+					.where(gte(User.rank, minStaffRank))
+					.orderBy(desc(User.rank), asc(User.username))
+					.limit(100),
+			// Staff badges are rendered on every public page; a roster that is a
+			// few minutes behind must never turn into a query on each visitor.
+			{ staleMs: 300 },
 		);
 
 		return apiJson({ data: staff }, { status: 200 });

src/app/api/teams/route.ts

@@ -8,20 +8,24 @@ import { apiCacheKey, cacheSafe, redisCache } from "@/lib/redis-cache";
 
 export async function GET(_req: Request) {
 	try {
-		const rows = await redisCache(apiCacheKey("teams"), 300, async () =>
-			cacheSafe(
-				await db
-					.select({
-						id: WebsiteTeams.id,
-						rankName: WebsiteTeams.rankName,
-						badge: WebsiteTeams.badge,
-						jobDescription: WebsiteTeams.jobDescription,
-						staffColor: WebsiteTeams.staffColor,
-					})
-					.from(WebsiteTeams)
-					.where(eq(WebsiteTeams.hiddenRank, false))
-					.orderBy(asc(WebsiteTeams.id)),
-			),
+		const rows = await redisCache(
+			apiCacheKey("teams"),
+			300,
+			async () =>
+				cacheSafe(
+					await db
+						.select({
+							id: WebsiteTeams.id,
+							rankName: WebsiteTeams.rankName,
+							badge: WebsiteTeams.badge,
+							jobDescription: WebsiteTeams.jobDescription,
+							staffColor: WebsiteTeams.staffColor,
+						})
+						.from(WebsiteTeams)
+						.where(eq(WebsiteTeams.hiddenRank, false))
+						.orderBy(asc(WebsiteTeams.id)),
+				),
+			{ staleMs: 600_000 },
 		);
 
 		// apiJson serialises BigInt ids → string automatically.

src/app/api/values/[id]/route.ts

@@ -76,6 +76,7 @@ export async function GET(
 
 				return cacheSafe({ data: { ...value, category } });
 			},
+			{ staleMs: 300_000 },
 		);
 
 		if (data === null) {

src/app/api/values/categories/route.ts

@@ -26,6 +26,7 @@ export async function GET(_req: Request) {
 							asc(WebsiteRareValueCategories.name),
 						),
 				),
+			{ staleMs: 600_000 },
 		);
 
 		return apiJson({ data });

src/app/api/values/route.ts

@@ -61,6 +61,7 @@ export async function GET(req: Request) {
 					},
 				});
 			},
+			{ staleMs: 300_000 },
 		);
 
 		return apiJson(data);

src/instrumentation.ts

@@ -39,4 +39,10 @@ export async function register() {
 		void sweepExpiredCloudflareBlocks();
 	}, 30000);
 	cloudflareSweep.unref();
+
+	// A deploy starts with an empty in-process cache, so prime the hot public
+	// keys before traffic arrives instead of letting the first visitors each pay
+	// for a miss. Never awaited: it must not delay the server becoming ready.
+	const { warmPublicCaches } = await import("@/lib/services/cache-warmup");
+	void warmPublicCaches();
 }

src/lib/auth/jwt-version-cache.ts

@@ -8,6 +8,24 @@ const MEMORY_TTL_MS = 60_000;
 const REDIS_TTL_SEC = 60;
 const memory = new Map<number, { version: number; expiresAt: number }>();
 
+// One entry per authenticated user, so without a cap a long-lived process grows
+// this map by every account that ever signs in. Entries are tiny, so a generous
+// bound costs almost nothing and keeps the leak impossible.
+const MAX_MEMORY_ENTRIES = 20_000;
+
+function remember(userId: number, version: number, now: number): void {
+	if (memory.size >= MAX_MEMORY_ENTRIES) {
+		for (const [id, entry] of memory) {
+			if (entry.expiresAt <= now) memory.delete(id);
+		}
+		if (memory.size >= MAX_MEMORY_ENTRIES) {
+			const oldest = memory.keys().next().value;
+			if (oldest !== undefined) memory.delete(oldest);
+		}
+	}
+	memory.set(userId, { version, expiresAt: now + MEMORY_TTL_MS });
+}
+
 function redisKey(userId: number): string {
 	return `jwtver:${userId}`;
 }
@@ -31,7 +49,7 @@ export async function getCachedJwtVersion(
 			if (raw !== null && raw !== undefined) {
 				const version = Number.parseInt(raw, 10);
 				if (Number.isFinite(version)) {
-					memory.set(userId, { version, expiresAt: now + MEMORY_TTL_MS });
+					remember(userId, version, now);
 					return version;
 				}
 			}
@@ -48,7 +66,7 @@ export async function getCachedJwtVersion(
 			.limit(1);
 		if (!row) return null;
 		const version = row.websiteJwtVersion;
-		memory.set(userId, { version, expiresAt: now + MEMORY_TTL_MS });
+		remember(userId, version, now);
 		if (redis) {
 			try {
 				await redis.setex(redisKey(userId), REDIS_TTL_SEC, String(version));

src/lib/cache-stats.test.ts

@@ -0,0 +1,99 @@
+// @ts-nocheck
+import { beforeEach, expect, it, vi } from "vitest";
+
+const state = vi.hoisted(() => ({
+	status: "ready",
+	get: vi.fn(),
+	setex: vi.fn(),
+}));
+
+vi.mock("@/lib/redis", () => ({
+	redis: {
+		get: (key: string) => state.get(key),
+		setex: (key: string, ttl: number, value: string) =>
+			state.setex(key, ttl, value),
+		get status() {
+			return state.status;
+		},
+	},
+}));
+vi.mock("@/lib/logger", () => ({
+	logger: { warn: vi.fn(), error: vi.fn(), info: vi.fn() },
+}));
+
+import { readCacheStats, recordCacheOutcome } from "./cache-stats";
+
+beforeEach(() => {
+	// The map is process-global on purpose; tests get a clean one.
+	globalThis.cacheStats?.clear();
+	state.status = "ready";
+	state.get.mockReset().mockResolvedValue(null);
+	state.setex.mockReset().mockResolvedValue("OK");
+});
+
+it("counts outcomes per key", async () => {
+	recordCacheOutcome("online_count", "hit");
+	recordCacheOutcome("online_count", "hit");
+	recordCacheOutcome("online_count", "miss");
+	recordCacheOutcome("online_count", "stale");
+	recordCacheOutcome("api_staff", "error");
+	recordCacheOutcome("api_staff", "evicted");
+
+	const report = await readCacheStats();
+	expect(report.keys.online_count).toEqual({
+		hits: 2,
+		misses: 1,
+		stale: 1,
+		errors: 0,
+		evictions: 0,
+	});
+	expect(report.keys.api_staff.errors).toBe(1);
+	expect(report.keys.api_staff.evictions).toBe(1);
+	expect(report.totals.keys).toBe(2);
+});
+
+it("counts a stale serve as a hit, since the origin was not called", async () => {
+	recordCacheOutcome("k", "hit");
+	recordCacheOutcome("k", "stale");
+	recordCacheOutcome("k", "stale");
+	const { totals } = await readCacheStats();
+	// 1 real hit + 2 stale serves, none of which reached the database.
+	expect(totals.hits).toBe(1);
+	expect(totals.stale).toBe(2);
+	expect(totals.hitRatio).toBe(1);
+});
+
+it("keeps the tracked key space bounded", async () => {
+	for (let i = 0; i < 600; i++) recordCacheOutcome(`key-${i}`, "hit");
+	const report = await readCacheStats();
+	expect(report.totals.keys).toBeLessThanOrEqual(500);
+});
+
+it("reports Redis as unavailable when it is down, and stays usable", async () => {
+	state.status = "end";
+	recordCacheOutcome("online_count", "hit");
+	const report = await readCacheStats();
+	expect(report.redisAvailable).toBe(false);
+	expect(report.shared).toBe(false);
+	expect(report.keys.online_count.hits).toBe(1);
+});
+
+it("prefers the shared snapshot from Redis when one exists", async () => {
+	recordCacheOutcome("local_only", "hit");
+	state.get.mockResolvedValue(
+		JSON.stringify({ shared_key: { hits: 9, misses: 1 } }),
+	);
+	const report = await readCacheStats();
+	expect(report.shared).toBe(true);
+	expect(report.keys.shared_key.hits).toBe(9);
+	expect(report.totals.hitRatio).toBe(0.9);
+});
+
+it("survives an unreadable shared snapshot", async () => {
+	recordCacheOutcome("local_only", "hit");
+	state.get.mockRejectedValue(Error("offline"));
+	const report = await readCacheStats();
+	expect(report.shared).toBe(false);
+	expect(report.redisAvailable).toBe(true);
+	expect(report.keys.local_only.hits).toBe(1);
+});

src/lib/cached-db.test.ts

@@ -0,0 +1,69 @@
+// @ts-nocheck
+import { beforeEach, expect, it, vi } from "vitest";
+
+const state = vi.hoisted(() => ({
+	deleted: [] as string[],
+	published: [] as string[],
+	failDelete: false,
+	status: "ready",
+	localKeys: new Set<string>(),
+}));
+
+vi.mock("@/lib/redis", () => ({
+	redis: {
+		get status() {
+			return state.status;
+		},
+		del: async (key: string) => {
+			if (state.failDelete) throw Error("redis write failed");
+			state.deleted.push(key);
+			return 1;
+		},
+		publish: async (channel: string, key: string) => {
+			state.published.push(`${channel}|${key}`);
+			return 1;
+		},
+		duplicate: () => ({
+			on: () => {},
+			subscribe: async () => 1,
+		}),
+	},
+}));
+vi.mock("@/lib/logger", () => ({
+	logger: { warn: vi.fn(), error: vi.fn(), info: vi.fn() },
+}));
+vi.mock("@/lib/cache-invalidation", () => ({
+	publishCacheInvalidation: async (key: string) => {
+		state.published.push(`broadcast|${key}`);
+	},
+	onCacheInvalidated: () => {},
+}));
+
+import { invalidateKey } from "./cached-db";
+
+beforeEach(() => {
+	state.deleted = [];
+	state.published = [];
+	state.failDelete = false;
+	state.status = "ready";
+});
+
+it("clears memory, Redis and the other instances", async () => {
+	expect(await invalidateKey("home:news")).toBe(1);
+	expect(state.deleted).toEqual(["home:news"]);
+	expect(state.published).toEqual(["broadcast|home:news"]);
+});
+
+it("still tells the other instances when the Redis delete fails", async () => {
+	// Returning early here would leave every other process serving the old value
+	// until its own TTL ran out, while this one already had the new value.
+	state.failDelete = true;
+	expect(await invalidateKey("home:news")).toBe(0);
+	expect(state.published).toEqual(["broadcast|home:news"]);
+});
+
+it("still tells the other instances when Redis is gone", async () => {
+	state.status = "end";
+	expect(await invalidateKey("home:news")).toBe(0);
+	expect(state.published).toEqual(["broadcast|home:news"]);
+});

src/lib/cached-db.ts

@@ -1,6 +1,8 @@
 import "server-only";
 
-import { cached, invalidateMemory } from "@/lib/cache";
+import { type CachedOptions, cached, invalidateMemory } from "@/lib/cache";
+import { publishCacheInvalidation } from "@/lib/cache-invalidation";
+import { logger } from "@/lib/logger";
 import { redis } from "@/lib/redis";
 
 const DEFAULT_CACHE_TTL = 60;
@@ -14,19 +16,28 @@ export async function cachedQuery<T>(
 	cacheKey: string,
 	queryFn: () => Promise<T>,
 	ttl: number = DEFAULT_CACHE_TTL,
+	options?: CachedOptions,
 ): Promise<T> {
-	return cached(cacheKey, ttl * 1000, queryFn);
+	return cached(cacheKey, ttl * 1000, queryFn, options);
 }
 
-/** Invalidate a single cache key immediately (memory + Redis). */
+/** Invalidate a single cache key immediately (memory + Redis + every instance). */
 export async function invalidateKey(key: string): Promise<number> {
 	invalidateMemory(key);
+	let removed = 0;
 	if (redis && redis.status !== "end") {
 		try {
-			return await redis.del(key);
-		} catch {
-			return 0;
+			removed = await redis.del(key);
+		} catch (error) {
+			// A failed delete is not a reason to stay quiet: the durable copy may
+			// still exist, but the other processes have to drop theirs either way,
+			// otherwise this instance is the only one that sees the new value.
+			logger.warn("[cache] redis delete failed", {
+				key,
+				error: String(error),
+			});
 		}
 	}
-	return 0;
+	await publishCacheInvalidation(key);
+	return removed;
 }

src/lib/imager-cache.test.ts

@@ -1,5 +1,7 @@
 // @ts-nocheck
-import { beforeEach, describe, expect, it, vi } from "vitest";
+import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { afterAll, beforeEach, describe, expect, it, vi } from "vitest";
 import {
 	avatarCacheDir,
 	imagingCacheKey,
@@ -10,12 +12,18 @@ import {
 const TEST_KEY = imagingCacheKey("test-render");
 const TEST_DIR = () => `${avatarCacheDir()}/unit`;
 
+// One scratch root for the whole file. A per-test root (pid + Date.now()) left a
+// new directory behind for every single test, which is how a hundred+ junk
+// directories ended up in the runtime cache tree.
+const SCRATCH_ROOT = `${process.cwd()}/storage/imaging/test-unit-${process.pid}`;
+
 beforeEach(() => {
 	// Isolate every run from the runtime cache and from prior runs.
-	vi.stubEnv(
-		"IMAGING_CACHE_ROOT",
-		`${process.cwd()}/storage/imaging/test-unit-${process.pid}-${Date.now()}`,
-	);
+	vi.stubEnv("IMAGING_CACHE_ROOT", SCRATCH_ROOT);
+});
+
+afterAll(async () => {
+	await rm(SCRATCH_ROOT, { recursive: true, force: true });
 });
 
 describe("imagingCacheKey", () => {
@@ -56,3 +64,69 @@ describe("readImagingCache / writeImagingCache", () => {
 		expect(await readImagingCache(TEST_DIR(), key)).not.toBeNull();
 	});
 });
+
+describe("pruneImagingCache", () => {
+	it("brings a directory that is over budget back down to its low-water mark", async () => {
+		// The budget is read per sweep, so a test can make it small instead of
+		// writing twenty thousand files.
+		vi.stubEnv("IMAGING_CACHE_MAX_ENTRIES", "20");
+		const dir = `${SCRATCH_ROOT}/prune-cap`;
+
+		// Fill past the budget before the first write of this directory, because
+		// that first write is the one that runs the sweep.
+		await mkdir(dir, { recursive: true });
+		for (let i = 0; i < 30; i++) {
+			const name = imagingCacheKey(`figure-${String(i).padStart(3, "0")}`);
+			await writeFile(join(dir, `${name}.img`), "x");
+			await writeFile(join(dir, `${name}.json`), "{}");
+		}
+
+		// Every file is fresh, so an age-based sweep would not remove one of
+		// them and the directory would keep growing. The write triggers a sweep.
+		await writeImagingCache(
+			dir,
+			imagingCacheKey("trigger-sweep"),
+			new Uint8Array([1]),
+			"image/png",
+		);
+
+		// Back inside the budget: 20 * 0.9 = 18 survivors, plus the record the
+		// triggering write just added. Records are counted as .img/.json pairs,
+		// not as individual files.
+		const remaining = (await readdir(dir)).filter((n) => n.endsWith(".img"));
+		expect(remaining.length).toBeLessThanOrEqual(19);
+		expect(remaining.length).toBeGreaterThan(0);
+
+		// Newest survive, oldest go: the prune is least-recently-used, not random.
+		expect(remaining).toContain(`${imagingCacheKey("figure-029")}.img`);
+		expect(remaining).not.toContain(`${imagingCacheKey("figure-000")}.img`);
+
+		// A pair is removed together, so no .json is left without its image.
+		for (const image of remaining)
+			expect(await readdir(dir)).toContain(
+				`${image.replace(/\.img$/, "")}.json`,
+			);
+	});
+
+	it("leaves a directory that is within budget alone", async () => {
+		vi.stubEnv("IMAGING_CACHE_MAX_ENTRIES", "20");
+		const dir = `${SCRATCH_ROOT}/prune-under-cap`;
+		for (let i = 0; i < 5; i++) {
+			await writeImagingCache(
+				dir,
+				imagingCacheKey(`kept-${i}`),
+				new Uint8Array([i]),
+				"image/png",
+			);
+		}
+		const before = await readdir(dir);
+		await writeImagingCache(
+			dir,
+			imagingCacheKey("another"),
+			new Uint8Array([1]),
+			"image/png",
+		);
+		// Only the new record was added; nothing that was already there was dropped.
+		expect((await readdir(dir)).length).toBe(before.length + 2);
+	});
+});

src/lib/redis-cache.ts

@@ -1,6 +1,6 @@
 import "server-only";
 
-import { cached } from "@/lib/cache";
+import { type CachedOptions, cached } from "@/lib/cache";
 
 /**
  * Cache the result of a fetch function, memory-first with a Redis fallback
@@ -10,8 +10,9 @@ export async function redisCache<T>(
 	key: string,
 	ttlSeconds: number,
 	fetch: () => Promise<T>,
+	options?: CachedOptions,
 ): Promise<T> {
-	return cached(key, ttlSeconds * 1000, fetch);
+	return cached(key, ttlSeconds * 1000, fetch, options);
 }
 
 /**

src/lib/runtime-avatar-route.test.ts

@@ -1,21 +1,27 @@
 // @ts-nocheck
+import { rm } from "node:fs/promises";
 import { NextRequest } from "next/server";
-import { afterEach, beforeEach, expect, it, vi } from "vitest";
+import { afterAll, afterEach, beforeEach, expect, it, vi } from "vitest";
 import { GET } from "../app/api/imaging/avatar/route";
 
+// One scratch root for the whole file, removed afterwards: a per-test root left a
+// new directory behind for every test, polluting the runtime cache tree.
+const SCRATCH_ROOT = `${process.cwd()}/storage/imaging/test-route-${process.pid}`;
+
 beforeEach(() => {
-	// Fresh, isolated cache root per test run so the disk cache can never leak
-	// state between runs or into the runtime cache.
-	vi.stubEnv(
-		"IMAGING_CACHE_ROOT",
-		`${process.cwd()}/storage/imaging/test-route-${process.pid}-${Date.now()}`,
-	);
+	// Isolated cache root so the disk cache can never leak state between runs or
+	// into the runtime cache.
+	vi.stubEnv("IMAGING_CACHE_ROOT", SCRATCH_ROOT);
 });
 
 afterEach(() => {
 	vi.unstubAllEnvs();
 	vi.unstubAllGlobals();
 });
+
+afterAll(async () => {
+	await rm(SCRATCH_ROOT, { recursive: true, force: true });
+});
 it("forwards multicolor figures and preserves configured query options at runtime", async () => {
 	const figure = "hr-11782-40-40.hd-180-7-14.ch-11592-66.lg-10726-79-1408";
 	const fetcher = vi.fn(

src/lib/services/news-cache.ts

@@ -1,28 +1,83 @@
 import "server-only";
+
 import { randomUUID } from "node:crypto";
 import { cached } from "@/lib/cache";
+import { onCacheSignal, publishCacheSignal } from "@/lib/cache-invalidation";
 import { logger } from "@/lib/logger";
 import { redis } from "@/lib/redis";
 
 const REVISION_KEY = "cms:news:revision";
+
+/** News entries are namespaced by a revision, so a new revision drops them all. */
+const REVISION_CHANNEL = "cache:news-revision";
+
+// The revision is read from Redis at most this often per process. Reading it on
+// every call (as this used to) put a Redis round-trip in front of the in-process
+// fast path, which is exactly what the cache exists to avoid. A published article
+// still shows up immediately: the invalidation signal clears this copy, and the
+// window is only the fallback for when pub/sub is unavailable.
+const REVISION_TTL_MS = 1_000;
+
+let revision: { value: string; expiresAt: number } | null = null;
+let inFlightRevision: Promise<string> | null = null;
+
+async function readRevision(): Promise<string> {
+	const now = Date.now();
+	if (revision && revision.expiresAt > now) return revision.value;
+	inFlightRevision ??= (async () => {
+		try {
+			return (await redis?.get(REVISION_KEY)) ?? "0";
+		} catch {
+			return "0";
+		} finally {
+			inFlightRevision = null;
+		}
+	})();
+	const value = await inFlightRevision;
+	revision = { value, expiresAt: Date.now() + REVISION_TTL_MS };
+	return value;
+}
+
+function setRevision(value: string): void {
+	revision = { value, expiresAt: Date.now() + REVISION_TTL_MS };
+}
+
+// Another process published new news: forget the revision so the next read picks
+// up the new one instead of serving entries under the old namespace.
+onCacheSignal(REVISION_CHANNEL, () => {
+	revision = null;
+});
+
 export async function cacheNews<T>(
 	key: string,
 	ttlMs: number,
 	query: () => Promise<T>,
 ): Promise<T> {
-	if (!redis || redis.status === "end") return query();
-	let revision: string;
-	try {
-		revision = (await redis.get(REVISION_KEY)) ?? "0";
-	} catch {
-		return query();
-	}
-	return cached(`news:${revision}:${key}`, ttlMs, query);
+	// Every caller of cacheNews is a public news read, so the grace window lives
+	// here rather than at each call site. Publishing an article rotates the
+	// revision, which drops these entries immediately; the window only matters
+	// when that signal cannot be delivered.
+	const options = { staleMs: ttlMs };
+	// Without Redis the revision cannot be shared, so fall back to a fixed one:
+	// the in-process cache still works, which is far better than hitting the
+	// database on every request for the whole duration of a Redis outage.
+	if (!redis || redis.status === "end")
+		return cached(`news:0:${key}`, ttlMs, query, options);
+	return cached(`news:${await readRevision()}:${key}`, ttlMs, query, options);
 }
+
+async function rotateRevision(): Promise<string> {
+	const next = randomUUID();
+	if (redis) await redis.set(REVISION_KEY, next);
+	setRevision(next);
+	await publishCacheSignal(REVISION_CHANNEL, next);
+	return next;
+}
+
 export async function invalidateNewsCache(): Promise<void> {
 	if (!redis || redis.status === "end") return;
 	try {
-		await redis.set(REVISION_KEY, randomUUID());
+		await rotateRevision();
 	} catch (error) {
 		logger.error("News saved but public cache invalidation failed", {
 			module: "news",
@@ -36,5 +91,5 @@ export async function refreshNewsCacheForDelivery(): Promise<void> {
 	if (!redis) return;
 	if (redis.status === "end")
 		throw new Error("News cache connection is closed");
-	await redis.set(REVISION_KEY, randomUUID());
+	await rotateRevision();
 }