fix(news): retry rolled-back scheduled publication deadlocks

src/lib/services/news-scheduler.test.ts

@@ -15,6 +15,8 @@ const state = vi.hoisted(() => ({
 	operations: [] as unknown[][],
 	effects: [] as unknown[][],
 	failOutbox: false,
+	transactionCalls: 0,
+	failures: [] as Array<{ at: "select" | "outbox" | "commit"; error: unknown }>,
 	loseRace: false,
 	refresh: vi.fn(),
 }));
@@ -26,10 +28,14 @@ vi.mock("@/lib/db", () => {
 		value instanceof Date
 			? value
 			: new Date(`${String(value).replace(" ", "T")}Z`);
+	const failAt = (at: "select" | "outbox" | "commit") => {
+		if (state.failures[0]?.at === at) throw state.failures.shift()?.error;
+	};
 	const tx = {
 		execute: async (query: SQL) => {
 			const { sql, params } = dialect.sqlToQuery(query);
 			state.queries.push({ sql, params });
+			if (sql.startsWith("SELECT")) failAt("select");
 			if (sql.startsWith("SELECT"))
 				return [
 					state.articles
@@ -54,6 +60,7 @@ vi.mock("@/lib/db", () => {
 			if (sql.startsWith("INSERT INTO cms_operations"))
 				state.operations.push(params);
 			else if (sql.startsWith("INSERT INTO cms_outbox")) {
+				failAt("outbox");
 				if (state.failOutbox) throw Error("outbox unavailable");
 				state.effects.push(params);
 			} else throw Error(`Unexpected SQL ${sql}`);
@@ -63,17 +70,22 @@ vi.mock("@/lib/db", () => {
 	return {
 		db: {
 			transaction: async (work: (tx: unknown) => Promise<unknown>) => {
+				state.transactionCalls++;
 				const before = structuredClone({
 					articles: state.articles,
 					operations: state.operations,
 					effects: state.effects,
 				});
+				let result: unknown;
 				try {
-					return await work(tx);
+					result = await work(tx);
 				} catch (error) {
 					Object.assign(state, before);
 					throw error;
 				}
+				// A lost commit response may follow a durable commit: preserve its writes.
+				failAt("commit");
+				return result;
 			},
 		},
 	};
@@ -95,6 +107,8 @@ beforeEach(() => {
 	state.operations = [];
 	state.effects = [];
 	state.failOutbox = false;
+	state.transactionCalls = 0;
+	state.failures = [];
 	state.loseRace = false;
 	state.refresh.mockReset();
 });
@@ -174,3 +188,84 @@ it("records the article title and exact identifier for the operation screen", as
 		published: true,
 	});
 });
+
+const deadlock = () =>
+	Object.assign(new Error("Deadlock found"), {
+		code: "ER_LOCK_DEADLOCK",
+		errno: 1213,
+	});
+it.each(["select", "outbox"] as const)(
+	"retries a rolled-back deadlock at %s with the same cutoff and one durable effect",
+	async (at) => {
+		state.failures = [
+			{ at, error: new Error("Drizzle query failed", { cause: deadlock() }) },
+		];
+		expect(await publishDueArticles(now)).toBe(1);
+		expect(state.transactionCalls).toBe(2);
+		expect(state.articles[0].status).toBe("published");
+		expect(state.operations).toHaveLength(1);
+		expect(state.effects).toHaveLength(1);
+		expect(state.refresh).toHaveBeenCalledOnce();
+		const cutoffs = state.queries
+			.filter(({ sql }) => sql.startsWith("SELECT"))
+			.map(({ params }) => params[0]);
+		expect(cutoffs).toEqual([
+			"2030-01-02 00:00:00.000",
+			"2030-01-02 00:00:00.000",
+		]);
+	},
+);
+it("bounds deadlock retries to three whole transactions and preserves the final error", async () => {
+	const failure = deadlock();
+	state.failures = Array.from({ length: 3 }, () => ({
+		at: "outbox" as const,
+		error: failure,
+	}));
+	await expect(publishDueArticles(now)).rejects.toBe(failure);
+	expect(state.transactionCalls).toBe(3);
+	expect(state.articles[0].status).toBe("scheduled");
+	expect(state.operations).toHaveLength(0);
+	expect(state.effects).toHaveLength(0);
+	expect(state.refresh).not.toHaveBeenCalled();
+});
+it("recognizes the numeric MariaDB deadlock code through nested driver causes", async () => {
+	state.failures = [
+		{ at: "select", error: { cause: { cause: { errno: 1213 } } } },
+	];
+	expect(await publishDueArticles(now)).toBe(1);
+	expect(state.transactionCalls).toBe(2);
+});
+it.each([
+	Object.assign(new Error("lock wait timeout"), {
+		code: "ER_LOCK_WAIT_TIMEOUT",
+		errno: 1205,
+	}),
+	new Error("ER_LOCK_DEADLOCK in an untrusted message"),
+	Object.assign(new Error("connection lost"), {
+		code: "PROTOCOL_CONNECTION_LOST",
+	}),
+])("does not retry unrelated errors: %s", async (failure) => {
+	state.failures = [{ at: "outbox", error: failure }];
+	await expect(publishDueArticles(now)).rejects.toBe(failure);
+	expect(state.transactionCalls).toBe(1);
+	expect(state.operations).toHaveLength(0);
+	expect(state.effects).toHaveLength(0);
+});
+it("never retries an unknown commit outcome", async () => {
+	const failure = Object.assign(new Error("Commit response lost"), {
+		code: "PROTOCOL_CONNECTION_LOST",
+	});
+	state.failures = [{ at: "commit", error: failure }];
+	await expect(publishDueArticles(now)).rejects.toBe(failure);
+	expect(state.transactionCalls).toBe(1);
+	expect(state.operations).toHaveLength(1);
+	expect(state.effects).toHaveLength(1);
+	expect(state.refresh).not.toHaveBeenCalled();
+});
+it("bounds traversal of cyclic error causes", async () => {
+	const failure: { cause?: unknown } = {};
+	failure.cause = failure;
+	state.failures = [{ at: "select", error: failure }];
+	await expect(publishDueArticles(now)).rejects.toBe(failure);
+	expect(state.transactionCalls).toBe(1);
+});