fix(ci): publish registry blobs in bounded chunks
This commit is contained in:
1 parent
047cd9f3dd
commit
2af244b634
5 files changed
+74
-8
No files matched your search
@@ -244,6 +244,13 @@ by installations; existing remco image tags are not moved automatically.
|
||||
`:<full-commit>-migrations`. Only the application image runs the website; the
|
||||
migrations image is used temporarily for the matching database migrations.
|
||||
|
||||
Publication uses checksum-pinned regctl v0.11.6 with 8 MiB blob requests to
|
||||
avoid monolithic layer uploads exceeding reverse-proxy limits. Both images are
|
||||
exported and uploaded sequentially; temporary archives and credentials are removed
|
||||
on exit. The runner needs curl, sha256sum and temporary disk space for one Docker
|
||||
image archive. The remote image config digest is checked against the local image
|
||||
after each upload. A proxy must still allow the OCI registry PATCH/PUT endpoints.
|
||||
|
||||
For this repository the image base is
|
||||
`gitlab.epicnabbo.nl/simo/epicnext-cms`. Package access is controlled by Gitea.
|
||||
For private packages, run `docker login gitlab.epicnabbo.nl` on the installation
|
||||
|
||||
@@ -41,6 +41,29 @@ fi
|
||||
docker build --network=host --target migrations -t "$image-migrations" "$context"
|
||||
node scripts/verify-portable-image.mjs "$image" "$sha"
|
||||
# Publish only after the same application image passed both runtime configurations.
|
||||
docker push "$image-migrations"
|
||||
docker push "$image"
|
||||
# Bound each blob request below reverse-proxy upload limits. Pin the uploader
|
||||
# and verify its checksum before giving it access to the temporary Docker login.
|
||||
case "$(uname -m)" in
|
||||
x86_64) arch=amd64; checksum=8e0e62a497fcdb8048d18aa927a139613176ba0531f412bc541044e28f9856bd ;;
|
||||
aarch64|arm64) arch=arm64; checksum=a9b71a3ee79b2d1dbbd7d51fd5e8fa214722c192864235d3d8764463c751a1ff ;;
|
||||
*) echo "Unsupported registry uploader architecture" >&2; exit 1 ;;
|
||||
esac
|
||||
curl --fail --silent --show-error --location --retry 3 --connect-timeout 15 --max-time 120 \
|
||||
"https://github.com/regclient/regclient/releases/download/v0.11.6/regctl-linux-$arch" -o "$context/regctl"
|
||||
printf '%s %s\n' "$checksum" "$context/regctl" | sha256sum --check --status
|
||||
chmod 700 "$context/regctl"
|
||||
export REGCTL_CONFIG="$DOCKER_CONFIG/regctl.json"
|
||||
regctl() { "$context/regctl" "$@"; }
|
||||
regctl registry set "$registry" --blob-chunk 8388608 --blob-max 8388608
|
||||
for target in "$image-migrations" "$image"; do
|
||||
echo "Publishing $target with blob requests up to 8 MiB"
|
||||
docker image save --output "$context/image.tar" "$target"
|
||||
regctl image import "$target" "$context/image.tar"
|
||||
# Import may change compression/manifest representation, but the immutable
|
||||
# image config digest must still match the exact local image we verified.
|
||||
expected_config="$(docker image inspect --format '{{.Id}}' "$target")"
|
||||
remote_config="$(regctl manifest get "$target" --format '{{.GetConfig.Digest}}')"
|
||||
[[ "$remote_config" = "$expected_config" ]] || { echo "Published image config does not match verified local image" >&2; exit 1; }
|
||||
rm -f -- "$context/image.tar"
|
||||
done
|
||||
echo "Published application and migrations: $image"
|
||||
@@ -62,7 +62,7 @@ it("verifies portability before publishing and uses committed build context", ()
|
||||
expect(publish).toContain("git archive HEAD");
|
||||
expect(
|
||||
publish.indexOf("node scripts/verify-portable-image.mjs"),
|
||||
).toBeLessThan(publish.indexOf("docker push"));
|
||||
).toBeLessThan(publish.indexOf("regctl image import"));
|
||||
expect(publish).toContain("--password-stdin");
|
||||
expect(publish).not.toContain(":latest");
|
||||
});
|
||||
|
||||
@@ -17,7 +17,7 @@ const bash =
|
||||
.find((path) => existsSync(path)) ?? "bash")
|
||||
: "bash";
|
||||
const sha = "a".repeat(40);
|
||||
function simulate(scenario: string, namespace = "") {
|
||||
function simulate(scenario: string, namespace = "", expectedStatus = 0) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "cms-publish-test-"));
|
||||
try {
|
||||
const result = spawnSync(
|
||||
@@ -42,7 +42,7 @@ function simulate(scenario: string, namespace = "") {
|
||||
},
|
||||
);
|
||||
if (result.error) throw result.error;
|
||||
expect(result.status, result.stdout + result.stderr).toBe(0);
|
||||
expect(result.status, result.stdout + result.stderr).toBe(expectedStatus);
|
||||
return readFileSync(join(dir, "calls"), "utf8");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
@@ -57,7 +57,7 @@ describe("verified application image reuse", () => {
|
||||
expect(calls).not.toContain("docker build --network=host --build-arg");
|
||||
expect(
|
||||
calls.indexOf("verify scripts/verify-portable-image.mjs"),
|
||||
).toBeLessThan(calls.indexOf("docker push"));
|
||||
).toBeLessThan(calls.indexOf("regctl image import"));
|
||||
});
|
||||
it.each(["missing", "mismatch"])(
|
||||
"builds committed source when verification marker is %s",
|
||||
@@ -71,10 +71,31 @@ describe("verified application image reuse", () => {
|
||||
|
||||
it("uses the token account namespace instead of the repository owner", () => {
|
||||
const calls = simulate("verified");
|
||||
expect(calls).toContain(`docker push registry.invalid/simo/cms:${sha}`);
|
||||
expect(calls).toContain(
|
||||
`regctl image import registry.invalid/simo/cms:${sha}`,
|
||||
);
|
||||
expect(calls).not.toContain("registry.invalid/owner/cms");
|
||||
});
|
||||
it("supports an explicit organization namespace", () => {
|
||||
const calls = simulate("verified", "My-Org");
|
||||
expect(calls).toContain(`docker push registry.invalid/my-org/cms:${sha}`);
|
||||
expect(calls).toContain(
|
||||
`regctl image import registry.invalid/my-org/cms:${sha}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds uploads and verifies both published image configs", () => {
|
||||
const calls = simulate("verified");
|
||||
expect(calls).toContain("--blob-chunk 8388608 --blob-max 8388608");
|
||||
expect(calls).not.toContain("docker push");
|
||||
expect(calls.match(/regctl image import/g)).toHaveLength(2);
|
||||
expect(calls.match(/regctl manifest get/g)).toHaveLength(2);
|
||||
});
|
||||
it.each(["upload-fails", "wrong-config", "bad-checksum"])(
|
||||
"stops publication on %s",
|
||||
(scenario) => {
|
||||
const calls = simulate(scenario, "", 1);
|
||||
expect(calls).not.toContain(
|
||||
`regctl image import registry.invalid/simo/cms:${sha} `,
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -12,3 +12,18 @@ docker() {
|
||||
fi
|
||||
}
|
||||
export -f git tar node docker
|
||||
|
||||
curl() {
|
||||
local output="${@: -1}"
|
||||
cat > "$output" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
echo "regctl $*" >> "$TEST_DIR/calls"
|
||||
if [[ "$1 $2" = "image import" && "$SCENARIO" = upload-fails ]]; then exit 1; fi
|
||||
if [[ "$1 $2" = "manifest get" ]]; then
|
||||
if [[ "$SCENARIO" = wrong-config ]]; then echo sha256:wrong; else echo sha256:candidate; fi
|
||||
fi
|
||||
MOCK
|
||||
}
|
||||
sha256sum() { cat >/dev/null; [[ "$SCENARIO" != bad-checksum ]]; }
|
||||
uname() { echo x86_64; }
|
||||
export -f curl sha256sum uname
|
||||
Reference in new issue
Block a user