diff --git a/.github/scripts/docker-image.sh b/.github/scripts/docker-image.sh new file mode 100644 index 00000000..69b92abd --- /dev/null +++ b/.github/scripts/docker-image.sh @@ -0,0 +1,114 @@ +#!/bin/sh + +# Cache a container image as a tarball, keyed by digests looked up live from the registry: a cached +# image is only used if it is exactly what the registry serves now, so a poisoned cache entry is never used. +# +# usage: +# sh .github/scripts/docker-image.sh resolve +# prints `key=` and `digests=`, for $GITHUB_OUTPUT +# DIGESTS='' sh .github/scripts/docker-image.sh ensure +# loads if it holds one of $DIGESTS; otherwise pulls , checks it, and saves it to + +set -eu + +ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" + else + shasum -a 256 "$1" + fi | cut -d ' ' -f 1 +} + +# fetches a manifest into a file; digests are computed from its exact bytes +registry_get() { + curl -fsSL --retry 5 --retry-all-errors -H "Accept: ${ACCEPT}" -o "$2" "$1" +} + +resolve() { + IMAGE="$1" + HOST="${IMAGE%%/*}" + REST="${IMAGE#*/}" + REPO="${REST%:*}" + TAG="${REST##*:}" + OS="$(docker version --format '{{.Server.Os}}')" + ARCH="$(docker version --format '{{.Server.Arch}}')" + + WORK="$(mktemp -d)" + trap 'rm -rf "${WORK}"' EXIT + + registry_get "https://${HOST}/v2/${REPO}/manifests/${TAG}" "${WORK}/index.json" + INDEX_DIGEST="sha256:$(sha256_of "${WORK}/index.json")" + + if jq -e '.manifests' "${WORK}/index.json" >/dev/null; then + MANIFEST_DIGEST="$(jq -r --arg os "${OS}" --arg arch "${ARCH}" '[.manifests[] | select(.platform.os == $os and .platform.architecture == $arch)][0].digest // empty' "${WORK}/index.json")" + if [ -z "${MANIFEST_DIGEST}" ]; then + echo "${IMAGE} has no ${OS}/${ARCH} image" >&2 + exit 1 + fi + registry_get "https://${HOST}/v2/${REPO}/manifests/${MANIFEST_DIGEST}" "${WORK}/manifest.json" + if [ "sha256:$(sha256_of "${WORK}/manifest.json")" != "${MANIFEST_DIGEST}" ]; then + echo "the ${OS}/${ARCH} manifest of ${IMAGE} does not match its digest" >&2 + exit 1 + fi + else + MANIFEST_DIGEST="${INDEX_DIGEST}" + cp "${WORK}/index.json" "${WORK}/manifest.json" + fi + + CONFIG_DIGEST="$(jq -r '.config.digest' "${WORK}/manifest.json")" + echo "key=${CONFIG_DIGEST}" + echo "digests=${INDEX_DIGEST} ${MANIFEST_DIGEST} ${CONFIG_DIGEST}" +} + +# an image's ID is its config digest (or, with the containerd image store, its manifest or index digest) +image_matches() { + ID="$(docker image inspect --format '{{.Id}}' "$1" 2>/dev/null)" || return 1 + case " ${DIGESTS} " in + *" ${ID} "*) return 0 ;; + esac + return 1 +} + +ensure() { + IMAGE="$1" + TARBALL="$2" + + if [ -f "${TARBALL}" ]; then + if docker load -i "${TARBALL}" && image_matches "${IMAGE}"; then + echo "loaded ${IMAGE} from the cache" + return 0 + fi + echo "::warning::the cached ${IMAGE} does not match the registry's digests; pulling it instead" + docker image rm -f "${IMAGE}" >/dev/null 2>&1 || true + rm -f "${TARBALL}" + fi + + ATTEMPT=1 + until docker pull "${IMAGE}"; do + if [ "${ATTEMPT}" -ge 5 ]; then + echo "docker pull ${IMAGE} failed after ${ATTEMPT} attempts" >&2 + return 1 + fi + echo "docker pull failed, attempt ${ATTEMPT}/5" + sleep $((ATTEMPT * 5)) + ATTEMPT=$((ATTEMPT + 1)) + done + + # the tag can move between `resolve` and the pull; then this run uses the image, but does not cache it + if image_matches "${IMAGE}"; then + docker save -o "${TARBALL}" "${IMAGE}" + else + echo "::warning::the pulled ${IMAGE} no longer matches the digests looked up earlier; not caching it" + fi +} + +case "${1-}" in + resolve) resolve "${2-}" ;; + ensure) ensure "${2-}" "${3-}" ;; + *) + echo 'usage: docker-image.sh resolve | ensure ' >&2 + exit 2 + ;; +esac diff --git a/.github/workflows/tests-alpine.yml b/.github/workflows/tests-alpine.yml index b64c0557..941f497d 100644 --- a/.github/workflows/tests-alpine.yml +++ b/.github/workflows/tests-alpine.yml @@ -67,14 +67,22 @@ jobs: node-version: 'lts/*' skip-ls-check: true - run: npm ls urchin + # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's + # anonymous pull rate limits, and cache the image; a cached image is only used + # if it matches the digests the registry serves now + - name: 'Resolve the alpine:${{ matrix.alpine }} image digests' + id: image + run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/alpine:${{ matrix.alpine }} >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/docker-image.tar + key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }} + - name: 'Load or pull the alpine:${{ matrix.alpine }} image' + env: + DIGESTS: ${{ steps.image.outputs.digests }} + run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/alpine:${{ matrix.alpine }} "${RUNNER_TEMP}/docker-image.tar" - name: 'Run fast tests on Alpine ${{ matrix.alpine }} (${{ matrix.arch }})' run: | - # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's - # anonymous pull rate limits; retry to tolerate transient registry failures - for i in 1 2 3 4 5; do - docker pull mirror.gcr.io/library/alpine:${{ matrix.alpine }} && break - echo "docker pull failed, attempt $i/5"; sleep $((i * 5)) - done docker run --rm \ -v "${{ github.workspace }}:/workspace" \ -w /workspace \ @@ -151,14 +159,22 @@ jobs: - uses: actions/checkout@v6 with: submodules: false + # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's + # anonymous pull rate limits, and cache the image; a cached image is only used + # if it matches the digests the registry serves now + - name: 'Resolve the alpine:${{ matrix.alpine }} image digests' + id: image + run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/alpine:${{ matrix.alpine }} >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/docker-image.tar + key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }} + - name: 'Load or pull the alpine:${{ matrix.alpine }} image' + env: + DIGESTS: ${{ steps.image.outputs.digests }} + run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/alpine:${{ matrix.alpine }} "${RUNNER_TEMP}/docker-image.tar" - name: 'Install node ${{ matrix.node }} from a musl binary on Alpine ${{ matrix.alpine }}' run: | - # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's - # anonymous pull rate limits; retry to tolerate transient registry failures - for i in 1 2 3 4 5; do - docker pull mirror.gcr.io/library/alpine:${{ matrix.alpine }} && break - echo "docker pull failed, attempt $i/5"; sleep $((i * 5)) - done docker run --rm \ -v "${{ github.workspace }}:/workspace" \ -w /workspace \ diff --git a/.github/workflows/tests-installation-node.yml b/.github/workflows/tests-installation-node.yml index ed4dcff7..78ec23c5 100644 --- a/.github/workflows/tests-installation-node.yml +++ b/.github/workflows/tests-installation-node.yml @@ -64,15 +64,22 @@ jobs: skip-ls-check: true - run: npm ls urchin - run: npx which urchin + # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's + # anonymous pull rate limits, and cache the image; a cached image is only used + # if it matches the digests the registry serves now + - name: Resolve the ubuntu:16.04 image digests + id: image + run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/ubuntu:16.04 >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/docker-image.tar + key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }} + - name: Load or pull the ubuntu:16.04 image + env: + DIGESTS: ${{ steps.image.outputs.digests }} + run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/ubuntu:16.04 "${RUNNER_TEMP}/docker-image.tar" - name: Run installation_node tests in container run: | - # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's - # anonymous pull rate limits; retry to tolerate transient registry failures - for i in 1 2 3 4 5; do - docker pull mirror.gcr.io/library/ubuntu:16.04 && break - echo "docker pull failed, attempt $i/5" - sleep $((i * 5)) - done docker run --rm \ -v "${{ github.workspace }}:/workspace" \ -w /workspace \ diff --git a/.github/workflows/tests-xenial.yml b/.github/workflows/tests-xenial.yml index f9a727bd..9ba7538f 100644 --- a/.github/workflows/tests-xenial.yml +++ b/.github/workflows/tests-xenial.yml @@ -61,15 +61,22 @@ jobs: skip-ls-check: true - run: npm ls urchin - run: npx which urchin + # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's + # anonymous pull rate limits, and cache the image; a cached image is only used + # if it matches the digests the registry serves now + - name: Resolve the ubuntu:16.04 image digests + id: image + run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/ubuntu:16.04 >> "$GITHUB_OUTPUT" + - uses: actions/cache@v6 + with: + path: ${{ runner.temp }}/docker-image.tar + key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }} + - name: Load or pull the ubuntu:16.04 image + env: + DIGESTS: ${{ steps.image.outputs.digests }} + run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/ubuntu:16.04 "${RUNNER_TEMP}/docker-image.tar" - name: Run xenial tests in container run: | - # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's - # anonymous pull rate limits; retry to tolerate transient registry failures - for i in 1 2 3 4 5; do - docker pull mirror.gcr.io/library/ubuntu:16.04 && break - echo "docker pull failed, attempt $i/5" - sleep $((i * 5)) - done docker run --rm \ -v "${{ github.workspace }}:/workspace" \ -w /workspace \