From 50e73c4cd53c23260f71aa5cbcd2e0dcb2e8ef85 Mon Sep 17 00:00:00 2001 From: Jordan Harband Date: Wed, 7 Oct 2026 15:10:36 -0700 Subject: [PATCH] [actions] retry apt package installs through Ubuntu mirror outages Mirror outages have outlasted the existing retries, which covered only `apt-get update` (and not the WSL `additional-packages` install). Installs now go through `.github/scripts/apt-install.sh`, which retries each request within apt and retries the whole update+install with backoff for about 10 minutes. --- .github/scripts/apt-install.sh | 49 +++++++++++++++++++ .github/workflows/tests-fast.yml | 12 +++-- .github/workflows/tests-installation-iojs.yml | 8 +-- .github/workflows/tests-installation-node.yml | 16 +----- .github/workflows/tests-xenial.yml | 16 +----- .github/workflows/tests.yml | 2 +- .github/workflows/windows-nvm.yml | 27 +++++++++- 7 files changed, 91 insertions(+), 39 deletions(-) create mode 100644 .github/scripts/apt-install.sh diff --git a/.github/scripts/apt-install.sh b/.github/scripts/apt-install.sh new file mode 100644 index 00000000..40269809 --- /dev/null +++ b/.github/scripts/apt-install.sh @@ -0,0 +1,49 @@ +#!/bin/sh + +# Install apt packages, riding out Ubuntu/Debian mirror outages: apt itself +# retries each request, and the whole update+install is retried with backoff +# for about 10 minutes, since outages have lasted longer than apt's own retries. +# +# usage: sh .github/scripts/apt-install.sh ... +# Runs apt-get through sudo when not root. + +set -u + +if [ "$#" -eq 0 ]; then + echo 'usage: apt-install.sh ...' >&2 + exit 2 +fi + +SUDO='' +if [ "$(id -u)" != '0' ]; then + SUDO='sudo' +fi + +apt_get() { + $SUDO env DEBIAN_FRONTEND=noninteractive apt-get \ + -o Acquire::Retries=5 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + "$@" +} + +ATTEMPT=1 +MAX_ATTEMPTS=8 +DELAY=15 +while :; do + apt_get update || echo "apt-get update failed (attempt ${ATTEMPT}/${MAX_ATTEMPTS})" >&2 + if apt_get install -y "$@"; then + exit 0 + fi + if [ "${ATTEMPT}" -ge "${MAX_ATTEMPTS}" ]; then + echo "apt-get install failed after ${MAX_ATTEMPTS} attempts: $*" >&2 + exit 1 + fi + echo "apt-get install failed (attempt ${ATTEMPT}/${MAX_ATTEMPTS}); retrying in ${DELAY}s" >&2 + sleep "${DELAY}" + ATTEMPT=$((ATTEMPT + 1)) + DELAY=$((DELAY * 2)) + if [ "${DELAY}" -gt 120 ]; then + DELAY=120 + fi +done diff --git a/.github/workflows/tests-fast.yml b/.github/workflows/tests-fast.yml index 0bf37a7d..9251594f 100644 --- a/.github/workflows/tests-fast.yml +++ b/.github/workflows/tests-fast.yml @@ -60,11 +60,11 @@ jobs: git submodule update --init --recursive - name: Install zsh, additional shells, and awk variant run: | - sudo apt-get update - sudo apt-get install -y zsh ${{ matrix.awk }} + PACKAGES="zsh ${{ matrix.awk }}" if [ "${{ matrix.shell }}" != "sh" ] && [ "${{ matrix.shell }}" != "bash" ] && [ "${{ matrix.shell }}" != "zsh" ]; then - sudo apt-get install -y ${{ matrix.shell }} + PACKAGES="${PACKAGES} ${{ matrix.shell }}" fi + sh .github/scripts/apt-install.sh ${PACKAGES} # Set the selected awk as the default sudo update-alternatives --set awk /usr/bin/${{ matrix.awk }} shell: bash @@ -194,7 +194,11 @@ jobs: - uses: Vampire/setup-wsl@v7 with: distribution: ${{ matrix.wsl-distrib }} - additional-packages: git make zsh dash sudo curl wget ca-certificates nodejs npm + - name: Install packages with retries + # the Windows checkout has CRLF line endings + run: | + tr -d '\r' < "$(wslpath "${{ github.workspace }}")/.github/scripts/apt-install.sh" > /tmp/apt-install.sh + sh /tmp/apt-install.sh git make zsh dash sudo curl wget ca-certificates nodejs npm - name: 'Clone into the WSL filesystem as a non-root user' # the Windows checkout shows every file as executable, which urchin would run as tests run: | diff --git a/.github/workflows/tests-installation-iojs.yml b/.github/workflows/tests-installation-iojs.yml index eb4160da..360947e5 100644 --- a/.github/workflows/tests-installation-iojs.yml +++ b/.github/workflows/tests-installation-iojs.yml @@ -52,11 +52,11 @@ jobs: git submodule update --init --recursive - name: Install zsh and additional shells run: | - sudo apt-get update - sudo apt-get install -y zsh + PACKAGES='zsh' if [ "${{ matrix.shell }}" != "sh" ] && [ "${{ matrix.shell }}" != "bash" ] && [ "${{ matrix.shell }}" != "zsh" ]; then - sudo apt-get install -y ${{ matrix.shell }} + PACKAGES="${PACKAGES} ${{ matrix.shell }}" fi + sh .github/scripts/apt-install.sh ${PACKAGES} shell: bash - run: sudo ${{ matrix.shell }} --version 2> /dev/null || dpkg -s ${{ matrix.shell }} 2> /dev/null || which ${{ matrix.shell }} - run: wget --version @@ -95,7 +95,7 @@ jobs: shell: bash - name: Restore curl if: always() - run: sudo apt-get install curl -y + run: sh .github/scripts/apt-install.sh curl shell: bash installation_iojs_source_compile: diff --git a/.github/workflows/tests-installation-node.yml b/.github/workflows/tests-installation-node.yml index ef53c15f..a26e592e 100644 --- a/.github/workflows/tests-installation-node.yml +++ b/.github/workflows/tests-installation-node.yml @@ -83,21 +83,9 @@ jobs: bash -c ' set -ex - # Retry apt-get update up to 5 times due to flaky Ubuntu mirrors - # apt-get update can return 0 even with partial failures, so check for warnings - for i in 1 2 3 4 5; do - if apt-get update 2>&1 | tee /tmp/apt-update.log | grep -qE "^(W:|E:|Err:)"; then - echo "apt-get update had warnings/errors, attempt $i/5" - cat /tmp/apt-update.log - sleep $((i * 5)) - else - break - fi - done - - apt-get install -y git curl wget make build-essential python zsh libssl-dev + sh /workspace/.github/scripts/apt-install.sh git curl wget make build-essential python zsh libssl-dev if [ "$TEST_SHELL" != "sh" ] && [ "$TEST_SHELL" != "bash" ]; then - apt-get install -y $TEST_SHELL || true + sh /workspace/.github/scripts/apt-install.sh $TEST_SHELL || true fi # Use nvm to install Node.js for running urchin diff --git a/.github/workflows/tests-xenial.yml b/.github/workflows/tests-xenial.yml index ca946c50..f60e0c2a 100644 --- a/.github/workflows/tests-xenial.yml +++ b/.github/workflows/tests-xenial.yml @@ -79,21 +79,9 @@ jobs: bash -c ' set -ex - # Retry apt-get update up to 5 times due to flaky Ubuntu mirrors - # apt-get update can return 0 even with partial failures, so check for warnings - for i in 1 2 3 4 5; do - if apt-get update 2>&1 | tee /tmp/apt-update.log | grep -qE "^(W:|E:|Err:)"; then - echo "apt-get update had warnings/errors, attempt $i/5" - cat /tmp/apt-update.log - sleep $((i * 5)) - else - break - fi - done - - apt-get install -y git curl wget make build-essential python zsh libssl-dev + sh /workspace/.github/scripts/apt-install.sh git curl wget make build-essential python zsh libssl-dev if [ "$TEST_SHELL" != "sh" ] && [ "$TEST_SHELL" != "bash" ]; then - apt-get install -y $TEST_SHELL || true + sh /workspace/.github/scripts/apt-install.sh $TEST_SHELL || true fi # Use nvm to install Node.js for running urchin diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3facf241..5473add0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,7 +54,7 @@ jobs: azure.archive.ubuntu.com:80 packages.microsoft.com:443 - uses: actions/checkout@v6 - - run: sudo apt-get update; sudo apt-get install ${{ matrix.shell }} + - run: sh .github/scripts/apt-install.sh ${{ matrix.shell }} if: matrix.shell == 'zsh' || matrix.shell == 'ksh' # zsh (https://github.com/actions/runner-images/issues/264) and ksh are not in the ubuntu image shell: bash diff --git a/.github/workflows/windows-nvm.yml b/.github/workflows/windows-nvm.yml index 29333351..ee9ece3e 100644 --- a/.github/workflows/windows-nvm.yml +++ b/.github/workflows/windows-nvm.yml @@ -145,12 +145,35 @@ jobs: - '' - 'script' steps: - # For Ubuntu: install with packages directly + # For Ubuntu: `additional-packages` can't retry, so install in a separate step - if: ${{ !startsWith(matrix.wsl-distrib, 'Debian') }} uses: Vampire/setup-wsl@v7 with: distribution: ${{ matrix.wsl-distrib }} - additional-packages: bash git curl ca-certificates wget + # mirror outages can outlast apt's own retries, so retry update+install with + # backoff for about 10 minutes (as .github/scripts/apt-install.sh does; this + # job has no checkout) + - if: ${{ !startsWith(matrix.wsl-distrib, 'Debian') }} + name: 'Install packages with retries' + shell: 'wsl-bash {0}' + run: | + attempt=1 + delay=15 + while true; do + apt-get -o Acquire::Retries=5 update || true + if apt-get -o Acquire::Retries=5 install --yes bash git curl ca-certificates wget; then + break + fi + if [ "${attempt}" -ge 8 ]; then + echo "apt-get install failed after ${attempt} attempts" >&2 + exit 1 + fi + echo "apt-get install failed (attempt ${attempt}/8); retrying in ${delay}s" >&2 + sleep "${delay}" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + if [ "${delay}" -gt 120 ]; then delay=120; fi + done # For Debian, install without packages: `additional-packages` would run # `apt-get update` before we get to fix up sources.list