[actions] retry apt package installs through Ubuntu mirror outages

Mirror outages have outlasted the existing retries,
which covered only `apt-get update` (and not the WSL `additional-packages` install).
Installs now go through `.github/scripts/apt-install.sh`,
which retries each request within apt and retries the whole update+install with backoff for about 10 minutes.

.github/scripts/apt-install.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+
+# Install apt packages, riding out Ubuntu/Debian mirror outages: apt itself
+# retries each request, and the whole update+install is retried with backoff
+# for about 10 minutes, since outages have lasted longer than apt's own retries.
+#
+# usage: sh .github/scripts/apt-install.sh <package>...
+# Runs apt-get through sudo when not root.
+
+set -u
+
+if [ "$#" -eq 0 ]; then
+  echo 'usage: apt-install.sh <package>...' >&2
+  exit 2
+fi
+
+SUDO=''
+if [ "$(id -u)" != '0' ]; then
+  SUDO='sudo'
+fi
+
+apt_get() {
+  $SUDO env DEBIAN_FRONTEND=noninteractive apt-get \
+    -o Acquire::Retries=5 \
+    -o Acquire::http::Timeout=30 \
+    -o Acquire::https::Timeout=30 \
+    "$@"
+}
+
+ATTEMPT=1
+MAX_ATTEMPTS=8
+DELAY=15
+while :; do
+  apt_get update || echo "apt-get update failed (attempt ${ATTEMPT}/${MAX_ATTEMPTS})" >&2
+  if apt_get install -y "$@"; then
+    exit 0
+  fi
+  if [ "${ATTEMPT}" -ge "${MAX_ATTEMPTS}" ]; then
+    echo "apt-get install failed after ${MAX_ATTEMPTS} attempts: $*" >&2
+    exit 1
+  fi
+  echo "apt-get install failed (attempt ${ATTEMPT}/${MAX_ATTEMPTS}); retrying in ${DELAY}s" >&2
+  sleep "${DELAY}"
+  ATTEMPT=$((ATTEMPT + 1))
+  DELAY=$((DELAY * 2))
+  if [ "${DELAY}" -gt 120 ]; then
+    DELAY=120
+  fi
+done

.github/workflows/tests-fast.yml

@@ -60,11 +60,11 @@ jobs:
           git submodule update --init --recursive
       - name: Install zsh, additional shells, and awk variant
         run: |
-          sudo apt-get update
+          PACKAGES="zsh ${{ matrix.awk }}"
-          sudo apt-get install -y zsh ${{ matrix.awk }}
           if [ "${{ matrix.shell }}" != "sh" ] && [ "${{ matrix.shell }}" != "bash" ] && [ "${{ matrix.shell }}" != "zsh" ]; then
-            sudo apt-get install -y ${{ matrix.shell }}
+            PACKAGES="${PACKAGES} ${{ matrix.shell }}"
           fi
+          sh .github/scripts/apt-install.sh ${PACKAGES}
           # Set the selected awk as the default
           sudo update-alternatives --set awk /usr/bin/${{ matrix.awk }}
         shell: bash
@@ -194,7 +194,11 @@ jobs:
       - uses: Vampire/setup-wsl@v7
         with:
           distribution: ${{ matrix.wsl-distrib }}
-          additional-packages: git make zsh dash sudo curl wget ca-certificates nodejs npm
+      - name: Install packages with retries
+        # the Windows checkout has CRLF line endings
+        run: |
+          tr -d '\r' < "$(wslpath "${{ github.workspace }}")/.github/scripts/apt-install.sh" > /tmp/apt-install.sh
+          sh /tmp/apt-install.sh git make zsh dash sudo curl wget ca-certificates nodejs npm
       - name: 'Clone into the WSL filesystem as a non-root user'
         # the Windows checkout shows every file as executable, which urchin would run as tests
         run: |

.github/workflows/tests-installation-iojs.yml

@@ -52,11 +52,11 @@ jobs:
           git submodule update --init --recursive
       - name: Install zsh and additional shells
         run: |
-          sudo apt-get update
+          PACKAGES='zsh'
-          sudo apt-get install -y zsh
           if [ "${{ matrix.shell }}" != "sh" ] && [ "${{ matrix.shell }}" != "bash" ] && [ "${{ matrix.shell }}" != "zsh" ]; then
-            sudo apt-get install -y ${{ matrix.shell }}
+            PACKAGES="${PACKAGES} ${{ matrix.shell }}"
           fi
+          sh .github/scripts/apt-install.sh ${PACKAGES}
         shell: bash
       - run: sudo ${{ matrix.shell }} --version 2> /dev/null || dpkg -s ${{ matrix.shell }} 2> /dev/null || which ${{ matrix.shell }}
       - run: wget --version
@@ -95,7 +95,7 @@ jobs:
         shell: bash
       - name: Restore curl
         if: always()
-        run: sudo apt-get install curl -y
+        run: sh .github/scripts/apt-install.sh curl
         shell: bash
 
   installation_iojs_source_compile:

.github/workflows/tests-installation-node.yml

@@ -83,21 +83,9 @@ jobs:
             bash -c '
               set -ex
 
-              # Retry apt-get update up to 5 times due to flaky Ubuntu mirrors
+              sh /workspace/.github/scripts/apt-install.sh git curl wget make build-essential python zsh libssl-dev
-              # apt-get update can return 0 even with partial failures, so check for warnings
-              for i in 1 2 3 4 5; do
-                if apt-get update 2>&1 | tee /tmp/apt-update.log | grep -qE "^(W:|E:|Err:)"; then
-                  echo "apt-get update had warnings/errors, attempt $i/5"
-                  cat /tmp/apt-update.log
-                  sleep $((i * 5))
-                else
-                  break
-                fi
-              done
-
-              apt-get install -y git curl wget make build-essential python zsh libssl-dev
               if [ "$TEST_SHELL" != "sh" ] && [ "$TEST_SHELL" != "bash" ]; then
-                apt-get install -y $TEST_SHELL || true
+                sh /workspace/.github/scripts/apt-install.sh $TEST_SHELL || true
               fi
 
               # Use nvm to install Node.js for running urchin

.github/workflows/tests-xenial.yml

@@ -79,21 +79,9 @@ jobs:
             bash -c '
               set -ex
 
-              # Retry apt-get update up to 5 times due to flaky Ubuntu mirrors
+              sh /workspace/.github/scripts/apt-install.sh git curl wget make build-essential python zsh libssl-dev
-              # apt-get update can return 0 even with partial failures, so check for warnings
-              for i in 1 2 3 4 5; do
-                if apt-get update 2>&1 | tee /tmp/apt-update.log | grep -qE "^(W:|E:|Err:)"; then
-                  echo "apt-get update had warnings/errors, attempt $i/5"
-                  cat /tmp/apt-update.log
-                  sleep $((i * 5))
-                else
-                  break
-                fi
-              done
-
-              apt-get install -y git curl wget make build-essential python zsh libssl-dev
               if [ "$TEST_SHELL" != "sh" ] && [ "$TEST_SHELL" != "bash" ]; then
-                apt-get install -y $TEST_SHELL || true
+                sh /workspace/.github/scripts/apt-install.sh $TEST_SHELL || true
               fi
 
               # Use nvm to install Node.js for running urchin

.github/workflows/tests.yml

@@ -54,7 +54,7 @@ jobs:
             azure.archive.ubuntu.com:80
             packages.microsoft.com:443
       - uses: actions/checkout@v6
-      - run: sudo apt-get update; sudo apt-get install ${{ matrix.shell }}
+      - run: sh .github/scripts/apt-install.sh ${{ matrix.shell }}
         if: matrix.shell == 'zsh' || matrix.shell == 'ksh'
         # zsh (https://github.com/actions/runner-images/issues/264) and ksh are not in the ubuntu image
         shell: bash

.github/workflows/windows-nvm.yml

@@ -145,12 +145,35 @@ jobs:
           - ''
           - 'script'
     steps:
-      # For Ubuntu: install with packages directly
+      # For Ubuntu: `additional-packages` can't retry, so install in a separate step
       - if: ${{ !startsWith(matrix.wsl-distrib, 'Debian') }}
         uses: Vampire/setup-wsl@v7
         with:
           distribution: ${{ matrix.wsl-distrib }}
-          additional-packages: bash git curl ca-certificates wget
+      # mirror outages can outlast apt's own retries, so retry update+install with
+      # backoff for about 10 minutes (as .github/scripts/apt-install.sh does; this
+      # job has no checkout)
+      - if: ${{ !startsWith(matrix.wsl-distrib, 'Debian') }}
+        name: 'Install packages with retries'
+        shell: 'wsl-bash {0}'
+        run: |
+          attempt=1
+          delay=15
+          while true; do
+            apt-get -o Acquire::Retries=5 update || true
+            if apt-get -o Acquire::Retries=5 install --yes bash git curl ca-certificates wget; then
+              break
+            fi
+            if [ "${attempt}" -ge 8 ]; then
+              echo "apt-get install failed after ${attempt} attempts" >&2
+              exit 1
+            fi
+            echo "apt-get install failed (attempt ${attempt}/8); retrying in ${delay}s" >&2
+            sleep "${delay}"
+            attempt=$((attempt + 1))
+            delay=$((delay * 2))
+            if [ "${delay}" -gt 120 ]; then delay=120; fi
+          done
 
       # For Debian, install without packages: `additional-packages` would run
       # `apt-get update` before we get to fix up sources.list