Run maintainer and canary CI on self-hosted RunsOn runners

.github/actions/breeze/action.yml

@@ -29,6 +29,8 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: "RunsOn cost summary"
+      uses: ./.github/actions/runs-on-cost-summary
     - name: "Setup python"
       uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
       with:

.github/actions/prepare_breeze_and_image/action.yml

@@ -82,12 +82,13 @@ runs:
     - name: >
         Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }}
         ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }}
-      # yamllint disable-line rule:line-length
-      uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0  # restore/v1.0.0
+      uses: ./.github/actions/stash-restore
       with:
         key: "${{ inputs.image-type }}-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}\
           ${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}"
         path: "/mnt/"
+        # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
+        # stashes as S3 digests only.
         only-current-branch: 'true'
         fail-on-download: 'true'
     - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}"

.github/actions/prepare_single_ci_image/action.yml

@@ -36,11 +36,12 @@ runs:
   using: "composite"
   steps:
     - name: "Restore CI docker images ${{ inputs.platform }}:${{ inputs.python }}"
-      # yamllint disable-line rule:line-length
-      uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0  # restore/v1.0.0
+      uses: ./.github/actions/stash-restore
       with:
         key: ci-image-save-v3-${{ inputs.platform }}-${{ inputs.python }}
         path: "/mnt/"
+        # Must stay true: runs on RunsOn (canaries and release-branch pushes) save these image
+        # stashes as S3 digests only.
         only-current-branch: 'true'
         fail-on-download: 'true'
       if: contains(inputs.python-versions-list-as-string, inputs.python)

.github/actions/runs-on-cost-summary/action.yml

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+# Puts the job's EC2 cost next to the GitHub-hosted price in the job summary. Used by every job
+# that can run on RunsOn: through the breeze action, or directly by jobs that do not install Breeze.
+name: 'RunsOn cost summary'
+description: 'Add the EC2 cost of a RunsOn job to its summary; a no-op elsewhere.'
+runs:
+  using: 'composite'
+  steps:
+    - name: "RunsOn cost summary"
+      if: runner.environment == 'self-hosted'
+      uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3  # v2.5.0
+      with:
+        show_costs: summary

.github/actions/stash-restore/action.yml

@@ -0,0 +1,137 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+# apache/infrastructure-actions/stash/restore on GitHub-hosted runners. On RunsOn (the only
+# self-hosted runners) multi-GB image tarballs stay in the RunsOn stack's in-region S3 cache
+# bucket instead of leaving AWS as GitHub artifacts.
+#
+# Every RunsOn job can write anywhere in that bucket, so the bucket is not trusted: objects are
+# named by their sha256, and the digest itself travels as a small ASF stash artifact, which
+# GitHub scopes to the run / branch exactly like a regular stash. A blob that does not match the
+# digest the ASF restore found for this run, branch and repository is never extracted. The bucket's
+# lifecycle rule (S3CacheExpirationInDays) must outlive the longest retention-days of a digest
+# (6 days today), or a digest hit finds no blob.
+name: 'Restore stash'
+description: 'Restore a stash from GitHub artifacts, or from S3 on RunsOn runners.'
+inputs:
+  key:
+    description: 'Stash key.'
+    required: true
+  path:
+    description: >
+      Directory to restore into (GitHub-hosted only; the S3 tarball restores entries to the
+      paths they were saved from, and the restore then fails if any of them is outside this directory).
+    required: false
+    default: ${{ github.workspace }}
+  only-current-branch:
+    description: 'Do not fall back to the base branch stash.'
+    required: false
+    default: 'false'
+  fail-on-download:
+    description: 'Fail when a stash was found but could not be restored.'
+    required: false
+    default: 'false'
+outputs:
+  stash-hit:
+    description: "'true' if a stash was restored, 'false' otherwise."
+    value: ${{ steps.s3.outputs.stash-hit || steps.artifact.outputs.stash-hit }}
+runs:
+  using: 'composite'
+  steps:
+    - name: "Restore stash from GitHub artifacts"
+      id: artifact
+      if: runner.environment != 'self-hosted'
+      # yamllint disable-line rule:line-length
+      uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3  # restore/v1.0.1
+      with:
+        key: ${{ inputs.key }}
+        path: ${{ inputs.path }}
+        only-current-branch: ${{ inputs.only-current-branch }}
+        fail-on-download: ${{ inputs.fail-on-download }}
+    - name: "Restore stash digest from GitHub artifacts"
+      id: digest
+      if: runner.environment == 'self-hosted'
+      # yamllint disable-line rule:line-length
+      uses: apache/infrastructure-actions/stash/restore@c968777141a2afc98d1a09c9f6979455bfebaae3  # restore/v1.0.1
+      with:
+        key: ${{ inputs.key }}-s3
+        path: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}
+        only-current-branch: ${{ inputs.only-current-branch }}
+        fail-on-download: ${{ inputs.fail-on-download }}
+    - name: "Restore stash from S3"
+      id: s3
+      if: runner.environment == 'self-hosted'
+      shell: bash
+      env:
+        STASH_KEY: ${{ inputs.key }}
+        STASH_PATH: ${{ inputs.path }}
+        DIGEST_HIT: ${{ steps.digest.outputs.stash-hit }}
+        DIGEST_FILE: ${{ runner.temp }}/stash-digest-restore/${{ inputs.key }}/sha256
+        FAIL_ON_DOWNLOAD: ${{ inputs.fail-on-download }}
+      run: |
+        set -uo pipefail
+        payload="${RUNNER_TEMP}/stash-restore-$$.tar"
+        entries="${RUNNER_TEMP}/stash-restore-$$.entries"
+        miss() {
+          rm -f "${payload}" "${entries}"
+          echo "stash-hit=false" >> "${GITHUB_OUTPUT}"
+          if [[ "${FAIL_ON_DOWNLOAD}" == "true" && "${DIGEST_HIT}" == "true" ]]; then
+            echo "::error::$1"
+            exit 1
+          fi
+          echo "$1"
+          exit 0
+        }
+        if [[ "${DIGEST_HIT}" != "true" ]]; then
+          miss "No stash '${STASH_KEY}'"
+        fi
+        digest="$(cat "${DIGEST_FILE}")"
+        if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
+          miss "Malformed digest for stash '${STASH_KEY}'"
+        fi
+        bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
+        export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
+        object="s3://${bucket}/cache/airflow-stash/sha256/${digest}.tar"
+        if ! aws s3 cp --only-show-errors "${object}" "${payload}"; then
+          miss "Could not download ${object} for stash '${STASH_KEY}'"
+        fi
+        if ! echo "${digest}  ${payload}" | sha256sum --check --status; then
+          miss "${object} does not match the digest recorded for stash '${STASH_KEY}'"
+        fi
+        # GNU tar detects the compression stash-save chose.
+        if ! tar -xPvf "${payload}" > "${entries}"; then
+          miss "Could not extract ${object} for stash '${STASH_KEY}'"
+        fi
+        rm -f "${payload}"
+        # The S3 tarball ignores `path`: fail loudly if a caller saves and restores different
+        # directories, which would work on GitHub-hosted runners only.
+        restore_dir="$(realpath -ms "${STASH_PATH}")"
+        restore_dir="${restore_dir%/}"
+        outside=()
+        while IFS= read -r entry; do
+          [[ "${entry}" == /* ]] || entry="${GITHUB_WORKSPACE}/${entry}"
+          [[ "${entry}" == "${restore_dir}/"* ]] || outside+=("${entry}")
+        done < "${entries}"
+        rm -f "${entries}"
+        if [[ ${#outside[@]} -gt 0 ]]; then
+          printf '%s\n' "${outside[@]}"
+          echo "::error::Stash '${STASH_KEY}' restored the entries above outside of ${restore_dir}/"
+          exit 1
+        fi
+        echo "stash-hit=true" >> "${GITHUB_OUTPUT}"
+        echo "Restored stash '${STASH_KEY}' from ${object}"

.github/actions/stash-save/action.yml

@@ -0,0 +1,129 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+# apache/infrastructure-actions/stash/save on GitHub-hosted runners. On RunsOn the tarball goes
+# to the stack's S3 cache bucket under its sha256 and only the digest is saved as an ASF stash
+# artifact. See stash-restore for why.
+name: 'Save stash'
+description: 'Save a stash to GitHub artifacts, or to S3 on RunsOn runners.'
+inputs:
+  key:
+    description: 'Stash key.'
+    required: true
+  path:
+    description: 'File, directory or glob to stash.'
+    required: true
+  if-no-files-found:
+    description: "'error' fails when path matches nothing, 'warn' or 'ignore' do not."
+    required: false
+    default: 'warn'
+  retention-days:
+    description: >
+      Days the stash (on RunsOn: its digest) is kept. On RunsOn the stack's
+      S3CacheExpirationInDays lifecycle rule expires the tarball itself.
+    required: false
+    default: '5'
+  compression-level:
+    description: "Zlib compression level (0-9). On RunsOn, '0' stores the tarball uncompressed."
+    required: false
+    default: '6'
+runs:
+  using: 'composite'
+  steps:
+    - name: "Save stash to GitHub artifacts"
+      if: runner.environment != 'self-hosted'
+      uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9  # save/v1.0.2
+      with:
+        key: ${{ inputs.key }}
+        path: ${{ inputs.path }}
+        if-no-files-found: ${{ inputs.if-no-files-found }}
+        retention-days: ${{ inputs.retention-days }}
+        compression-level: ${{ inputs.compression-level }}
+    - name: "Save stash to S3"
+      id: s3
+      if: runner.environment == 'self-hosted'
+      shell: bash
+      env:
+        STASH_KEY: ${{ inputs.key }}
+        STASH_PATH: ${{ inputs.path }}
+        DIGEST_DIR: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
+        IF_NO_FILES_FOUND: ${{ inputs.if-no-files-found }}
+        COMPRESSION_LEVEL: ${{ inputs.compression-level }}
+      run: |
+        set -euo pipefail
+        bucket="${RUNS_ON_S3_BUCKET_CACHE:?not running on a RunsOn runner}"
+        export AWS_DEFAULT_REGION="${RUNS_ON_AWS_REGION:?not running on a RunsOn runner}"
+        shopt -s nullglob
+        # shellcheck disable=SC2206
+        candidates=( ${STASH_PATH} )
+        existing=()
+        for p in "${candidates[@]}"; do
+          if [[ -e "${p}" ]]; then
+            existing+=("${p}")
+          fi
+        done
+        if [[ ${#existing[@]} -eq 0 ]]; then
+          if [[ "${IF_NO_FILES_FOUND}" == "error" ]]; then
+            echo "::error::No files found for path '${STASH_PATH}'"
+            exit 1
+          fi
+          if [[ "${IF_NO_FILES_FOUND}" == "warn" ]]; then
+            echo "::warning::No files found for path '${STASH_PATH}', skipping save"
+          fi
+          exit 0
+        fi
+        # The tarball is streamed to S3 while being hashed: a multi-GB image never touches the
+        # runner's disk. Its key is only known at the end, so it lands under a temporary key and
+        # is copied server-side to its digest key.
+        prefix="s3://${bucket}/cache/airflow-stash"
+        upload="${prefix}/uploads/$(cat /proc/sys/kernel/random/uuid).tar"
+        digest_fifo="${RUNNER_TEMP}/stash-save-$$.fifo"
+        digest_file="${RUNNER_TEMP}/stash-save-$$.sha256"
+        trap 'rm -f "${digest_fifo}" "${digest_file}"' EXIT
+        mkfifo "${digest_fifo}"
+        sha256sum < "${digest_fifo}" | cut -d ' ' -f 1 > "${digest_file}" &
+        digest_pid=$!
+        compression=()
+        if [[ "${COMPRESSION_LEVEL}" != "0" ]]; then
+          compression=(-z)
+        fi
+        tar -cP "${compression[@]}" -f - "${existing[@]}" \
+          | tee "${digest_fifo}" \
+          | aws s3 cp --only-show-errors - "${upload}"
+        wait "${digest_pid}"
+        digest="$(cat "${digest_file}")"
+        if [[ ! "${digest}" =~ ^[0-9a-f]{64}$ ]]; then
+          echo "::error::Could not hash the stash '${STASH_KEY}'"
+          exit 1
+        fi
+        object="${prefix}/sha256/${digest}.tar"
+        aws s3 cp --only-show-errors --copy-props none "${upload}" "${object}"
+        # Leaving the upload behind only costs storage until the bucket's lifecycle rule expires it.
+        aws s3 rm --only-show-errors "${upload}" || echo "::warning::Could not remove ${upload}"
+        mkdir -p "${DIGEST_DIR}"
+        echo "${digest}" > "${DIGEST_DIR}/sha256"
+        echo "saved=true" >> "${GITHUB_OUTPUT}"
+        echo "Saved ${existing[*]} to ${object}"
+    - name: "Save stash digest to GitHub artifacts"
+      if: runner.environment == 'self-hosted' && steps.s3.outputs.saved == 'true'
+      uses: apache/infrastructure-actions/stash/save@7d3afde57801d8c9a2db9730c28ae4e036cc2ab9  # save/v1.0.2
+      with:
+        key: ${{ inputs.key }}-s3
+        path: ${{ runner.temp }}/stash-digest-save/${{ inputs.key }}
+        if-no-files-found: error
+        retention-days: ${{ inputs.retention-days }}

.github/workflows/additional-ci-image-checks.yml

@@ -62,7 +62,7 @@ jobs:
     # layer that builds Python from source, which takes 11-18 minutes for Python 3.11+ on hosted runners.
     timeout-minutes: 35
     name: Check that image builds quickly
-    runs-on: ${{ fromJSON(inputs.runners) }}
+    runs-on: ${{ format(fromJSON(inputs.runners)[0], 8, 32) }}
     env:
       UPGRADE_TO_NEWER_DEPENDENCIES: false
       PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"