From f9226a97de7b1d3e72cb947e76ff0063b3809608 Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 2 Jul 2026 22:07:31 +0000 Subject: [PATCH 1/2] fix(ci): free runner disk after image loads; diagnose e2e rollout timeouts External PR #178's run (28623232380) failed with the controller rollout timing out after a fresh install in the image-refresh lane. The job summary shows the runner bottomed out at 1.4GB free disk - inside the kubelet eviction threshold of the k3d node, the exact image-GC / pod-not-ready failure mode load-image.sh pins against. Nothing about the failure was fork-specific: the untrusted path (artifact handoff, local build, import) worked end to end. - Delete the multi-GB docker-load tarballs right after loading in every consumer job; they were sitting on disk for the whole e2e run. - fail-fast: false on the e2e matrix: quickstart's failure cancelled E2E (full) mid-run, losing its signal and erroring its report upload. - deploy-controller.sh: widen the rollout wait to 300s (fresh installs also wait on cert-manager issuing the admission cert) and dump node conditions, pods, describe, and events on timeout so the next occurrence names its cause in the log. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 27 +++++++++++++++++++++------ hack/e2e/deploy-controller.sh | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef23390c..03eb6466 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -263,7 +263,10 @@ jobs: - name: Load CI container image (PR) if: github.event_name == 'pull_request' - run: docker load -i ci-container-image.tar + # rm after load: the multi-GB tarball otherwise stays on disk for the whole + # job — e2e runners have been observed within 1.5GB of the kubelet eviction + # threshold (run 28623232380). + run: docker load -i ci-container-image.tar && rm -f ci-container-image.tar - name: Lint and package Helm chart run: | @@ -324,7 +327,10 @@ jobs: - name: Load CI container image (PR) if: github.event_name == 'pull_request' - run: docker load -i ci-container-image.tar + # rm after load: the multi-GB tarball otherwise stays on disk for the whole + # job — e2e runners have been observed within 1.5GB of the kubelet eviction + # threshold (run 28623232380). + run: docker load -i ci-container-image.tar && rm -f ci-container-image.tar - name: Run lint tasks run: | @@ -391,7 +397,10 @@ jobs: - name: Load CI container image (PR) if: github.event_name == 'pull_request' - run: docker load -i ci-container-image.tar + # rm after load: the multi-GB tarball otherwise stays on disk for the whole + # job — e2e runners have been observed within 1.5GB of the kubelet eviction + # threshold (run 28623232380). + run: docker load -i ci-container-image.tar && rm -f ci-container-image.tar - name: Run tests run: | @@ -445,7 +454,7 @@ jobs: - name: Load project image (PR) if: github.event_name == 'pull_request' - run: docker load -i project-image.tar + run: docker load -i project-image.tar && rm -f project-image.tar - name: Pull project image (trusted) if: github.event_name != 'pull_request' @@ -489,6 +498,9 @@ jobs: packages: read id-token: write strategy: + # Don't let one lane's flake cancel the other mid-run — a cancelled lane + # loses its signal *and* fails its report upload with "No files found". + fail-fast: false matrix: include: - name: quickstart @@ -557,7 +569,10 @@ jobs: - name: Load CI container image (PR) if: github.event_name == 'pull_request' - run: docker load -i ci-container-image.tar + # rm after load: the multi-GB tarball otherwise stays on disk for the whole + # job — e2e runners have been observed within 1.5GB of the kubelet eviction + # threshold (run 28623232380). + run: docker load -i ci-container-image.tar && rm -f ci-container-image.tar - name: Login to Docker registry if: github.event_name != 'pull_request' @@ -576,7 +591,7 @@ jobs: # Make PROJECT_IMAGE available in the runner's Docker daemon so the k3d # image import (load delivery) finds it locally instead of pulling. if: github.event_name == 'pull_request' - run: docker load -i project-image.tar + run: docker load -i project-image.tar && rm -f project-image.tar - name: Start disk usage sampler run: | diff --git a/hack/e2e/deploy-controller.sh b/hack/e2e/deploy-controller.sh index dc7a991b..500b8130 100755 --- a/hack/e2e/deploy-controller.sh +++ b/hack/e2e/deploy-controller.sh @@ -75,5 +75,21 @@ else kubectl --context "${CTX}" -n "${NAMESPACE}" \ rollout restart "deployment/${deploy}" fi -kubectl --context "${CTX}" -n "${NAMESPACE}" \ - rollout status "deployment/${deploy}" --timeout=180s +# 300s: a fresh-namespace install waits on cert-manager issuing the admission +# cert before the pod can mount it, and CI runners near their disk-eviction +# threshold start pods slowly; 180s was too tight in the tail (CI run +# 28623232380). On timeout, dump the state that explains *why* the pod never +# became ready — node conditions expose DiskPressure, describe/events expose +# mount and scheduling problems — so a recurrence is diagnosable from the log. +if ! kubectl --context "${CTX}" -n "${NAMESPACE}" \ + rollout status "deployment/${deploy}" --timeout=300s; then + echo "=== rollout timed out; dumping diagnostics ===" >&2 + kubectl --context "${CTX}" get nodes \ + -o custom-columns='NAME:.metadata.name,CONDITIONS:.status.conditions[?(@.status=="True")].type' >&2 || true + kubectl --context "${CTX}" -n "${NAMESPACE}" get deploy,rs,pods -o wide >&2 || true + kubectl --context "${CTX}" -n "${NAMESPACE}" describe pods \ + -l "${CONTROLLER_DEPLOY_SELECTOR}" >&2 || true + kubectl --context "${CTX}" -n "${NAMESPACE}" get events \ + --sort-by=.metadata.creationTimestamp >&2 || true + exit 1 +fi From 6df58ab81e0b784c895584d0cbd90dcd6c02c12d Mon Sep 17 00:00:00 2001 From: Simon Koudijs Date: Thu, 2 Jul 2026 22:32:50 +0000 Subject: [PATCH 2/2] fix(e2e): stop k3d DiskPressure evictions with absolute kubelet thresholds The diagnostics added in the previous commit named the cause directly: the k3d node reported DiskPressure and evicted/rejected the controller pod, while the runner still had 3.3GB free. k3s defaults kubelet eviction to nodefs.available<5% - ~3.6GB of a 72GB CI disk - and the pressure condition then lingers for the default 5m transition period, longer than the rollout wait. - start-cluster.sh: absolute eviction thresholds (1Gi) and a 30s pressure transition period on all k3d nodes. Percentages don't express "actually almost full" when the node shares the host disk. (Invalidates _cluster-ready: next local e2e run rebuilds the cluster.) - ci.yml e2e job: free ~18GB of unused preinstalled toolchains (android, CodeQL, dotnet) before downloading artifacts. Co-Authored-By: Claude Fable 5 --- .github/workflows/ci.yml | 13 +++++++++++++ test/e2e/cluster/start-cluster.sh | 11 +++++++++++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03eb6466..a71bd8cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -553,6 +553,19 @@ jobs: with: persist-credentials: false + - name: Free runner disk + # The e2e stack (CI container, project image, k3d node volumes, local + # rebuilds) needs tens of GB; runners have been observed dipping below + # the k3d node's kubelet eviction threshold mid-run (28624646437). + # Drop the biggest preinstalled toolchains we never use. + run: | + df -h / | tail -1 + sudo rm -rf /usr/local/lib/android /opt/hostedtoolcache/CodeQL /usr/share/dotnet & + RM_PID=$! + docker system prune -f >/dev/null 2>&1 || true + wait "${RM_PID}" + df -h / | tail -1 + - name: Download release bundle artifact if: matrix.needs_artifact uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 diff --git a/test/e2e/cluster/start-cluster.sh b/test/e2e/cluster/start-cluster.sh index 714fbe32..75784090 100644 --- a/test/e2e/cluster/start-cluster.sh +++ b/test/e2e/cluster/start-cluster.sh @@ -215,6 +215,17 @@ create_cluster() { "--kube-apiserver-arg=audit-webhook-batch-max-size=10@server:0" "--kube-apiserver-arg=max-requests-inflight=${KUBE_APISERVER_MAX_REQUESTS_INFLIGHT}@server:0" "--kube-apiserver-arg=max-mutating-requests-inflight=${KUBE_APISERVER_MAX_MUTATING_REQUESTS_INFLIGHT}@server:0" + # k3s defaults kubelet eviction to nodefs.available<5% — on a CI runner's + # ~72GB disk that is ~3.6GB, so the node reports DiskPressure and evicts + # the controller while gigabytes are still free (CI run 28624646437). The + # k3d node shares the host filesystem, so percentage thresholds don't + # express "actually almost full"; use absolute ones, and drop the + # pressure condition quickly (default transition period is 5m) so a + # recovered node schedules pods again within the rollout wait. + "--kubelet-arg=eviction-hard=nodefs.available<1Gi,imagefs.available<1Gi@server:*" + "--kubelet-arg=eviction-hard=nodefs.available<1Gi,imagefs.available<1Gi@agent:*" + "--kubelet-arg=eviction-pressure-transition-period=30s@server:*" + "--kubelet-arg=eviction-pressure-transition-period=30s@agent:*" ) echo "🚀 Creating k3d cluster '${CLUSTER_NAME}' with audit webhook support..."