From c7135b3a68fe9403209dc01b693300b3e2df2c30 Mon Sep 17 00:00:00 2001 From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:27:25 +0200 Subject: [PATCH 1/5] Unblock dnceng.pkgs.visualstudio.com feed and fail fast on env build failures Run 26877237491 failed: the AWF firewall blocked all 432 requests to dnceng.pkgs.visualstudio.com (the legacy AzDO host backing the dotnet8/9/10 NuGet feeds in NuGet.config), so the whole-repo restore/build could not complete. The 'defaults'/'dotnet' allowlists only cover pkgs.dev.azure.com. Add dnceng.pkgs.visualstudio.com to network.allowed so restore can reach the runtime-pack feeds. The agent also burned the entire 25M effective-token budget (22 min) looping on the failing build before being hard-railed, so it never reached the quarantine fallback or opened a PR. Harden Step 6 to fail fast: a single environmental/network build failure must go straight to the Step 7d quarantine fallback instead of retrying or attempting a fix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../flaky-test-detector.agent.lock.yml | 38 +++++++++---------- .../workflows/flaky-test-detector.agent.md | 13 +++++++ 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/.github/workflows/flaky-test-detector.agent.lock.yml b/.github/workflows/flaky-test-detector.agent.lock.yml index cce3e6358cc..c795322b449 100644 --- a/.github/workflows/flaky-test-detector.agent.lock.yml +++ b/.github/workflows/flaky-test-detector.agent.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cedcae8581b055361ad8bf932965d25d2dece6d9c283b744ef4e43a91e075c6f","body_hash":"44a994da419345b70fa38055b1e00c7c810cfbaa387d8a7e9eeb11f4f095a2aa","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"afc07f764ec14a3a7211551e609bcb280b047192e40e6b334c77c024f4ae90ef","body_hash":"13df646679f9c6aed645390b79c64f1f19ad693fbc4c71c86da011883137a58e","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]} # ___ _ _ # / _ \ | | (_) @@ -112,7 +112,7 @@ jobs: GH_AW_INFO_EXPERIMENTAL: "false" GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" GH_AW_INFO_STAGED: "false" - GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","dotnet","dev.azure.com"]' + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","dotnet","dev.azure.com","dnceng.pkgs.visualstudio.com"]' GH_AW_INFO_FIREWALL_ENABLED: "true" GH_AW_INFO_AWF_VERSION: "v0.25.58" GH_AW_INFO_AWMG_VERSION: "" @@ -190,23 +190,23 @@ jobs: run: | bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh" { - cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF' + cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF' - GH_AW_PROMPT_7d78f9d1e18171e7_EOF + GH_AW_PROMPT_a6ee41c0302209cd_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md" cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md" cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md" cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md" - cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF' + cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF' Tools: add_comment(max:12), create_issue(max:5), create_pull_request, missing_tool, missing_data, noop - GH_AW_PROMPT_7d78f9d1e18171e7_EOF + GH_AW_PROMPT_a6ee41c0302209cd_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md" - cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF' + cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF' - GH_AW_PROMPT_7d78f9d1e18171e7_EOF + GH_AW_PROMPT_a6ee41c0302209cd_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md" - cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF' + cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF' The following GitHub context information is available for this workflow: {{#if github.actor}} @@ -235,12 +235,12 @@ jobs: {{/if}} - GH_AW_PROMPT_7d78f9d1e18171e7_EOF + GH_AW_PROMPT_a6ee41c0302209cd_EOF cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md" - cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF' + cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF' {{#runtime-import .github/workflows/flaky-test-detector.agent.md}} - GH_AW_PROMPT_7d78f9d1e18171e7_EOF + GH_AW_PROMPT_a6ee41c0302209cd_EOF } > "$GH_AW_PROMPT" - name: Interpolate variables and render templates uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 @@ -452,9 +452,9 @@ jobs: mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs" mkdir -p /tmp/gh-aw/safeoutputs mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs - cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_33cb5dba8cdacd38_EOF' + cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_561d63edf1e77524_EOF' {"add_comment":{"max":12,"target":"*"},"create_issue":{"labels":["flaky-test"],"max":5,"title_prefix":"[Flaky Test] "},"create_pull_request":{"allowed_files":["src/**/*.cs"],"base_branch":"main","draft":true,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":1,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}} - GH_AW_SAFE_OUTPUTS_CONFIG_33cb5dba8cdacd38_EOF + GH_AW_SAFE_OUTPUTS_CONFIG_561d63edf1e77524_EOF - name: Generate Safe Outputs Tools env: GH_AW_TOOLS_META_JSON: | @@ -724,7 +724,7 @@ jobs: mkdir -p /home/runner/.copilot GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node) - cat << GH_AW_MCP_CONFIG_1d050a98c9c44ad9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" + cat << GH_AW_MCP_CONFIG_c4f87d21fe3af2c3_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs" { "mcpServers": { "safeoutputs": { @@ -749,7 +749,7 @@ jobs: "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" } } - GH_AW_MCP_CONFIG_1d050a98c9c44ad9_EOF + GH_AW_MCP_CONFIG_c4f87d21fe3af2c3_EOF - name: Mount MCP servers as CLIs id: mount-mcp-clis continue-on-error: true @@ -791,7 +791,7 @@ jobs: export GH_AW_NODE_BIN export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK" (umask 177 && touch /tmp/gh-aw/agent-stdio.log) - printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.58/awf-config.schema.json","network":{"allowDomains":["*.vsblob.vsassets.io","api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.nuget.org","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","azuresearch-usnc.nuget.org","azuresearch-ussc.nuget.org","builds.dotnet.microsoft.com","ci.dot.net","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","dc.services.visualstudio.com","dev.azure.com","dist.nuget.org","dot.net","dotnet.microsoft.com","dotnetcli.blob.core.windows.net","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","nuget.org","nuget.pkg.github.com","nugetregistryv2prod.blob.core.windows.net","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","oneocsp.microsoft.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","pkgs.dev.azure.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com","www.microsoft.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.58"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" + printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.58/awf-config.schema.json","network":{"allowDomains":["*.vsblob.vsassets.io","api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.nuget.org","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","azuresearch-usnc.nuget.org","azuresearch-ussc.nuget.org","builds.dotnet.microsoft.com","ci.dot.net","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","dc.services.visualstudio.com","dev.azure.com","dist.nuget.org","dnceng.pkgs.visualstudio.com","dot.net","dotnet.microsoft.com","dotnetcli.blob.core.windows.net","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","nuget.org","nuget.pkg.github.com","nugetregistryv2prod.blob.core.windows.net","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","oneocsp.microsoft.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","pkgs.dev.azure.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com","www.microsoft.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.58"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json" GH_AW_MODEL_MULTIPLIERS_PATH="/tmp/gh-aw/model_multipliers.json" node "${RUNNER_TEMP}/gh-aw/actions/merge_awf_model_multipliers.cjs" cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS="" @@ -903,7 +903,7 @@ jobs: uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }} - GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" + GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} with: @@ -1501,7 +1501,7 @@ jobs: env: GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }} GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }} - GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" + GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com" GITHUB_SERVER_URL: ${{ github.server_url }} GITHUB_API_URL: ${{ github.api_url }} GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":12,\"target\":\"*\"},\"create_issue\":{\"labels\":[\"flaky-test\"],\"max\":5,\"title_prefix\":\"[Flaky Test] \"},\"create_pull_request\":{\"allowed_files\":[\"src/**/*.cs\"],\"base_branch\":\"main\",\"draft\":true,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}" diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md index 11c9878ba76..011ccbcb80c 100644 --- a/.github/workflows/flaky-test-detector.agent.md +++ b/.github/workflows/flaky-test-detector.agent.md @@ -21,6 +21,10 @@ network: - defaults - dotnet - dev.azure.com + # Legacy AzDO package host used by the dotnet8/dotnet9/dotnet10 feeds in NuGet.config. + # Distinct from pkgs.dev.azure.com (covered by `dotnet`); without it, NuGet restore of the + # runtime packs is blocked by the firewall and the whole-repo build fails. + - dnceng.pkgs.visualstudio.com tools: edit: @@ -294,6 +298,15 @@ Build the whole repo a single time up front so every subsequent reproduction/val ./build.sh ``` +**Fail fast on an environmental build failure — do not loop.** If this first `./build.sh` fails for +**environmental/network reasons** (NuGet restore cannot reach a feed, a blocked domain, SDK download +failure — *not* a compile or test error), **do not retry the build, do not investigate, and do not +attempt any determinism code fix.** Re-running the build against a blocked feed burns the entire token +budget on NuGet retries and huge logs, and the run gets cut off before it can open a PR. Instead, go +**straight to the quarantine fallback (Step 7d) for every selected test**, note in the PR that local +reproduction was blocked by the environment, and finish the run. One failed build attempt is enough to +make this determination — a missing/unreachable feed will not fix itself on a second attempt. + The whole-repo build takes ~2-3 minutes — **never cancel it**. `./build.sh` also builds every test project, so each test's runnable assembly already exists under `artifacts/bin//Debug//.dll` for the reproduction loops below. Network From 05345f3535b334559eaf1a3fa2d45834a47cb1df Mon Sep 17 00:00:00 2001 From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:42:17 +0200 Subject: [PATCH 2/5] Defer quarantine to next run and harden issue dedup Analysis of run 26877237491 showed the agent created 4 tracking issues via create_issue, then exhausted the 25M effective-token budget trying to discover their issue numbers to write [ActiveIssue(.../issues/NNNN)] attributes in the same run. That is impossible: create_issue is a safe output filed by a post-run job and returns no number during the run. - Step 5: a flake is quarantine-eligible only if its tracking issue was already OPEN before this run (real number readable now). A just-created issue does not count and must not be quarantined this run; it becomes eligible next run. Explicitly forbid polling/guessing for a just-created issue's number (the behavior that exhausted the budget). - Step 4: note create_issue returns no number; stress the dedup marker must be copied exactly as (the agent dropped the '-id', which would orphan the issue from future marker searches); add a title-search fallback for legacy issues lacking the marker; note the benign 'Malformed version:' gh stderr warning so it isn't misread as a command failure. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/flaky-test-detector.agent.md | 30 +++++++++++++++---- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md index 011ccbcb80c..f6dd4a26a99 100644 --- a/.github/workflows/flaky-test-detector.agent.md +++ b/.github/workflows/flaky-test-detector.agent.md @@ -196,6 +196,13 @@ recently-closed), e.g. via the `github` tools or: gh issue list --repo dotnet/msbuild --state all --search '"" in:body' --json number,state,title ``` +Older tracking issues may **predate the marker convention** and have none, so if the marker search +finds nothing, also fall back to a **title search** for the test's short name before concluding no +issue exists (e.g. `gh issue list --repo dotnet/msbuild --state all --search ' in:title'`) +— this avoids re-filing a duplicate of a pre-existing issue. (Note: the sandboxed `gh` may print a +benign `Malformed version:` warning to stderr; it is harmless — judge success by the JSON on stdout +and the exit code, not by that line.) + - **If a related issue is OPEN:** post an `add_comment` to that issue number with the **new** evidence (latest sources, build URLs, dates, legs/TFMs). Do not open a duplicate. - **If a related issue exists but is CLOSED recently** (e.g. within ~30 days): do **not** open a @@ -216,10 +223,15 @@ gh issue list --repo dotnet/msbuild --state all --search '"` without `-id` will not be found by future runs and + causes duplicate issues): ``` ``` @@ -238,9 +250,17 @@ Build the set of tests to fix or quarantine in today's combined PR. A candidate of these hold: - It was classified as a **likely flake** in Step 3 (never act on a regression). -- It has an **OPEN** `flaky-test` tracking issue (the one created or updated in Step 4). Do **not** use a - recently-*closed* issue as the quarantine target — an `[ActiveIssue]` URL must point at an open issue. - If only a closed issue exists, skip the test from today's PR (Step 4 already commented on it). +- It has a `flaky-test` tracking issue that was **already OPEN before this run started** (a + pre-existing issue found in Step 4, with a real issue number you can read **now**). A brand-new + issue you filed via `create_issue` **this run does not count** — safe-output issues are created by a + post-run job and **return no number during the agent run**, so you cannot reference one in an + `[ActiveIssue(".../issues/")]` attribute yet. **Do not** attempt to discover the number of an + issue you just created (it does not exist yet — polling `gh issue list` / `gh api` / guessing ranges + for it is wasted effort that can exhaust the run's token budget). Such a test was tracked this run + and becomes quarantine-eligible on the **next** run, once its issue is open with a real number. Do + **not** use a recently-*closed* issue as the quarantine target either — an `[ActiveIssue]` URL must + point at an issue that is open and already has a number. If only a just-created or closed issue + exists, skip the test from today's PR (Step 4 already filed/commented). - It is **not already covered by an open `flaky-test` PR** (this is the primary cross-run dedup — a previous run's combined PR may still be open and unmerged, and its quarantines live on **that PR's branch, not `main`**, so they will not show up in your fresh `main` checkout). Fetch the bodies of all From 7e1c946ec435f093dd2f7a407486f41c0a9d59ed Mon Sep 17 00:00:00 2001 From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:44:40 +0200 Subject: [PATCH 3/5] Document two-run quarantine cadence and exact dedup marker in skill Mirror the workflow behavioral fixes in the skill doc: explain that create_issue is a deferred safe output returning no number during the run, so a new flake is filed one run and quarantined the next (never poll for a just-created issue's number); and stress the dedup marker must be copied exactly with the -id segment, with a title-search fallback for legacy issues. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/flaky-test-detector/SKILL.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md index 5c0a465f17f..f452a8cf971 100644 --- a/.github/skills/flaky-test-detector/SKILL.md +++ b/.github/skills/flaky-test-detector/SKILL.md @@ -207,7 +207,16 @@ The detector surfaces evidence; it does **not** by itself prove flakiness. Befor ```html ``` - Search both open and recently-closed issues for this marker before filing a new one. + Copy the marker **exactly**, including the `-id` segment — `` (no `-id`) + will not be found by later searches and orphans the issue, causing duplicates. Search both open + and recently-closed issues for this marker before filing a new one; if the marker search finds + nothing, fall back to a title search to catch legacy issues created before the marker convention. +- **Two-run quarantine cadence (file first, quarantine next run):** quarantining a test requires the + tracking issue's **number** for the `[ActiveIssue(.../issues/NNNN)]` URL, but the daily workflow + files issues with the gh-aw `create_issue` safe output, which is deferred to a **post-run** job and + returns **no number during the run**. So a brand-new flake is only filed this run and becomes + **quarantine-eligible the next run**, once its issue exists and is open with a readable number. Never + poll, guess, or wait for the number of an issue created in the current run. - **Quarantine syntax (use exactly):** quarantine with `[ActiveIssue]` from `Microsoft.DotNet.XUnitV3Extensions` (namespace `Xunit`, already referenced and imported by every test project) — **not** `[Fact(Skip=...)]`. Add it above the existing `[Fact]`/`[Theory]`, keeping From 27a411d42e3aa7ba2f8dc9918811339e835ef4cd Mon Sep 17 00:00:00 2001 From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:59:28 +0200 Subject: [PATCH 4/5] Tell the agent to run the detector synchronously Run 26877237491 ran the detector twice with identical inputs: it was launched backgrounded with heavy -MaxBuilds/-MaxArtifactDownloads, polled after ~65s while the scan was still running, saw no -JsonOut file yet (the script writes JSON only on completion), misread that as a failure, and relaunched -- re-downloading every AzDO artifact. Instruct the agent to run the detector in the foreground and wait for exit, and note that a missing -JsonOut file mid-run means 'not finished', not 'failed'. Mirror a short note in the skill doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/flaky-test-detector/SKILL.md | 5 +++++ .github/workflows/flaky-test-detector.agent.md | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md index f452a8cf971..c023a38438b 100644 --- a/.github/skills/flaky-test-detector/SKILL.md +++ b/.github/skills/flaky-test-detector/SKILL.md @@ -85,6 +85,11 @@ The script writes a **human-readable report to stderr** and the **structured JSO stdout** (and `-JsonOut` if given). `gh` must be installed and authenticated for the approval filter and existing-issue cross-reference to work. +> **Run it synchronously.** At high `-MaxBuilds`/`-MaxArtifactDownloads` the scan can take several +> minutes and writes stdout / `-JsonOut` **only on completion**. Wait for the process to exit before +> reading the JSON — don't background it or treat a not-yet-written `-JsonOut` file as a failure (it +> just means the scan is still running). Re-running wastefully re-downloads every artifact. + ### JSON report shape ```jsonc diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md index f6dd4a26a99..76f331df51e 100644 --- a/.github/workflows/flaky-test-detector.agent.md +++ b/.github/workflows/flaky-test-detector.agent.md @@ -116,6 +116,14 @@ raised in step so the larger build set does not re-trip the artifact-download ca The script writes a human-readable progress report to the log/host stream and the structured JSON report to stdout (also written to `flaky-report.json`). Parse the JSON. +**Run it synchronously and wait for it to exit.** At these `-MaxBuilds`/`-MaxArtifactDownloads` +values the scan downloads and parses many artifacts and can take **several minutes**. It writes the +JSON to stdout and to `-JsonOut` **only on completion** — there is no partial file mid-run. Do **not** +background it (no `&`) and do **not** poll-then-bail: a missing `-JsonOut` file or empty stdout while +the process is still running means *not finished yet*, **not** failure. Read the JSON only after the +process has exited. Re-running because the file "wasn't there yet" just re-downloads every artifact and +wastes the run's time and token budget. The same applies to the Step 1b quarantine scan below. + ## Step 1b — Scan the quarantine pipeline (backlog signal) Also scan the **quarantine pipeline** (AzDO definition **344**, `azure-pipelines/quarantine.yml`), From 05f0e35f9c7a317c059076ff76cabb364a0e86dc Mon Sep 17 00:00:00 2001 From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com> Date: Wed, 3 Jun 2026 13:05:46 +0200 Subject: [PATCH 5/5] Merge flaky-test-detector skill into the workflow The flaky-test-detector SKILL.md was a separate discoverable skill whose content the daily workflow already re-stated in its own steps (thresholds, dedup marker, quarantine/un-quarantine conventions, flake-vs-regression classification, assembly->project mapping). Its only unique content was the evidence-source model and a gloss of the detector JSON fields. Fold that essential background into a new 'Background' section of flaky-test-detector.agent.md so the workflow is self-contained, move the detector script to .github/workflows/scripts/Get-FlakyTests.ps1 next to the workflow that calls it (a scripts/ dir with no SKILL.md would be a malformed skill), update the two invocation paths and the assembly-mapping reference, and delete SKILL.md. No other workflow or pipeline referenced the skill or script; the workflow body is runtime-imported so no lock recompile is needed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/skills/flaky-test-detector/SKILL.md | 262 ------------------ .../workflows/flaky-test-detector.agent.md | 35 ++- .../scripts/Get-FlakyTests.ps1 | 0 3 files changed, 28 insertions(+), 269 deletions(-) delete mode 100644 .github/skills/flaky-test-detector/SKILL.md rename .github/{skills/flaky-test-detector => workflows}/scripts/Get-FlakyTests.ps1 (100%) diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md deleted file mode 100644 index c023a38438b..00000000000 --- a/.github/skills/flaky-test-detector/SKILL.md +++ /dev/null @@ -1,262 +0,0 @@ ---- -name: flaky-test-detector -description: Detect flaky tests in dotnet/msbuild by scanning recent Azure DevOps CI builds (approved PR validation + rolling builds on main) for test failures that recur across multiple independent evidence sources. Use when asked to find flaky tests, investigate intermittent CI failures, triage the flaky-test label, or produce a flaky-test report. ---- - -# Flaky Test Detector - -This skill identifies **flaky tests** in dotnet/msbuild: tests that fail intermittently rather -than because of a real product regression. It works entirely against **anonymously accessible** -Azure DevOps APIs on the public CI (`dnceng-public` / `public`, pipeline `msbuild-pr`, -definition **75**), so it needs no Azure DevOps credentials. - -## When to Use - -- User asks to find or triage flaky tests, or to investigate intermittent CI failures. -- Periodic (scheduled) flaky-test sweeps. -- Before deciding whether to quarantine a test or open/refresh a `flaky-test` tracking issue. -- To gather evidence (which PRs / rolling builds, which legs/TFMs, error signatures) for a - specific suspected-flaky test. - -## Core Idea: Evidence Sources - -A test is flagged as flaky when it fails across several **independent evidence sources**. A -source is one of: - -1. **A Pull Request** — all failed PR-validation builds for one PR collapse into a single - source. Only **non-draft, approved (or merged) PRs targeting `main`** count, because a - failure in code reviewers signed off on is unlikely to be caused by that PR's own changes. -2. **A single failed rolling/CI build on `main`** (reason `individualCI` / `batchedCI` / - `schedule`). `main` is expected to be green, so a test failing there is strong, independent - evidence of flakiness (or a regression — see below). - -A test failing across many *unrelated* approved PRs **and/or** multiple rolling `main` builds -cannot be explained by any one change, which is the signature of flakiness. - -> **Scope (current):** `main` branch only. Builds on `vs18.x` / `exp/*` branches are ignored. - -## Data Path (all anonymous on dnceng-public) - -The Azure DevOps **Test Management API is NOT anonymously accessible**, so this skill uses -published pipeline **artifacts** instead: - -1. **Build list** — `/_apis/build/builds?definitions=75&resultFilter=failed&minTime=...` returns - failed builds with `reason`, `sourceBranch`, and `triggerInfo['pr.number']`. -2. **Timeline** — `/_apis/build/builds/{id}/timeline` exposes which *legs* (jobs) failed, used - to download only the relevant artifacts. -3. **Artifacts list** — `/_apis/build/builds/{id}/artifacts` lists per-leg containers named - `" test logs"` and `" build logs"`. -4. **Artifact zip** — `...artifacts?artifactName=&$format=zip` downloads the container. - - `" test logs"` zips contain the **`.trx`** result files (failed test names + error - message/stack trace). - - `" build logs"` zips (misleadingly named) contain the **full xUnit console `.log`** - output and a `.binlog`. The scan does **not** download these; the fix phase pulls them on - demand when it needs stdout for diagnosis. -5. **TRX parsing** — failed tests come from `//UnitTestResult[@outcome='Failed']/@testName`. The - `@testName` is fully-qualified and may include a parameterized suffix, e.g. - `Namespace.Class.Method(runtimeToUse: "NET", ...)`. The detector stores both the raw name and - a normalized name (parameter suffix stripped). The assembly/TFM/arch are encoded in the TRX - file name, e.g. `Microsoft.Build.Engine.UnitTests_net472_x86.trx`. - -> **TRX has no stdout.** The `` element only contains `` (message + stack -> trace). For console output, use the `"... build logs"` artifacts. - -## Running the Detector - -```pwsh -pwsh -File .github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 -``` - -Common options: - -| Parameter | Default | Purpose | -|-----------|---------|---------| -| `-MinSources` | `3` | Distinct sources (PRs + rolling builds) required to flag a test. | -| `-DaysBack` | `14` | Look-back window. | -| `-MaxBuilds` | `60` | Max failed builds (all reasons) to fetch. | -| `-MaxArtifactDownloads` | `150` | Hard cap; tripping it sets `scanComplete: false`. | -| `-TargetBranch` | `main` | Branch to scope to (PR base + rolling source branch). | -| `-NoApprovalFilter` | _off_ | Bypass the non-draft/approved PR filter (smoke testing only). | -| `-AllLegs` | _off_ | Download every "test logs" artifact, not just failed legs. | -| `-IncludePassed` | _off_ | Also record **passing** observations and emit a `passedTests` aggregate. Forces `-AllLegs` and queries **all completed** builds (not just failed). Used against the quarantine pipeline (def 344) to drive un-quarantining. | -| `-JsonOut ` | — | Also write the structured JSON report to a file. | - -The script writes a **human-readable report to stderr** and the **structured JSON report to -stdout** (and `-JsonOut` if given). `gh` must be installed and authenticated for the approval -filter and existing-issue cross-reference to work. - -> **Run it synchronously.** At high `-MaxBuilds`/`-MaxArtifactDownloads` the scan can take several -> minutes and writes stdout / `-JsonOut` **only on completion**. Wait for the process to exit before -> reading the JSON — don't background it or treat a not-yet-written `-JsonOut` file as a failure (it -> just means the scan is still running). Re-running wastefully re-downloads every artifact. - -### JSON report shape - -```jsonc -{ - "scanComplete": true, // false => truncated; DO NOT act on the data - "targetBranch": "main", - "minSources": 3, - "buildsScanned": 18, - "prSources": 9, - "rollingSources": 4, - "flakyTests": [ - { - "testName": "Namespace.Class.Method", // normalized (no param suffix) - "distinctSources": 4, - "distinctPRs": 3, - "prNumbers": [13458, 13501, 13620], - "rollingBuildIds": [1443528], - "totalFailures": 7, - "legs": ["CoreOnLinux", "FullOnWindows Release"], - "tfms": ["net472", "net10.0"], - "assemblies": ["Microsoft.Build.Engine.UnitTests"], - "rawVariants": ["...Method(runtimeToUse: \"NET\")"], - "errorHashes": ["a1b2c3d4"], // short SHA-256 of each error message - "firstSeen": "2026-05-20", - "lastSeen": "2026-05-30", - "sampleBuildUrl": "https://dev.azure.com/.../_build/results?buildId=...", - "sampleError": "Assert.Equal() Failure ...", - "relatedIssues": [ { "number": 1234, "title": "...", "state": "OPEN" } ] - } - ], - "passedTests": [ // ONLY populated with -IncludePassed (e.g. def 344) - { - "testName": "Namespace.Class.Method", // normalized (no param suffix) - "totalPassed": 11, // total passing UnitTestResults observed in the window - "distinctBuilds": 6, // distinct quarantine builds it was seen green in - "buildIds": [1450001, 1450200], - "distinctDays": 4, // distinct calendar days with a green observation - "legs": ["CoreOnWindows", "CoreOnLinux", "CoreOnMacOS"], - "tfms": ["net472", "net10.0"], - "assemblies": ["Microsoft.Build.Engine.UnitTests"], - "firstSeen": "2026-05-20", - "lastSeen": "2026-05-30" - } - ] -} -``` - -## Quarantine-Pipeline Health (definition 344) - -The same detector, run with `-DefinitionId 344 -IncludePassed`, reads the scheduled **quarantine -pipeline** (`azure-pipelines/quarantine.yml`), which re-runs **only** the quarantined -(`[ActiveIssue]` / `Category=failing`) tests on Windows/Linux/macOS twice daily. This is the signal -for **clearing the quarantine backlog**: - -```pwsh -pwsh -File .github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 ` - -DefinitionId 344 -MinSources 2 -IncludePassed -DaysBack 21 -JsonOut quarantine-health.json -``` - -- Because most quarantine builds finish **green** (they succeed unless a quarantined test fails), - `-IncludePassed` queries **all completed** builds (`statusFilter=completed`, not `resultFilter=failed`) - and forces `-AllLegs` so green legs are scanned too. -- Each quarantine build runs each quarantined test **once per leg**, so a pass/fail **rate** only emerges - by **aggregating across many builds** over the window (twice-daily × N days). `distinctBuilds` / - `distinctDays` in `passedTests` are how you gauge that. -- In this mode: `flakyTests` = quarantined tests **still flaking** in 344; `passedTests` = quarantined - tests **observed passing** there. A test in **both** is still flaky. - -### Un-quarantine criteria (avoiding false un-quarantines) - -A "green" reading must mean the test **ran and passed**, not merely "was absent from failures" (it could -be absent because a leg did not run or an artifact was missing). Only un-quarantine a test when **all** -hold: - -- `scanComplete == true` for the 344 scan (a truncated scan is biased — do nothing). -- the test is in `passedTests` with `distinctBuilds >= 4` **and** `distinctDays >= 3`, **and** -- the test is **not** in the 344 `flakyTests` (zero failures over the window), **and** -- the green evidence covers the platform scope the `[ActiveIssue]` applies to: an **unconditional** - quarantine needs green legs on **Windows, Linux, and macOS**; a **platform-scoped** one needs green on - the matching leg(s). When green on only some platforms, **narrow** the attribute rather than removing - it — never fully un-quarantine on partial evidence. - -Un-quarantining does **not** require local reproduction: def 344 is real CI across many builds/days and -is stronger evidence than a single local pass on one OS. Removing the attribute = deleting the -`[ActiveIssue(...)]` line above the test. - -## Interpreting Results — Flake vs Regression - -The detector surfaces evidence; it does **not** by itself prove flakiness. Before acting: - -- **`scanComplete` must be `true`.** A truncated scan is biased — never file issues, edit sources, - or open a PR from it. Widen `-MaxBuilds` / `-MaxArtifactDownloads` and re-run. -- **Spread over time and sources is the signal.** Prefer tests whose failures span multiple days - and multiple sources over a burst within one source. -- **Rolling-only failures may be a real regression.** If a test only fails on consecutive rolling - `main` builds (no scattered PR evidence) and the failures are identical, treat it as a likely - **regression**, not a flake — that is a `noop` for quarantine; flag it for human attention. -- **Check `relatedIssues`** to avoid duplicate filing; an existing open issue should be updated, - not duplicated. -- **A recently-closed (fixed) issue is not automatically a recurrence.** The look-back window - (`-DaysBack`) routinely includes builds from *before* a fix merged. Before commenting "recurred" - on a closed/fixed issue, confirm at least one failure's build **start time** is strictly after the - fixing PR merged (use the closing PR's merge commit time, not just `closedAt`; prefer build start - times over the date-only `lastSeen`). If all evidence predates the fix, it is stale — take no - action on that test. - -## Tiered Thresholds (recommended) - -| Action | Suggested bar | -|--------|---------------| -| Mention in report | ≥ 2 distinct sources | -| File / update a `flaky-test` tracking issue | ≥ 3 distinct sources | -| Quarantine the test (`[ActiveIssue]`) | ≥ 5 distinct sources **or** failures spanning multiple days | - -## Conventions - -- **Existing label:** `flaky-test`. -- **Issue dedup marker:** every flaky-test tracking issue body must contain a hidden, stable - marker so future runs can find it: - ```html - - ``` - Copy the marker **exactly**, including the `-id` segment — `` (no `-id`) - will not be found by later searches and orphans the issue, causing duplicates. Search both open - and recently-closed issues for this marker before filing a new one; if the marker search finds - nothing, fall back to a title search to catch legacy issues created before the marker convention. -- **Two-run quarantine cadence (file first, quarantine next run):** quarantining a test requires the - tracking issue's **number** for the `[ActiveIssue(.../issues/NNNN)]` URL, but the daily workflow - files issues with the gh-aw `create_issue` safe output, which is deferred to a **post-run** job and - returns **no number during the run**. So a brand-new flake is only filed this run and becomes - **quarantine-eligible the next run**, once its issue exists and is open with a readable number. Never - poll, guess, or wait for the number of an issue created in the current run. -- **Quarantine syntax (use exactly):** quarantine with `[ActiveIssue]` from - `Microsoft.DotNet.XUnitV3Extensions` (namespace `Xunit`, already referenced and imported by every - test project) — **not** `[Fact(Skip=...)]`. Add it above the existing `[Fact]`/`[Theory]`, keeping - the test method intact: - ```csharp - [ActiveIssue("https://github.com/dotnet/msbuild/issues/NNNN")] - [Fact] - // platform-scoped — re-validated on the matching OS leg of the pipeline (see note below): - [ActiveIssue("https://github.com/dotnet/msbuild/issues/NNNN", TestPlatforms.Linux)] - ``` - The issue URL is the tracking-issue URL; do **not** add a `"Flaky:"` prefix. `[ActiveIssue]` stamps - the `Category=failing` trait, which normal CI excludes and the scheduled `azure-pipelines/quarantine.yml` - pipeline runs on its own to keep collecting signal on quarantined tests. **Prefer the unconditional - form** unless the flake is clearly platform-specific: that pipeline runs on **Windows, Linux and - macOS**, so a platform-scoped quarantine (`Windows`, `Linux`, `OSX`, `AnyUnix`) is still re-validated - on the matching leg. Only scope when the flake is confined to that platform. -- **Theory granularity:** quarantining a `[Theory]` disables *all* rows. Prefer method-level evidence - and note the failing parameter distribution (`rawVariants`) before quarantining a `[Theory]`. - -## Locating a Test's Source - -Map the failing test's **assembly** (from the TRX file name, e.g. -`Microsoft.Build.Engine.UnitTests`) to its test project under `src/` (e.g. -`src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.csproj`) rather than doing a repo-wide -text search. Then locate the class/method within that project. - -## Related Workflows - -- `.github/workflows/flaky-test-detector.agent.md` — the scheduled **daily** workflow that runs this - skill end to end: it scans CI, files/updates `flaky-test` tracking issues, then reproduces and either - applies a minimal determinism fix or quarantines each new candidate. It **also** consumes the - quarantine-pipeline health (def 344, via `-IncludePassed`) to **un-quarantine** consistently-green - tests and **re-fix** tests still flaking there, opening **one combined draft PR per run** with all of - that day's quarantines, fixes, and un-quarantines. -- `azure-pipelines/quarantine.yml` — scheduled (twice-daily) AzDO pipeline (definition **344**) that - runs **only** the quarantined (`[ActiveIssue]` / `Category=failing`) tests, so quarantined tests keep - producing pass/fail signal. A test that has gone consistently green there is a candidate to - un-quarantine (see **Quarantine-Pipeline Health** above for the criteria). diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md index 76f331df51e..5038eac4200 100644 --- a/.github/workflows/flaky-test-detector.agent.md +++ b/.github/workflows/flaky-test-detector.agent.md @@ -81,10 +81,30 @@ You are an automated maintenance agent for the **dotnet/msbuild** repository. Yo track them as GitHub issues, and, in a **single combined draft pull request per run**, either apply a minimal determinism fix or quarantine them so CI stops being disrupted. -Read the skill at `.github/skills/flaky-test-detector/SKILL.md` for full background on the data path, -the "evidence source" model (approved PRs + rolling `main` builds), the JSON schema, thresholds, the -assembly → test-project mapping, quarantine conventions, and the determinism-fix vs. quarantine -decision. Follow it. +## Background — evidence model and detector output + +A test is **flaky** when it fails across multiple **independent evidence sources**, where one source +is either **(a)** a single **approved, non-draft PR targeting `main`** — all of that PR's failed +validation builds collapse into one source, since a reviewer-approved PR is unlikely to be broken by +its own diff — or **(b)** a single **failed rolling/CI build on `main`** (`main` is expected green, so +each such failure is independent evidence). A test failing across many *unrelated* approved PRs and/or +multiple rolling builds cannot be explained by any one change — the signature of flakiness (vs. a +regression; see Step 3). **Scope: `main` only.** + +The detector script `.github/workflows/scripts/Get-FlakyTests.ps1` reaches the **anonymously +accessible** public Azure DevOps build APIs (`dnceng-public`/`public`, PR pipeline definition **75**; +quarantine pipeline definition **344** with `-DefinitionId 344`), downloads only the failed legs' +test-log artifacts, parses the `.trx` files, and emits one JSON report — the **only** source of truth +(never invent data). Per flagged test the JSON gives: normalized `testName` (`Namespace.Class.Method`, +parameter suffix stripped), `distinctSources`/`distinctPRs`/`prNumbers`/`rollingBuildIds`, +`totalFailures`, `legs`/`tfms`/`assemblies`, `errorHashes` (one short hash per distinct failure +signature), `rawVariants` (the parameterized `[Theory]` rows that failed), `firstSeen`/`lastSeen`, +`sampleBuildUrl`, `sampleError`, and `relatedIssues` (existing `flaky-test` issues). `scanComplete: +false` means the scan was truncated and is biased — **do not act on it** (Step 2). + +Map a test's `assemblies[0]` (from the TRX file name, e.g. `Microsoft.Build.Engine.UnitTests`) to its +project under `src/` (e.g. `src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.csproj`) rather than a +repo-wide text search, then locate the class/method within it. ## Overall shape @@ -103,7 +123,7 @@ decision. Follow it. Run the detector script via the `bash` tool: ```bash -pwsh -File .github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 -TargetBranch main -DaysBack 14 -MinSources 3 -MaxBuilds 200 -MaxArtifactDownloads 400 -JsonOut flaky-report.json +pwsh -File .github/workflows/scripts/Get-FlakyTests.ps1 -TargetBranch main -DaysBack 14 -MinSources 3 -MaxBuilds 200 -MaxArtifactDownloads 400 -JsonOut flaky-report.json ``` `-MaxBuilds` must stay comfortably above the number of failed builds the PR pipeline (definition 75) @@ -133,7 +153,7 @@ tests that have gone consistently green, and re-attempting fixes on tests still the **same detector** with `-IncludePassed`, which also records passing observations: ```bash -pwsh -File .github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 -DefinitionId 344 -TargetBranch main -DaysBack 21 -MinSources 2 -MaxBuilds 150 -MaxArtifactDownloads 400 -IncludePassed -JsonOut quarantine-health.json +pwsh -File .github/workflows/scripts/Get-FlakyTests.ps1 -DefinitionId 344 -TargetBranch main -DaysBack 21 -MinSources 2 -MaxBuilds 150 -MaxArtifactDownloads 400 -IncludePassed -JsonOut quarantine-health.json ``` This emits the usual JSON plus a `passedTests` array (per normalized test: `distinctBuilds`, @@ -360,7 +380,8 @@ as part of the fix. ### 7a — Locate tests and build their projects (sequential) -For each selected test, map `assemblies[0]` to its test project using the skill's convention (e.g. +For each selected test, map `assemblies[0]` to its test project using the assembly → project mapping +from the Background section (e.g. `Microsoft.Build.Engine.UnitTests` → `src/Build.UnitTests/Microsoft.Build.Engine.UnitTests.csproj`) and find the test method source. For each **distinct** candidate project, run one fast incremental build so the test assembly is current (no-op if `./build.sh` already built it): diff --git a/.github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 b/.github/workflows/scripts/Get-FlakyTests.ps1 similarity index 100% rename from .github/skills/flaky-test-detector/scripts/Get-FlakyTests.ps1 rename to .github/workflows/scripts/Get-FlakyTests.ps1