From c7135b3a68fe9403209dc01b693300b3e2df2c30 Mon Sep 17 00:00:00 2001
From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com>
Date: Wed, 3 Jun 2026 12:27:25 +0200
Subject: [PATCH 1/5] Unblock dnceng.pkgs.visualstudio.com feed and fail fast
on env build failures
Run 26877237491 failed: the AWF firewall blocked all 432 requests to
dnceng.pkgs.visualstudio.com (the legacy AzDO host backing the dotnet8/9/10
NuGet feeds in NuGet.config), so the whole-repo restore/build could not
complete. The 'defaults'/'dotnet' allowlists only cover pkgs.dev.azure.com.
Add dnceng.pkgs.visualstudio.com to network.allowed so restore can reach the
runtime-pack feeds.
The agent also burned the entire 25M effective-token budget (22 min) looping
on the failing build before being hard-railed, so it never reached the
quarantine fallback or opened a PR. Harden Step 6 to fail fast: a single
environmental/network build failure must go straight to the Step 7d quarantine
fallback instead of retrying or attempting a fix.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
.../flaky-test-detector.agent.lock.yml | 38 +++++++++----------
.../workflows/flaky-test-detector.agent.md | 13 +++++++
2 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/.github/workflows/flaky-test-detector.agent.lock.yml b/.github/workflows/flaky-test-detector.agent.lock.yml
index cce3e6358cc..c795322b449 100644
--- a/.github/workflows/flaky-test-detector.agent.lock.yml
+++ b/.github/workflows/flaky-test-detector.agent.lock.yml
@@ -1,4 +1,4 @@
-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"cedcae8581b055361ad8bf932965d25d2dece6d9c283b744ef4e43a91e075c6f","body_hash":"44a994da419345b70fa38055b1e00c7c810cfbaa387d8a7e9eeb11f4f095a2aa","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"}
+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"afc07f764ec14a3a7211551e609bcb280b047192e40e6b334c77c024f4ae90ef","body_hash":"13df646679f9c6aed645390b79c64f1f19ad693fbc4c71c86da011883137a58e","compiler_version":"v0.77.5","strict":true,"agent_id":"copilot"}
# gh-aw-manifest: {"version":1,"secrets":["COPILOT_GITHUB_TOKEN","GH_AW_CI_TRIGGER_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/checkout","sha":"de0fac2e4500dabe0009e67214ff5f5447ce83dd","version":"v6.0.2"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e","version":"v6.4.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"3ea13c02d765410340d533515cb31a7eef2baaf0","version":"v0.77.5"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/cli-proxy:0.25.58"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.25.58"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.3.22"},{"image":"ghcr.io/github/github-mcp-server:v1.1.0"},{"image":"node:lts-alpine","digest":"sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14","pinned_image":"node:lts-alpine@sha256:2bdb65ed1dab192432bc31c95f94155ca5ad7fc1392fb7eb7526ab682fa5bf14"}]}
# ___ _ _
# / _ \ | | (_)
@@ -112,7 +112,7 @@ jobs:
GH_AW_INFO_EXPERIMENTAL: "false"
GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true"
GH_AW_INFO_STAGED: "false"
- GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","dotnet","dev.azure.com"]'
+ GH_AW_INFO_ALLOWED_DOMAINS: '["defaults","dotnet","dev.azure.com","dnceng.pkgs.visualstudio.com"]'
GH_AW_INFO_FIREWALL_ENABLED: "true"
GH_AW_INFO_AWF_VERSION: "v0.25.58"
GH_AW_INFO_AWMG_VERSION: ""
@@ -190,23 +190,23 @@ jobs:
run: |
bash "${RUNNER_TEMP}/gh-aw/actions/create_prompt_first.sh"
{
- cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF'
+ cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF'
- GH_AW_PROMPT_7d78f9d1e18171e7_EOF
+ GH_AW_PROMPT_a6ee41c0302209cd_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/xpia.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/temp_folder_prompt.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/markdown.md"
cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_prompt.md"
- cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF'
+ cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF'
Tools: add_comment(max:12), create_issue(max:5), create_pull_request, missing_tool, missing_data, noop
- GH_AW_PROMPT_7d78f9d1e18171e7_EOF
+ GH_AW_PROMPT_a6ee41c0302209cd_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/safe_outputs_create_pull_request.md"
- cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF'
+ cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF'
- GH_AW_PROMPT_7d78f9d1e18171e7_EOF
+ GH_AW_PROMPT_a6ee41c0302209cd_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/mcp_cli_tools_prompt.md"
- cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF'
+ cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF'
The following GitHub context information is available for this workflow:
{{#if github.actor}}
@@ -235,12 +235,12 @@ jobs:
{{/if}}
- GH_AW_PROMPT_7d78f9d1e18171e7_EOF
+ GH_AW_PROMPT_a6ee41c0302209cd_EOF
cat "${RUNNER_TEMP}/gh-aw/prompts/cli_proxy_with_safeoutputs_prompt.md"
- cat << 'GH_AW_PROMPT_7d78f9d1e18171e7_EOF'
+ cat << 'GH_AW_PROMPT_a6ee41c0302209cd_EOF'
{{#runtime-import .github/workflows/flaky-test-detector.agent.md}}
- GH_AW_PROMPT_7d78f9d1e18171e7_EOF
+ GH_AW_PROMPT_a6ee41c0302209cd_EOF
} > "$GH_AW_PROMPT"
- name: Interpolate variables and render templates
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
@@ -452,9 +452,9 @@ jobs:
mkdir -p "${RUNNER_TEMP}/gh-aw/safeoutputs"
mkdir -p /tmp/gh-aw/safeoutputs
mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs
- cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_33cb5dba8cdacd38_EOF'
+ cat > "${RUNNER_TEMP}/gh-aw/safeoutputs/config.json" << 'GH_AW_SAFE_OUTPUTS_CONFIG_561d63edf1e77524_EOF'
{"add_comment":{"max":12,"target":"*"},"create_issue":{"labels":["flaky-test"],"max":5,"title_prefix":"[Flaky Test] "},"create_pull_request":{"allowed_files":["src/**/*.cs"],"base_branch":"main","draft":true,"excluded_files":[".github/**"],"labels":["flaky-test"],"max":1,"max_patch_files":100,"max_patch_size":1024,"protect_top_level_dot_folders":true,"protected_files":["package.json","bun.lockb","bunfig.toml","deno.json","deno.jsonc","deno.lock","global.json","NuGet.Config","Directory.Packages.props","mix.exs","mix.lock","go.mod","go.sum","stack.yaml","stack.yaml.lock","pom.xml","build.gradle","build.gradle.kts","settings.gradle","settings.gradle.kts","gradle.properties","package-lock.json","yarn.lock","pnpm-lock.yaml","npm-shrinkwrap.json","requirements.txt","Pipfile","Pipfile.lock","pyproject.toml","setup.py","setup.cfg","Gemfile","Gemfile.lock","uv.lock","CODEOWNERS","DESIGN.md","README.md","CONTRIBUTING.md","CHANGELOG.md","SECURITY.md","CODE_OF_CONDUCT.md","AGENTS.md","CLAUDE.md","GEMINI.md"],"protected_files_policy":"request_review","title_prefix":"[Flaky Test] "},"create_report_incomplete_issue":{},"missing_data":{},"missing_tool":{},"noop":{"max":1,"report-as-issue":"false"},"report_incomplete":{}}
- GH_AW_SAFE_OUTPUTS_CONFIG_33cb5dba8cdacd38_EOF
+ GH_AW_SAFE_OUTPUTS_CONFIG_561d63edf1e77524_EOF
- name: Generate Safe Outputs Tools
env:
GH_AW_TOOLS_META_JSON: |
@@ -724,7 +724,7 @@ jobs:
mkdir -p /home/runner/.copilot
GH_AW_NODE=$(which node 2>/dev/null || command -v node 2>/dev/null || echo node)
- cat << GH_AW_MCP_CONFIG_1d050a98c9c44ad9_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
+ cat << GH_AW_MCP_CONFIG_c4f87d21fe3af2c3_EOF | "$GH_AW_NODE" "${RUNNER_TEMP}/gh-aw/actions/start_mcp_gateway.cjs"
{
"mcpServers": {
"safeoutputs": {
@@ -749,7 +749,7 @@ jobs:
"payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}"
}
}
- GH_AW_MCP_CONFIG_1d050a98c9c44ad9_EOF
+ GH_AW_MCP_CONFIG_c4f87d21fe3af2c3_EOF
- name: Mount MCP servers as CLIs
id: mount-mcp-clis
continue-on-error: true
@@ -791,7 +791,7 @@ jobs:
export GH_AW_NODE_BIN
export COPILOT_API_KEY="$COPILOT_DUMMY_BYOK"
(umask 177 && touch /tmp/gh-aw/agent-stdio.log)
- printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.58/awf-config.schema.json","network":{"allowDomains":["*.vsblob.vsassets.io","api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.nuget.org","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","azuresearch-usnc.nuget.org","azuresearch-ussc.nuget.org","builds.dotnet.microsoft.com","ci.dot.net","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","dc.services.visualstudio.com","dev.azure.com","dist.nuget.org","dot.net","dotnet.microsoft.com","dotnetcli.blob.core.windows.net","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","nuget.org","nuget.pkg.github.com","nugetregistryv2prod.blob.core.windows.net","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","oneocsp.microsoft.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","pkgs.dev.azure.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com","www.microsoft.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.58"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json"
+ printf '%s\n' '{"$schema":"https://github.com/github/gh-aw-firewall/releases/download/v0.25.58/awf-config.schema.json","network":{"allowDomains":["*.vsblob.vsassets.io","api.business.githubcopilot.com","api.enterprise.githubcopilot.com","api.github.com","api.githubcopilot.com","api.individual.githubcopilot.com","api.nuget.org","api.snapcraft.io","archive.ubuntu.com","azure.archive.ubuntu.com","azuresearch-usnc.nuget.org","azuresearch-ussc.nuget.org","builds.dotnet.microsoft.com","ci.dot.net","crl.geotrust.com","crl.globalsign.com","crl.identrust.com","crl.sectigo.com","crl.thawte.com","crl.usertrust.com","crl.verisign.com","crl3.digicert.com","crl4.digicert.com","crls.ssl.com","dc.services.visualstudio.com","dev.azure.com","dist.nuget.org","dnceng.pkgs.visualstudio.com","dot.net","dotnet.microsoft.com","dotnetcli.blob.core.windows.net","github.com","host.docker.internal","json-schema.org","json.schemastore.org","keyserver.ubuntu.com","nuget.org","nuget.pkg.github.com","nugetregistryv2prod.blob.core.windows.net","ocsp.digicert.com","ocsp.geotrust.com","ocsp.globalsign.com","ocsp.identrust.com","ocsp.sectigo.com","ocsp.ssl.com","ocsp.thawte.com","ocsp.usertrust.com","ocsp.verisign.com","oneocsp.microsoft.com","packagecloud.io","packages.cloud.google.com","packages.microsoft.com","pkgs.dev.azure.com","ppa.launchpad.net","raw.githubusercontent.com","registry.npmjs.org","s.symcb.com","s.symcd.com","security.ubuntu.com","telemetry.enterprise.githubcopilot.com","ts-crl.ws.symantec.com","ts-ocsp.ws.symantec.com","www.googleapis.com","www.microsoft.com"]},"apiProxy":{"enabled":true,"enableTokenSteering":true,"maxRuns":500,"maxEffectiveTokens":25000000,"models":{"agent":["sonnet-6x","gpt-5.4","gpt-5.3","gemini-pro","any"],"antigravity":["copilot/antigravity*","google/antigravity*","gemini/antigravity*"],"any":["copilot/*","anthropic/*","openai/*","google/*","gemini/*"],"claude":["agent"],"codex":["agent"],"coding":["copilot/gpt-5*codex*","openai/gpt-5*codex*","gpt-5-codex"],"computer-use":["copilot/*computer-use*","google/*computer-use*","gemini/*computer-use*","openai/*computer-use*"],"copilot":["agent"],"deep-research":["copilot/deep-research*","copilot/o3-deep-research*","copilot/o4-mini-deep-research*","google/deep-research*","gemini/deep-research*","openai/o3-deep-research*","openai/o4-mini-deep-research*"],"gemini":["agent"],"gemini-3-flash":["copilot/gemini-3*flash*","google/gemini-3*flash*","gemini/gemini-3*flash*"],"gemini-3-pro":["copilot/gemini-3*pro*","google/gemini-3*pro*","gemini/gemini-3*pro*"],"gemini-3.1-flash":["copilot/gemini-3.1*flash*","google/gemini-3.1*flash*","gemini/gemini-3.1*flash*"],"gemini-3.1-pro":["copilot/gemini-3.1*pro*","google/gemini-3.1*pro*","gemini/gemini-3.1*pro*"],"gemini-3.5-flash":["copilot/gemini-3.5*flash*","google/gemini-3.5*flash*","gemini/gemini-3.5*flash*"],"gemini-flash":["copilot/gemini-*flash*","google/gemini-*flash*","gemini/gemini-*flash*"],"gemini-flash-lite":["copilot/gemini-*flash*lite*","google/gemini-*flash*lite*","gemini/gemini-*flash*lite*"],"gemini-pro":["copilot/gemini-*pro*","google/gemini-*pro*","gemini/gemini-*pro*"],"gemma":["copilot/gemma*","google/gemma*","gemini/gemma*"],"gpt-5":["copilot/gpt-5*","openai/gpt-5*"],"gpt-5-codex":["copilot/gpt-5*codex*","openai/gpt-5*codex*"],"gpt-5-mini":["copilot/gpt-5*mini*","openai/gpt-5*mini*"],"gpt-5-nano":["copilot/gpt-5*nano*","openai/gpt-5*nano*"],"gpt-5-pro":["copilot/gpt-5*pro*","openai/gpt-5*pro*"],"gpt-5.2":["copilot/gpt-5.2*","openai/gpt-5.2*"],"gpt-5.3":["copilot/gpt-5.3*","openai/gpt-5.3*"],"gpt-5.4":["copilot/gpt-5.4*","openai/gpt-5.4*"],"gpt-5.5":["copilot/gpt-5.5*","openai/gpt-5.5*"],"haiku":["copilot/*haiku*","anthropic/*haiku*"],"large":["sonnet","gpt-5-pro","gpt-5","gemini-pro"],"mini":["haiku","gpt-5-mini","gpt-5-nano","gemini-flash-lite"],"opus":["copilot/*opus*","anthropic/*opus*"],"opusplan":["opus?effort=high"],"reasoning":["copilot/o1*","copilot/o3*","copilot/o4*","openai/o1*","openai/o3*","openai/o4*"],"robotics":["copilot/*robotics*","google/*robotics*","gemini/*robotics*"],"small":["mini"],"sonnet":["copilot/*sonnet*","anthropic/*sonnet*"],"sonnet-6x":["copilot/*sonnet-4-5-*","anthropic/*sonnet-4-5-*","copilot/*sonnet-4-6*","anthropic/*sonnet-4-6*"],"summarization":["haiku","gpt-5-mini","gemini-flash-lite","mini"],"vision":["copilot/gemini-*image*","gemini/gemini-*image*","copilot/gemini-*flash*","gemini/gemini-*flash*"]}},"container":{"imageTag":"0.25.58"}}' > "${RUNNER_TEMP}/gh-aw/awf-config.json"
GH_AW_MODEL_MULTIPLIERS_PATH="/tmp/gh-aw/model_multipliers.json" node "${RUNNER_TEMP}/gh-aw/actions/merge_awf_model_multipliers.cjs"
cp "${RUNNER_TEMP}/gh-aw/awf-config.json" /tmp/gh-aw/awf-config.json
GH_AW_DOCKER_HOST_PATH_PREFIX_ARGS=""
@@ -903,7 +903,7 @@ jobs:
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
- GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com"
+ GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com"
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_API_URL: ${{ github.api_url }}
with:
@@ -1501,7 +1501,7 @@ jobs:
env:
GH_AW_AGENT_OUTPUT: ${{ steps.setup-agent-output-env.outputs.GH_AW_AGENT_OUTPUT }}
GH_AW_COMMENT_ID: ${{ needs.activation.outputs.comment_id }}
- GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com"
+ GH_AW_ALLOWED_DOMAINS: "*.vsblob.vsassets.io,api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.nuget.org,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,azuresearch-usnc.nuget.org,azuresearch-ussc.nuget.org,builds.dotnet.microsoft.com,ci.dot.net,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,dc.services.visualstudio.com,dev.azure.com,dist.nuget.org,dnceng.pkgs.visualstudio.com,dot.net,dotnet.microsoft.com,dotnetcli.blob.core.windows.net,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,nuget.org,nuget.pkg.github.com,nugetregistryv2prod.blob.core.windows.net,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,oneocsp.microsoft.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,pkgs.dev.azure.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com,www.googleapis.com,www.microsoft.com"
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_API_URL: ${{ github.api_url }}
GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"add_comment\":{\"max\":12,\"target\":\"*\"},\"create_issue\":{\"labels\":[\"flaky-test\"],\"max\":5,\"title_prefix\":\"[Flaky Test] \"},\"create_pull_request\":{\"allowed_files\":[\"src/**/*.cs\"],\"base_branch\":\"main\",\"draft\":true,\"excluded_files\":[\".github/**\"],\"labels\":[\"flaky-test\"],\"max\":1,\"max_patch_files\":100,\"max_patch_size\":1024,\"protect_top_level_dot_folders\":true,\"protected_files\":[\"package.json\",\"bun.lockb\",\"bunfig.toml\",\"deno.json\",\"deno.jsonc\",\"deno.lock\",\"global.json\",\"NuGet.Config\",\"Directory.Packages.props\",\"mix.exs\",\"mix.lock\",\"go.mod\",\"go.sum\",\"stack.yaml\",\"stack.yaml.lock\",\"pom.xml\",\"build.gradle\",\"build.gradle.kts\",\"settings.gradle\",\"settings.gradle.kts\",\"gradle.properties\",\"package-lock.json\",\"yarn.lock\",\"pnpm-lock.yaml\",\"npm-shrinkwrap.json\",\"requirements.txt\",\"Pipfile\",\"Pipfile.lock\",\"pyproject.toml\",\"setup.py\",\"setup.cfg\",\"Gemfile\",\"Gemfile.lock\",\"uv.lock\",\"CODEOWNERS\",\"DESIGN.md\",\"README.md\",\"CONTRIBUTING.md\",\"CHANGELOG.md\",\"SECURITY.md\",\"CODE_OF_CONDUCT.md\",\"AGENTS.md\",\"CLAUDE.md\",\"GEMINI.md\"],\"protected_files_policy\":\"request_review\",\"title_prefix\":\"[Flaky Test] \"},\"create_report_incomplete_issue\":{},\"missing_data\":{},\"missing_tool\":{},\"noop\":{\"max\":1,\"report-as-issue\":\"false\"},\"report_incomplete\":{}}"
diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md
index 11c9878ba76..011ccbcb80c 100644
--- a/.github/workflows/flaky-test-detector.agent.md
+++ b/.github/workflows/flaky-test-detector.agent.md
@@ -21,6 +21,10 @@ network:
- defaults
- dotnet
- dev.azure.com
+ # Legacy AzDO package host used by the dotnet8/dotnet9/dotnet10 feeds in NuGet.config.
+ # Distinct from pkgs.dev.azure.com (covered by `dotnet`); without it, NuGet restore of the
+ # runtime packs is blocked by the firewall and the whole-repo build fails.
+ - dnceng.pkgs.visualstudio.com
tools:
edit:
@@ -294,6 +298,15 @@ Build the whole repo a single time up front so every subsequent reproduction/val
./build.sh
```
+**Fail fast on an environmental build failure — do not loop.** If this first `./build.sh` fails for
+**environmental/network reasons** (NuGet restore cannot reach a feed, a blocked domain, SDK download
+failure — *not* a compile or test error), **do not retry the build, do not investigate, and do not
+attempt any determinism code fix.** Re-running the build against a blocked feed burns the entire token
+budget on NuGet retries and huge logs, and the run gets cut off before it can open a PR. Instead, go
+**straight to the quarantine fallback (Step 7d) for every selected test**, note in the PR that local
+reproduction was blocked by the environment, and finish the run. One failed build attempt is enough to
+make this determination — a missing/unreachable feed will not fix itself on a second attempt.
+
The whole-repo build takes ~2-3 minutes — **never cancel it**. `./build.sh` also builds every test
project, so each test's runnable assembly already exists under
`artifacts/bin//Debug//.dll` for the reproduction loops below. Network
From 05345f3535b334559eaf1a3fa2d45834a47cb1df Mon Sep 17 00:00:00 2001
From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com>
Date: Wed, 3 Jun 2026 12:42:17 +0200
Subject: [PATCH 2/5] Defer quarantine to next run and harden issue dedup
Analysis of run 26877237491 showed the agent created 4 tracking issues via
create_issue, then exhausted the 25M effective-token budget trying to
discover their issue numbers to write [ActiveIssue(.../issues/NNNN)]
attributes in the same run. That is impossible: create_issue is a safe
output filed by a post-run job and returns no number during the run.
- Step 5: a flake is quarantine-eligible only if its tracking issue was
already OPEN before this run (real number readable now). A just-created
issue does not count and must not be quarantined this run; it becomes
eligible next run. Explicitly forbid polling/guessing for a just-created
issue's number (the behavior that exhausted the budget).
- Step 4: note create_issue returns no number; stress the dedup marker must
be copied exactly as (the agent dropped the
'-id', which would orphan the issue from future marker searches); add a
title-search fallback for legacy issues lacking the marker; note the
benign 'Malformed version:' gh stderr warning so it isn't misread as a
command failure.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
.../workflows/flaky-test-detector.agent.md | 30 +++++++++++++++----
1 file changed, 25 insertions(+), 5 deletions(-)
diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md
index 011ccbcb80c..f6dd4a26a99 100644
--- a/.github/workflows/flaky-test-detector.agent.md
+++ b/.github/workflows/flaky-test-detector.agent.md
@@ -196,6 +196,13 @@ recently-closed), e.g. via the `github` tools or:
gh issue list --repo dotnet/msbuild --state all --search '"" in:body' --json number,state,title
```
+Older tracking issues may **predate the marker convention** and have none, so if the marker search
+finds nothing, also fall back to a **title search** for the test's short name before concluding no
+issue exists (e.g. `gh issue list --repo dotnet/msbuild --state all --search ' in:title'`)
+— this avoids re-filing a duplicate of a pre-existing issue. (Note: the sandboxed `gh` may print a
+benign `Malformed version:` warning to stderr; it is harmless — judge success by the JSON on stdout
+and the exit code, not by that line.)
+
- **If a related issue is OPEN:** post an `add_comment` to that issue number with the **new** evidence
(latest sources, build URLs, dates, legs/TFMs). Do not open a duplicate.
- **If a related issue exists but is CLOSED recently** (e.g. within ~30 days): do **not** open a
@@ -216,10 +223,15 @@ gh issue list --repo dotnet/msbuild --state all --search '"` without `-id` will not be found by future runs and
+ causes duplicate issues):
```
```
@@ -238,9 +250,17 @@ Build the set of tests to fix or quarantine in today's combined PR. A candidate
of these hold:
- It was classified as a **likely flake** in Step 3 (never act on a regression).
-- It has an **OPEN** `flaky-test` tracking issue (the one created or updated in Step 4). Do **not** use a
- recently-*closed* issue as the quarantine target — an `[ActiveIssue]` URL must point at an open issue.
- If only a closed issue exists, skip the test from today's PR (Step 4 already commented on it).
+- It has a `flaky-test` tracking issue that was **already OPEN before this run started** (a
+ pre-existing issue found in Step 4, with a real issue number you can read **now**). A brand-new
+ issue you filed via `create_issue` **this run does not count** — safe-output issues are created by a
+ post-run job and **return no number during the agent run**, so you cannot reference one in an
+ `[ActiveIssue(".../issues/")]` attribute yet. **Do not** attempt to discover the number of an
+ issue you just created (it does not exist yet — polling `gh issue list` / `gh api` / guessing ranges
+ for it is wasted effort that can exhaust the run's token budget). Such a test was tracked this run
+ and becomes quarantine-eligible on the **next** run, once its issue is open with a real number. Do
+ **not** use a recently-*closed* issue as the quarantine target either — an `[ActiveIssue]` URL must
+ point at an issue that is open and already has a number. If only a just-created or closed issue
+ exists, skip the test from today's PR (Step 4 already filed/commented).
- It is **not already covered by an open `flaky-test` PR** (this is the primary cross-run dedup — a
previous run's combined PR may still be open and unmerged, and its quarantines live on **that PR's
branch, not `main`**, so they will not show up in your fresh `main` checkout). Fetch the bodies of all
From 7e1c946ec435f093dd2f7a407486f41c0a9d59ed Mon Sep 17 00:00:00 2001
From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com>
Date: Wed, 3 Jun 2026 12:44:40 +0200
Subject: [PATCH 3/5] Document two-run quarantine cadence and exact dedup
marker in skill
Mirror the workflow behavioral fixes in the skill doc: explain that
create_issue is a deferred safe output returning no number during the run,
so a new flake is filed one run and quarantined the next (never poll for a
just-created issue's number); and stress the dedup marker must be copied
exactly with the -id segment, with a title-search fallback for legacy
issues.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
.github/skills/flaky-test-detector/SKILL.md | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md
index 5c0a465f17f..f452a8cf971 100644
--- a/.github/skills/flaky-test-detector/SKILL.md
+++ b/.github/skills/flaky-test-detector/SKILL.md
@@ -207,7 +207,16 @@ The detector surfaces evidence; it does **not** by itself prove flakiness. Befor
```html
```
- Search both open and recently-closed issues for this marker before filing a new one.
+ Copy the marker **exactly**, including the `-id` segment — `` (no `-id`)
+ will not be found by later searches and orphans the issue, causing duplicates. Search both open
+ and recently-closed issues for this marker before filing a new one; if the marker search finds
+ nothing, fall back to a title search to catch legacy issues created before the marker convention.
+- **Two-run quarantine cadence (file first, quarantine next run):** quarantining a test requires the
+ tracking issue's **number** for the `[ActiveIssue(.../issues/NNNN)]` URL, but the daily workflow
+ files issues with the gh-aw `create_issue` safe output, which is deferred to a **post-run** job and
+ returns **no number during the run**. So a brand-new flake is only filed this run and becomes
+ **quarantine-eligible the next run**, once its issue exists and is open with a readable number. Never
+ poll, guess, or wait for the number of an issue created in the current run.
- **Quarantine syntax (use exactly):** quarantine with `[ActiveIssue]` from
`Microsoft.DotNet.XUnitV3Extensions` (namespace `Xunit`, already referenced and imported by every
test project) — **not** `[Fact(Skip=...)]`. Add it above the existing `[Fact]`/`[Theory]`, keeping
From 27a411d42e3aa7ba2f8dc9918811339e835ef4cd Mon Sep 17 00:00:00 2001
From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com>
Date: Wed, 3 Jun 2026 12:59:28 +0200
Subject: [PATCH 4/5] Tell the agent to run the detector synchronously
Run 26877237491 ran the detector twice with identical inputs: it was
launched backgrounded with heavy -MaxBuilds/-MaxArtifactDownloads, polled
after ~65s while the scan was still running, saw no -JsonOut file yet
(the script writes JSON only on completion), misread that as a failure, and
relaunched -- re-downloading every AzDO artifact.
Instruct the agent to run the detector in the foreground and wait for exit,
and note that a missing -JsonOut file mid-run means 'not finished', not
'failed'. Mirror a short note in the skill doc.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
.github/skills/flaky-test-detector/SKILL.md | 5 +++++
.github/workflows/flaky-test-detector.agent.md | 8 ++++++++
2 files changed, 13 insertions(+)
diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md
index f452a8cf971..c023a38438b 100644
--- a/.github/skills/flaky-test-detector/SKILL.md
+++ b/.github/skills/flaky-test-detector/SKILL.md
@@ -85,6 +85,11 @@ The script writes a **human-readable report to stderr** and the **structured JSO
stdout** (and `-JsonOut` if given). `gh` must be installed and authenticated for the approval
filter and existing-issue cross-reference to work.
+> **Run it synchronously.** At high `-MaxBuilds`/`-MaxArtifactDownloads` the scan can take several
+> minutes and writes stdout / `-JsonOut` **only on completion**. Wait for the process to exit before
+> reading the JSON — don't background it or treat a not-yet-written `-JsonOut` file as a failure (it
+> just means the scan is still running). Re-running wastefully re-downloads every artifact.
+
### JSON report shape
```jsonc
diff --git a/.github/workflows/flaky-test-detector.agent.md b/.github/workflows/flaky-test-detector.agent.md
index f6dd4a26a99..76f331df51e 100644
--- a/.github/workflows/flaky-test-detector.agent.md
+++ b/.github/workflows/flaky-test-detector.agent.md
@@ -116,6 +116,14 @@ raised in step so the larger build set does not re-trip the artifact-download ca
The script writes a human-readable progress report to the log/host stream and the structured JSON
report to stdout (also written to `flaky-report.json`). Parse the JSON.
+**Run it synchronously and wait for it to exit.** At these `-MaxBuilds`/`-MaxArtifactDownloads`
+values the scan downloads and parses many artifacts and can take **several minutes**. It writes the
+JSON to stdout and to `-JsonOut` **only on completion** — there is no partial file mid-run. Do **not**
+background it (no `&`) and do **not** poll-then-bail: a missing `-JsonOut` file or empty stdout while
+the process is still running means *not finished yet*, **not** failure. Read the JSON only after the
+process has exited. Re-running because the file "wasn't there yet" just re-downloads every artifact and
+wastes the run's time and token budget. The same applies to the Step 1b quarantine scan below.
+
## Step 1b — Scan the quarantine pipeline (backlog signal)
Also scan the **quarantine pipeline** (AzDO definition **344**, `azure-pipelines/quarantine.yml`),
From 05f0e35f9c7a317c059076ff76cabb364a0e86dc Mon Sep 17 00:00:00 2001
From: Viktor Hofer <7412651+ViktorHofer@users.noreply.github.com>
Date: Wed, 3 Jun 2026 13:05:46 +0200
Subject: [PATCH 5/5] Merge flaky-test-detector skill into the workflow
The flaky-test-detector SKILL.md was a separate discoverable skill whose
content the daily workflow already re-stated in its own steps (thresholds,
dedup marker, quarantine/un-quarantine conventions, flake-vs-regression
classification, assembly->project mapping). Its only unique content was the
evidence-source model and a gloss of the detector JSON fields.
Fold that essential background into a new 'Background' section of
flaky-test-detector.agent.md so the workflow is self-contained, move the
detector script to .github/workflows/scripts/Get-FlakyTests.ps1 next to the
workflow that calls it (a scripts/ dir with no SKILL.md would be a malformed
skill), update the two invocation paths and the assembly-mapping reference,
and delete SKILL.md. No other workflow or pipeline referenced the skill or
script; the workflow body is runtime-imported so no lock recompile is needed.
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
.github/skills/flaky-test-detector/SKILL.md | 262 ------------------
.../workflows/flaky-test-detector.agent.md | 35 ++-
.../scripts/Get-FlakyTests.ps1 | 0
3 files changed, 28 insertions(+), 269 deletions(-)
delete mode 100644 .github/skills/flaky-test-detector/SKILL.md
rename .github/{skills/flaky-test-detector => workflows}/scripts/Get-FlakyTests.ps1 (100%)
diff --git a/.github/skills/flaky-test-detector/SKILL.md b/.github/skills/flaky-test-detector/SKILL.md
deleted file mode 100644
index c023a38438b..00000000000
--- a/.github/skills/flaky-test-detector/SKILL.md
+++ /dev/null
@@ -1,262 +0,0 @@
----
-name: flaky-test-detector
-description: Detect flaky tests in dotnet/msbuild by scanning recent Azure DevOps CI builds (approved PR validation + rolling builds on main) for test failures that recur across multiple independent evidence sources. Use when asked to find flaky tests, investigate intermittent CI failures, triage the flaky-test label, or produce a flaky-test report.
----
-
-# Flaky Test Detector
-
-This skill identifies **flaky tests** in dotnet/msbuild: tests that fail intermittently rather
-than because of a real product regression. It works entirely against **anonymously accessible**
-Azure DevOps APIs on the public CI (`dnceng-public` / `public`, pipeline `msbuild-pr`,
-definition **75**), so it needs no Azure DevOps credentials.
-
-## When to Use
-
-- User asks to find or triage flaky tests, or to investigate intermittent CI failures.
-- Periodic (scheduled) flaky-test sweeps.
-- Before deciding whether to quarantine a test or open/refresh a `flaky-test` tracking issue.
-- To gather evidence (which PRs / rolling builds, which legs/TFMs, error signatures) for a
- specific suspected-flaky test.
-
-## Core Idea: Evidence Sources
-
-A test is flagged as flaky when it fails across several **independent evidence sources**. A
-source is one of:
-
-1. **A Pull Request** — all failed PR-validation builds for one PR collapse into a single
- source. Only **non-draft, approved (or merged) PRs targeting `main`** count, because a
- failure in code reviewers signed off on is unlikely to be caused by that PR's own changes.
-2. **A single failed rolling/CI build on `main`** (reason `individualCI` / `batchedCI` /
- `schedule`). `main` is expected to be green, so a test failing there is strong, independent
- evidence of flakiness (or a regression — see below).
-
-A test failing across many *unrelated* approved PRs **and/or** multiple rolling `main` builds
-cannot be explained by any one change, which is the signature of flakiness.
-
-> **Scope (current):** `main` branch only. Builds on `vs18.x` / `exp/*` branches are ignored.
-
-## Data Path (all anonymous on dnceng-public)
-
-The Azure DevOps **Test Management API is NOT anonymously accessible**, so this skill uses
-published pipeline **artifacts** instead:
-
-1. **Build list** — `/_apis/build/builds?definitions=75&resultFilter=failed&minTime=...` returns
- failed builds with `reason`, `sourceBranch`, and `triggerInfo['pr.number']`.
-2. **Timeline** — `/_apis/build/builds/{id}/timeline` exposes which *legs* (jobs) failed, used
- to download only the relevant artifacts.
-3. **Artifacts list** — `/_apis/build/builds/{id}/artifacts` lists per-leg containers named
- `" test logs"` and `" build logs"`.
-4. **Artifact zip** — `...artifacts?artifactName=&$format=zip` downloads the container.
- - `" test logs"` zips contain the **`.trx`** result files (failed test names + error
- message/stack trace).
- - `" build logs"` zips (misleadingly named) contain the **full xUnit console `.log`**
- output and a `.binlog`. The scan does **not** download these; the fix phase pulls them on
- demand when it needs stdout for diagnosis.
-5. **TRX parsing** — failed tests come from `//UnitTestResult[@outcome='Failed']/@testName`. The
- `@testName` is fully-qualified and may include a parameterized suffix, e.g.
- `Namespace.Class.Method(runtimeToUse: "NET", ...)`. The detector stores both the raw name and
- a normalized name (parameter suffix stripped). The assembly/TFM/arch are encoded in the TRX
- file name, e.g. `Microsoft.Build.Engine.UnitTests_net472_x86.trx`.
-
-> **TRX has no stdout.** The `