From 5a37820fe7faf6cf49aae2e606cd786acebb3411 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 22 Sep 2026 10:16:11 +0200 Subject: [PATCH 1/5] Prevent CI scan duplicates after inconclusive lookups Route KBE searches through an integrity-gated helper that retains author metadata and complete responses. Refuse filing from filtered, failed, malformed, or incomplete lookups, and use the correct search tool for fix PRs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 853fa924-7d9d-4d5b-9990-f7ac0d9d5e88 --- .github/workflows/ci-failure-scan.lock.yml | 3 +- .github/workflows/ci-failure-scan.md | 14 +- .github/workflows/evals/README.md | 9 + .../workflows/evals/ci-failure-scan.eval.yaml | 11 +- .../evals/test_ci_failure_scan_search.py | 222 ++++++++++++++++++ .../shared/create-kbe.instructions.md | 55 ++++- .github/workflows/shared/search-kbe.sh | 94 ++++++++ 7 files changed, 395 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/evals/test_ci_failure_scan_search.py create mode 100644 .github/workflows/shared/search-kbe.sh diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index c7d8e6ef6d0ac3..cde8bfcaae590a 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9098a4e3e6ccdd1f9f936b64ee6ac4362232803d71959645d9e6023543fec795","body_hash":"8258800d731e8cf437014b12f868c5aec40be23451abe644e2dd1f3b68ef40c5","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"f0d80c7e319d4dcf8081a63a543d93303e3b94e5276c9fccdc3a5d06e76b1107","body_hash":"a3d26df3372121f8603d52417ad59a98f664d696988d51816d4131ed9a547093","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -676,6 +676,7 @@ jobs: export DEBUG="*" export GH_AW_ENGINE="copilot" + export GH_AW_MCP_CLI_SERVERS='["github","safeoutputs"]' MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index 9ebecc66e39f72..dcafb93a523f82 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -40,6 +40,7 @@ concurrency: cancel-in-progress: true tools: + cli-proxy: true github: toolsets: [pull_requests, repos, issues, search] min-integrity: approved @@ -90,6 +91,10 @@ The agent runs read-only. All writes go through `safe-outputs`. 10. **All intermediate state under `/tmp/gh-aw/agent/`.** Each bash invocation is a fresh subshell; persist anything you want to keep. 11. **AzDO API: anonymous only.** Stay on `_apis/build/...`. Never call `_apis/test/...` or `vstmr.dev.azure.com` (both redirect to sign-in). 12. **Don't add `area-*` references to issue titles.** Multi-area titles produce multi-label assignments from the labeler bot. +13. **An inconclusive lookup must never create an issue.** Use + `.github/workflows/shared/search-kbe.sh` for all duplicate and fix-PR + searches. Preserve its evidence and inspect the complete summary. A filtered, + failed, malformed, or incomplete lookup is not "no existing KBE". ## What this run must accomplish @@ -284,6 +289,12 @@ printf '%s\t%s\t%s\n' "$xkey" "aw_" "" >> /tmp/gh-aw/agent/filed.ts #### Step 4.2 through Step 4.6 — Run the shared KBE lookup flow +Run every search through `.github/workflows/shared/search-kbe.sh` as described +in the shared instructions. Do not substitute an ad-hoc `github search_issues` +pipeline. Only complete `no_match` results and fully verified nonmatching +candidates can support filing; a `blocked` result stops emission for that +signature until the lookup succeeds. + Follow exactly these sections from `.github/workflows/shared/create-kbe.instructions.md`, in this order: 1. `` / `## Search for an existing KBE` @@ -309,6 +320,7 @@ Record the same lookup outcomes described there, retaining any - `existing-PR #` - `skipped: recently-closed dup #, needs human review` - `skipped: integrity-filtered candidate, needs human review` +- `skipped: lookup incomplete, needs human review` #### Step 4.7 — Verify the candidate KBE actually matches @@ -365,7 +377,7 @@ Per signature, append one outcome line to `/tmp/gh-aw/agent/coverage/. `` is one of: `filed-issue #aw_`, `existing-kbe #`, `existing-PR #`, `skipped: `. -A skipped signature MUST have a reason. Recognized values: `build canceled`, `< 2 occurrences and not blocking`, `cap reached`, `infra noise — no stable signature`, `signature absent from follow-up build #`, `stale build window (>14d)`, `no follow-up build yet — defer to next run`, `fix already merged after source build`, `fix recently merged in #`, `dup of filed-issue #aw_ earlier in this run`, `cross-def dup of filed-issue #aw_ earlier in this run`, `representative KBE filed as #aw_`, `leg-level failure filed as #aw_`, `ambiguous dup #/#, needs human review`, `integrity-filtered candidate, needs human review`, `suspected infra outage`, `weak signature`, `signature did not match failure.log (N=)`, `native assert not in xunit log`. The list is non-exhaustive but additions SHOULD reuse one of these phrasings to keep the feedback workflow's tally aggregation stable. +A skipped signature MUST have a reason. Recognized values: `build canceled`, `< 2 occurrences and not blocking`, `cap reached`, `infra noise — no stable signature`, `signature absent from follow-up build #`, `stale build window (>14d)`, `no follow-up build yet — defer to next run`, `fix already merged after source build`, `fix recently merged in #`, `dup of filed-issue #aw_ earlier in this run`, `cross-def dup of filed-issue #aw_ earlier in this run`, `representative KBE filed as #aw_`, `leg-level failure filed as #aw_`, `ambiguous dup #/#, needs human review`, `integrity-filtered candidate, needs human review`, `lookup incomplete, needs human review`, `suspected infra outage`, `weak signature`, `signature did not match failure.log (N=)`, `native assert not in xunit log`. The list is non-exhaustive but additions SHOULD reuse one of these phrasings to keep the feedback workflow's tally aggregation stable. At end of run, print this table to the agent log: diff --git a/.github/workflows/evals/README.md b/.github/workflows/evals/README.md index 272a504328e0af..58a8b9f00c6fbc 100644 --- a/.github/workflows/evals/README.md +++ b/.github/workflows/evals/README.md @@ -80,6 +80,15 @@ is failing at eval time. ## Run locally +The deterministic scanner lookup tests need Python 3, Bash, and jq, but no +credentials or network access. They exercise author metadata, complete empty +results, filtered candidates, CLI-proxy arrays and MCP text envelopes, +issue and pull-request routing, incomplete responses, and tool failures: + +```bash +python3 .github/workflows/evals/test_ci_failure_scan_search.py +``` + The deterministic fixer tests need Python 3, Bash, jq, and Node with the eval dependencies installed (`npm ci --prefix .github/workflows/evals`), but no credentials or network access during testing. They exercise the shared intake diff --git a/.github/workflows/evals/ci-failure-scan.eval.yaml b/.github/workflows/evals/ci-failure-scan.eval.yaml index 8c10f9cbada284..bda4acf7a33376 100644 --- a/.github/workflows/evals/ci-failure-scan.eval.yaml +++ b/.github/workflows/evals/ci-failure-scan.eval.yaml @@ -25,7 +25,7 @@ environment: - GITHUB_READ_ONLY=1 - --env - GITHUB_TOOLSETS=pull_requests,repos,issues,search - - ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 + - ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e stimuli: - name: live-kbe-safe-output @@ -48,7 +48,7 @@ stimuli: 1. Find a recent failed build on `main` of an outer-loop pipeline via the dnceng-public AzDO REST API. It is anonymous and needs no auth, so stay on `_apis/build/...`. Start with `runtime-extra-platforms`, which is definition 154, and fall back to other outer-loop defs if needed. `curl -fsSL "https://dev.azure.com/dnceng-public/public/_apis/build/builds?definitions=154&branchName=refs/heads/main&statusFilter=completed&resultFilter=failed&\$top=5&api-version=7.1"` 2. Open that build's timeline at `/_apis/build/builds/{id}/timeline?api-version=7.1`, pick a failed leg, and read its log at `/_apis/build/builds/{id}/logs/{logId}?api-version=7.1` to extract a concrete error or assertion signature. Save the leg log you extract from as `failure.log` under /tmp and grep it for your signature to get the hit count. - 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. Run `gh search issues 'in:title [ci-scan]' --repo dotnet/runtime --state open` and also search by your signature text. Assume you would file only if none matches. + 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. This eval exposes native GitHub MCP tools rather than production's CLI proxy. Use `search_issues` with `owner: "dotnet"`, `repo: "runtime"`, and `fields: ["number", "title", "state", "user", "labels", "html_url"]`. Search open and recently closed issues by your signature text following the shared search variations. Inspect complete results without discarding author metadata, filtered markers, errors, or result counts. A failed, filtered, malformed, or incomplete lookup is inconclusive, not an empty result. Do not emit a KBE from an inconclusive lookup or when a matching issue exists; report the reason instead. Do not use `gh` or direct GitHub API reads as a fallback. Then **emit the create-issue safe-output you would file, as a single markdown document at `./out/kbe.md`**, in exactly this shape. @@ -110,8 +110,11 @@ stimuli: name: searched-existing-kbe config: required: - - name: '^(bash|powershell)$' - command: 'gh (search|issue)' + - name: '(^|[-_.])search_issues$' + args: + owner: '^dotnet$' + repo: '^runtime$' + pattern: '"fields"\s*:\s*\[(?=[^\]]*"number")(?=[^\]]*"title")(?=[^\]]*"state")(?=[^\]]*"user")(?=[^\]]*"labels")(?=[^\]]*"html_url")' - type: prompt name: kbe-format-conformant diff --git a/.github/workflows/evals/test_ci_failure_scan_search.py b/.github/workflows/evals/test_ci_failure_scan_search.py new file mode 100644 index 00000000000000..a9b8d92c2703fb --- /dev/null +++ b/.github/workflows/evals/test_ci_failure_scan_search.py @@ -0,0 +1,222 @@ +#!/usr/bin/env python3 + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +WORKFLOWS = Path(__file__).resolve().parents[1] +SEARCH_SCRIPT = WORKFLOWS / "shared/search-kbe.sh" + + +def issue(number=42, **overrides): + return { + "number": number, + "title": "[ci-scan] Test failure", + "state": "open", + "user": {"login": "github-actions[bot]", "type": "Bot"}, + "labels": [{"name": "Known Build Error"}], + "html_url": f"https://github.com/dotnet/runtime/issues/{number}", + **overrides, + } + + +def response(items=None, **overrides): + items = [] if items is None else items + return { + "items": items, + "total_count": len(items), + "incomplete_results": False, + **overrides, + } + + +class SearchTests(unittest.TestCase): + def run_search(self, value, *, fail=False, raw=False, status="candidates", + kind="issues", query='is:issue is:open "Missing BBF_PROF_WEIGHT flag"'): + with tempfile.TemporaryDirectory() as directory: + temp = Path(directory) + fixture = value if raw else json.dumps(value) + (temp / "fixture").write_text(fixture) + output = temp / "lookup" + output.mkdir() + (output / "summary.json").write_text('{"status":"no_match"}') + env = { + **os.environ, + "TEST_DIRECTORY": directory, + "TEST_FAIL": "1" if fail else "0", + } + mock = """ +github() { + printf '%s\\0' "$@" > "$TEST_DIRECTORY/arguments" + cat > "$TEST_DIRECTORY/request.json" + cat "$TEST_DIRECTORY/fixture" + if [ "$TEST_FAIL" = "1" ]; then + echo "Simulated search failure" >&2 + return 1 + fi +} +""" + result = subprocess.run( + ["bash", "-c", mock + SEARCH_SCRIPT.read_text(), "search-kbe", + kind, query, str(output)], + env=env, capture_output=True, text=True, + ) + self.assertEqual( + (temp / "arguments").read_text().split("\0")[:-1], + ["search_issues" if kind == "issues" else "search_pull_requests", "."], + ) + request = json.loads((temp / "request.json").read_text()) + self.assertEqual(request["owner"], "dotnet") + self.assertEqual(request["repo"], "runtime") + self.assertEqual( + request["fields"], + ["number", "title", "state", "user", "labels", "html_url"], + ) + self.assertEqual(request["perPage"], 100) + self.assertIn("repo:dotnet/runtime", request["query"]) + self.assertIn('"Missing BBF_PROF_WEIGHT flag"', request["query"]) + self.assertEqual((output / "response.json").read_text(), fixture) + summary = json.loads((output / "summary.json").read_text()) + self.assertEqual(summary["status"], status, result.stderr) + self.assertEqual(json.loads(result.stdout), summary) + if status == "blocked": + self.assertNotEqual(result.returncode, 0) + self.assertTrue(summary["reason"]) + self.assertTrue(result.stderr) + else: + self.assertEqual(result.returncode, 0, result.stderr) + return summary + + def test_preserves_candidate_metadata(self): + candidates = [issue(134292), issue(134169)] + summary = self.run_search(response(candidates)) + self.assertEqual(summary["items"], candidates) + self.assertEqual(summary["total_count"], 2) + + def test_only_complete_empty_response_is_no_match(self): + summary = self.run_search(response(), status="no_match") + self.assertEqual(summary["items"], []) + + def test_uses_pull_request_search_for_fix_queries(self): + for state in ("open", "closed"): + with self.subTest(state=state): + candidate = issue(state=state, html_url="https://github.com/dotnet/runtime/pull/42") + summary = self.run_search( + response([candidate]), kind="pull-requests", + query=f'is:pr is:{state} "Missing BBF_PROF_WEIGHT flag"', + ) + self.assertEqual(summary["items"], [candidate]) + + def test_filtered_results_never_become_no_match(self): + for value in ( + response(["[Filtered]"]), + response([issue(), "[Filtered]"]), + response([], filtered="[Filtered]"), + {"content": [{"type": "text", "text": "[Filtered]"}]}, + response([], warning="[DIFC-FILTERED]"), + ): + with self.subTest(value=value): + summary = self.run_search(value, status="blocked") + self.assertEqual(summary["reason"], "integrity-filtered candidate") + + def test_accepts_mcp_text_envelope(self): + for value in (response(), response([issue()])): + with self.subTest(value=value): + summary = self.run_search( + {"content": [{"type": "text", "text": json.dumps(value)}]}, + status="candidates" if value["items"] else "no_match", + ) + self.assertEqual(summary["items"], value["items"]) + + def test_accepts_cli_proxy_content_array(self): + for value in (response(), response([issue()])): + with self.subTest(value=value): + summary = self.run_search( + [value], + status="candidates" if value["items"] else "no_match", + ) + self.assertEqual(summary["items"], value["items"]) + self.run_search([response(["[Filtered]"])], status="blocked") + self.run_search([response(), response()], status="blocked") + + def test_blocks_incomplete_and_malformed_results(self): + for value in ( + response(incomplete_results=True), + response(total_count=1), + response([issue()], total_count=1000), + response([issue(user=None)]), + response([issue(user={})]), + response([issue(number="42")]), + response([issue(labels=None)]), + {"items": []}, + {"message": "API rate limit exceeded"}, + {"isError": True, "content": [{"type": "text", "text": "Request failed"}]}, + {"content": []}, + {"content": [{"type": "text", "text": "not JSON"}]}, + None, + [], + ): + with self.subTest(value=value): + self.run_search(value, status="blocked") + + def test_blocks_non_json_and_multiple_documents(self): + for value in ("", "Sign in", "{}\n{}", "null\n", "truncated {"): + with self.subTest(value=value): + self.run_search(value, raw=True, status="blocked") + + def test_tool_failure_cannot_reuse_previous_no_match(self): + self.run_search(response(), fail=True, status="blocked") + + def test_workflow_uses_guarded_searches(self): + workflow = (WORKFLOWS / "ci-failure-scan.md").read_text() + self.assertIn(" cli-proxy: true\n", workflow) + self.assertIn(".github/workflows/shared/search-kbe.sh", workflow) + tally = workflow.split("Recognized values:", 1)[1] + self.assertIn("`lookup incomplete, needs human review`", tally) + instructions = (WORKFLOWS / "shared/create-kbe.instructions.md").read_text() + self.assertIn("If any lookup returns a `[Filtered]` marker", instructions) + self.assertNotIn("linked-tracker: integrity-filtered", instructions) + + def test_eval_uses_production_github_server(self): + prefix = "# gh-aw-manifest: " + manifest = next( + line[len(prefix):] + for line in (WORKFLOWS / "ci-failure-scan.lock.yml").read_text().splitlines() + if line.startswith(prefix) + ) + image_prefix = "ghcr.io/github/github-mcp-server:" + production_image = next( + container["pinned_image"] + for container in json.loads(manifest)["containers"] + if container["image"].startswith(image_prefix) + ) + eval_image = next( + line.strip()[2:] + for line in (WORKFLOWS / "evals/ci-failure-scan.eval.yaml").read_text().splitlines() + if line.strip().startswith("- " + image_prefix) + ) + self.assertEqual(eval_image, production_image) + + def test_invalid_arguments_fail_before_lookup(self): + for args in ( + [], + [""], + ["issues", "", "/tmp/unused-kbe-search"], + ["issues", "query", ""], + ["invalid", "query", "/tmp/unused-kbe-search"], + ): + with self.subTest(args=args): + result = subprocess.run( + ["bash", str(SEARCH_SCRIPT), *args], + capture_output=True, text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Usage:", result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/shared/create-kbe.instructions.md b/.github/workflows/shared/create-kbe.instructions.md index efce4aa721b69d..8a7d72dabc4705 100644 --- a/.github/workflows/shared/create-kbe.instructions.md +++ b/.github/workflows/shared/create-kbe.instructions.md @@ -40,13 +40,55 @@ responsible for: ## Search for an existing KBE Search open `dotnet/runtime` issues with the `Known Build Error` label. Try -these variations in order, scanning the first ~10 results of each. GitHub -best-match ranking can place noisier hits above the correct one. +these variations in order and inspect every returned candidate. Narrow overly +broad queries instead of truncating results. GitHub best-match ranking can +place noisier hits above the correct one. -For every `search_issues` call in this flow, include `user` in the requested +For every `search_issues` or `search_pull_requests` call, include `user` in the requested `fields`, even when the author is not otherwise needed. The integrity gateway uses `user.login` to recognize trusted bots before filtering search results. +### Preserve lookup results + +When using the workflow's `github` CLI proxy, run the checked-in helper for +every issue or PR search in this file, including open, recently closed, and +merged searches. Run it from the repository root, using a separate evidence +directory for each query: + +```bash +bash .github/workflows/shared/search-kbe.sh issues \ + 'is:issue is:open label:"Known Build Error" "distinctive assertion text"' \ + /tmp/gh-aw/agent/kbe-search/signature-1-open +``` + +Use `pull-requests` instead of `issues` for open or merged fix-PR queries. +The two MCP search tools enforce different issue-type qualifiers. + +The helper supplies the author and label fields, preserves `request.json` and +the complete `response.json`, and emits `summary.json`. It continues to use the +integrity-gated proxy, not `gh` or a direct GitHub API request. + +- `candidates` means inspect the returned issues with the full candidate + verification below. A search hit alone does not prove a duplicate. +- `no_match` means only this query returned a complete, valid empty result. + Continue the remaining required search variations before deciding to file. +- `blocked` or any nonzero exit means this lookup is inconclusive, not empty. + Do not emit a KBE for that signature. Narrow an overbroad query and rerun; + otherwise record `skipped: integrity-filtered candidate, needs human review` + for a filtered response or `skipped: lookup incomplete, needs human review` + for a failed, malformed, or incomplete response. + +Never pipe lookup output through `grep`, `head`, or a projection that discards +filtered markers, errors, author metadata, or result counts. Never replace a +failed lookup with an empty array or bypass integrity filtering. + +Callers using native MCP tools instead of the CLI proxy must request +`fields: ["number", "title", "state", "user", "labels", "html_url"]` and apply +the same result checks before interpreting the response. A filtered candidate +body or comments read is also inconclusive, not evidence of a different failure. + +### Search variations + 1. Full `[FAIL]` line. 2. Assertion text. 3. Exception class + test name. @@ -112,14 +154,13 @@ If two candidate KBEs share more than 70% of their `ErrorMessage` / `ErrorPattern` tokens, do **not** guess: record `skipped: ambiguous dup #/#, needs human review` and stop. -If a KBE-labeled search returns a `[Filtered]` marker, treat it as a likely +If any lookup returns a `[Filtered]` marker, treat it as a possible existing-KBE hit and record `skipped: integrity-filtered candidate, needs human review` instead of creating a fresh KBE. -If variation 5 returns a `[Filtered]` marker, record -`linked-tracker: integrity-filtered, needs human review` for cross-linking, but -do not treat it as a KBE substitute. +This includes variation 5 and searches without a KBE label filter. A hidden +result does not establish whether the issue is an unlabeled tracker or a KBE. On any visible hit whose title or body references the same test class on any platform, record `existing-kbe #` (or `linked-tracker #` for variation 5 diff --git a/.github/workflows/shared/search-kbe.sh b/.github/workflows/shared/search-kbe.sh new file mode 100644 index 00000000000000..9cdac678679ad6 --- /dev/null +++ b/.github/workflows/shared/search-kbe.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage="Usage: search-kbe.sh " +if [ "$#" -ne 3 ] || [ -z "$2" ] || [ -z "$3" ]; then + echo "$usage" >&2 + exit 1 +fi + +case "$1" in + issues) tool="search_issues" ;; + pull-requests) tool="search_pull_requests" ;; + *) echo "$usage" >&2; exit 1 ;; +esac + +directory="$3" +mkdir -p "$directory" +jq -n '{status: "blocked", reason: "lookup did not complete"}' > "$directory/summary.json" +jq -n --arg query "repo:dotnet/runtime $2" '{ + owner: "dotnet", + repo: "runtime", + query: $query, + fields: ["number", "title", "state", "user", "labels", "html_url"], + perPage: 100 +}' > "$directory/request.json" + +if ! github "$tool" . < "$directory/request.json" > "$directory/response.json"; then + echo "KBE lookup failed; do not create an issue from this search." >&2 + cat "$directory/summary.json" + exit 1 +fi + +if ! jq -s ' + def filtered: + any(.. | strings; test("\\[Filtered\\]|\\[DIFC-FILTERED\\]"; "i")); + def blocked($reason): + {status: "blocked", reason: $reason}; + def nonempty_string: + type == "string" and length > 0; + def candidate: + type == "object" and + (.number | type == "number" and . > 0 and floor == .) and + (.title | nonempty_string) and + (.state == "open" or .state == "closed") and + (.user.login | nonempty_string) and + (.labels | type == "array") and + (.html_url | nonempty_string); + + if length != 1 then + blocked("expected one JSON response") + elif filtered then + blocked("integrity-filtered candidate") + else + .[0] | + if type == "array" and length == 1 then .[0] else . end | + if type != "object" then error("expected an object") + elif .isError == true or has("error") then error("tool reported an error") + elif has("content") then + if (.content | type == "array" and length == 1) and + .content[0].type == "text" then + .content[0].text | fromjson + else error("expected one MCP text result") + end + else . + end | + if filtered then + blocked("integrity-filtered candidate") + elif type != "object" or .isError == true or has("error") then + blocked("invalid search result") + elif .incomplete_results != false or (.items | type != "array") or + (.total_count | type != "number") then + blocked("incomplete or invalid search result") + elif .total_count != (.items | length) then + blocked("not all candidates returned; narrow the query") + elif all(.items[]; candidate) | not then + blocked("candidate metadata is missing or invalid") + else + { + status: (if .total_count == 0 then "no_match" else "candidates" end), + total_count, + items + } + end + end +' "$directory/response.json" > "$directory/summary.json"; then + jq -n '{status: "blocked", reason: "malformed search response"}' > "$directory/summary.json" +fi + +cat "$directory/summary.json" +if ! jq -e '.status == "no_match" or .status == "candidates"' "$directory/summary.json" > /dev/null; then + echo "KBE lookup is inconclusive; do not create an issue from this search." >&2 + exit 1 +fi From 130968defcb68f6950eb5622ee591484b6b3b7b8 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 22 Sep 2026 10:28:45 +0200 Subject: [PATCH 2/5] Keep CI scan deduplication changes in Markdown Move lookup safeguards into the shared KBE instructions. Remove the new helper and tests, and restore workflow configuration and eval changes so the PR only modifies the shared Markdown file. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 853fa924-7d9d-4d5b-9990-f7ac0d9d5e88 --- .github/workflows/ci-failure-scan.lock.yml | 3 +- .github/workflows/ci-failure-scan.md | 14 +- .github/workflows/evals/README.md | 9 - .../workflows/evals/ci-failure-scan.eval.yaml | 11 +- .../evals/test_ci_failure_scan_search.py | 222 ------------------ .../shared/create-kbe.instructions.md | 80 +++---- .github/workflows/shared/search-kbe.sh | 94 -------- 7 files changed, 44 insertions(+), 389 deletions(-) delete mode 100644 .github/workflows/evals/test_ci_failure_scan_search.py delete mode 100644 .github/workflows/shared/search-kbe.sh diff --git a/.github/workflows/ci-failure-scan.lock.yml b/.github/workflows/ci-failure-scan.lock.yml index cde8bfcaae590a..c7d8e6ef6d0ac3 100644 --- a/.github/workflows/ci-failure-scan.lock.yml +++ b/.github/workflows/ci-failure-scan.lock.yml @@ -1,4 +1,4 @@ -# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"f0d80c7e319d4dcf8081a63a543d93303e3b94e5276c9fccdc3a5d06e76b1107","body_hash":"a3d26df3372121f8603d52417ad59a98f664d696988d51816d4131ed9a547093","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} +# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"9098a4e3e6ccdd1f9f936b64ee6ac4362232803d71959645d9e6023543fec795","body_hash":"8258800d731e8cf437014b12f868c5aec40be23451abe644e2dd1f3b68ef40c5","compiler_version":"v0.86.2","strict":true,"agent_id":"copilot","agent_model":"claude-opus-4.8","engine_versions":{"copilot":"1.0.79"}} # gh-aw-manifest: {"version":1,"secrets":["COPILOT_PAT_0","COPILOT_PAT_1","COPILOT_PAT_2","COPILOT_PAT_3","COPILOT_PAT_4","COPILOT_PAT_5","COPILOT_PAT_6","COPILOT_PAT_7","COPILOT_PAT_8","COPILOT_PAT_9","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"373c709c69115d41ff229c7e5df9f8788daa9553","version":"v9"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]} # This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md # @@ -676,7 +676,6 @@ jobs: export DEBUG="*" export GH_AW_ENGINE="copilot" - export GH_AW_MCP_CLI_SERVERS='["github","safeoutputs"]' MCP_GATEWAY_UID=$(id -u 2>/dev/null || echo '0') MCP_GATEWAY_GID=$(id -g 2>/dev/null || echo '0') source "${RUNNER_TEMP}/gh-aw/actions/resolve_docker_socket_gid.sh" diff --git a/.github/workflows/ci-failure-scan.md b/.github/workflows/ci-failure-scan.md index dcafb93a523f82..9ebecc66e39f72 100644 --- a/.github/workflows/ci-failure-scan.md +++ b/.github/workflows/ci-failure-scan.md @@ -40,7 +40,6 @@ concurrency: cancel-in-progress: true tools: - cli-proxy: true github: toolsets: [pull_requests, repos, issues, search] min-integrity: approved @@ -91,10 +90,6 @@ The agent runs read-only. All writes go through `safe-outputs`. 10. **All intermediate state under `/tmp/gh-aw/agent/`.** Each bash invocation is a fresh subshell; persist anything you want to keep. 11. **AzDO API: anonymous only.** Stay on `_apis/build/...`. Never call `_apis/test/...` or `vstmr.dev.azure.com` (both redirect to sign-in). 12. **Don't add `area-*` references to issue titles.** Multi-area titles produce multi-label assignments from the labeler bot. -13. **An inconclusive lookup must never create an issue.** Use - `.github/workflows/shared/search-kbe.sh` for all duplicate and fix-PR - searches. Preserve its evidence and inspect the complete summary. A filtered, - failed, malformed, or incomplete lookup is not "no existing KBE". ## What this run must accomplish @@ -289,12 +284,6 @@ printf '%s\t%s\t%s\n' "$xkey" "aw_" "" >> /tmp/gh-aw/agent/filed.ts #### Step 4.2 through Step 4.6 — Run the shared KBE lookup flow -Run every search through `.github/workflows/shared/search-kbe.sh` as described -in the shared instructions. Do not substitute an ad-hoc `github search_issues` -pipeline. Only complete `no_match` results and fully verified nonmatching -candidates can support filing; a `blocked` result stops emission for that -signature until the lookup succeeds. - Follow exactly these sections from `.github/workflows/shared/create-kbe.instructions.md`, in this order: 1. `` / `## Search for an existing KBE` @@ -320,7 +309,6 @@ Record the same lookup outcomes described there, retaining any - `existing-PR #` - `skipped: recently-closed dup #, needs human review` - `skipped: integrity-filtered candidate, needs human review` -- `skipped: lookup incomplete, needs human review` #### Step 4.7 — Verify the candidate KBE actually matches @@ -377,7 +365,7 @@ Per signature, append one outcome line to `/tmp/gh-aw/agent/coverage/. `` is one of: `filed-issue #aw_`, `existing-kbe #`, `existing-PR #`, `skipped: `. -A skipped signature MUST have a reason. Recognized values: `build canceled`, `< 2 occurrences and not blocking`, `cap reached`, `infra noise — no stable signature`, `signature absent from follow-up build #`, `stale build window (>14d)`, `no follow-up build yet — defer to next run`, `fix already merged after source build`, `fix recently merged in #`, `dup of filed-issue #aw_ earlier in this run`, `cross-def dup of filed-issue #aw_ earlier in this run`, `representative KBE filed as #aw_`, `leg-level failure filed as #aw_`, `ambiguous dup #/#, needs human review`, `integrity-filtered candidate, needs human review`, `lookup incomplete, needs human review`, `suspected infra outage`, `weak signature`, `signature did not match failure.log (N=)`, `native assert not in xunit log`. The list is non-exhaustive but additions SHOULD reuse one of these phrasings to keep the feedback workflow's tally aggregation stable. +A skipped signature MUST have a reason. Recognized values: `build canceled`, `< 2 occurrences and not blocking`, `cap reached`, `infra noise — no stable signature`, `signature absent from follow-up build #`, `stale build window (>14d)`, `no follow-up build yet — defer to next run`, `fix already merged after source build`, `fix recently merged in #`, `dup of filed-issue #aw_ earlier in this run`, `cross-def dup of filed-issue #aw_ earlier in this run`, `representative KBE filed as #aw_`, `leg-level failure filed as #aw_`, `ambiguous dup #/#, needs human review`, `integrity-filtered candidate, needs human review`, `suspected infra outage`, `weak signature`, `signature did not match failure.log (N=)`, `native assert not in xunit log`. The list is non-exhaustive but additions SHOULD reuse one of these phrasings to keep the feedback workflow's tally aggregation stable. At end of run, print this table to the agent log: diff --git a/.github/workflows/evals/README.md b/.github/workflows/evals/README.md index 58a8b9f00c6fbc..272a504328e0af 100644 --- a/.github/workflows/evals/README.md +++ b/.github/workflows/evals/README.md @@ -80,15 +80,6 @@ is failing at eval time. ## Run locally -The deterministic scanner lookup tests need Python 3, Bash, and jq, but no -credentials or network access. They exercise author metadata, complete empty -results, filtered candidates, CLI-proxy arrays and MCP text envelopes, -issue and pull-request routing, incomplete responses, and tool failures: - -```bash -python3 .github/workflows/evals/test_ci_failure_scan_search.py -``` - The deterministic fixer tests need Python 3, Bash, jq, and Node with the eval dependencies installed (`npm ci --prefix .github/workflows/evals`), but no credentials or network access during testing. They exercise the shared intake diff --git a/.github/workflows/evals/ci-failure-scan.eval.yaml b/.github/workflows/evals/ci-failure-scan.eval.yaml index bda4acf7a33376..8c10f9cbada284 100644 --- a/.github/workflows/evals/ci-failure-scan.eval.yaml +++ b/.github/workflows/evals/ci-failure-scan.eval.yaml @@ -25,7 +25,7 @@ environment: - GITHUB_READ_ONLY=1 - --env - GITHUB_TOOLSETS=pull_requests,repos,issues,search - - ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e + - ghcr.io/github/github-mcp-server:v1.5.0@sha256:e25564dccc9110a70a77b9df560cbde11aa392fcb5f08b9abe5c4ebc6d146ea4 stimuli: - name: live-kbe-safe-output @@ -48,7 +48,7 @@ stimuli: 1. Find a recent failed build on `main` of an outer-loop pipeline via the dnceng-public AzDO REST API. It is anonymous and needs no auth, so stay on `_apis/build/...`. Start with `runtime-extra-platforms`, which is definition 154, and fall back to other outer-loop defs if needed. `curl -fsSL "https://dev.azure.com/dnceng-public/public/_apis/build/builds?definitions=154&branchName=refs/heads/main&statusFilter=completed&resultFilter=failed&\$top=5&api-version=7.1"` 2. Open that build's timeline at `/_apis/build/builds/{id}/timeline?api-version=7.1`, pick a failed leg, and read its log at `/_apis/build/builds/{id}/logs/{logId}?api-version=7.1` to extract a concrete error or assertion signature. Save the leg log you extract from as `failure.log` under /tmp and grep it for your signature to get the hit count. - 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. This eval exposes native GitHub MCP tools rather than production's CLI proxy. Use `search_issues` with `owner: "dotnet"`, `repo: "runtime"`, and `fields: ["number", "title", "state", "user", "labels", "html_url"]`. Search open and recently closed issues by your signature text following the shared search variations. Inspect complete results without discarding author metadata, filtered markers, errors, or result counts. A failed, filtered, malformed, or incomplete lookup is inconclusive, not an empty result. Do not emit a KBE from an inconclusive lookup or when a matching issue exists; report the reason instead. Do not use `gh` or direct GitHub API reads as a fallback. + 3. Before filing, check whether an equivalent KBE already exists so you would not duplicate it. Run `gh search issues 'in:title [ci-scan]' --repo dotnet/runtime --state open` and also search by your signature text. Assume you would file only if none matches. Then **emit the create-issue safe-output you would file, as a single markdown document at `./out/kbe.md`**, in exactly this shape. @@ -110,11 +110,8 @@ stimuli: name: searched-existing-kbe config: required: - - name: '(^|[-_.])search_issues$' - args: - owner: '^dotnet$' - repo: '^runtime$' - pattern: '"fields"\s*:\s*\[(?=[^\]]*"number")(?=[^\]]*"title")(?=[^\]]*"state")(?=[^\]]*"user")(?=[^\]]*"labels")(?=[^\]]*"html_url")' + - name: '^(bash|powershell)$' + command: 'gh (search|issue)' - type: prompt name: kbe-format-conformant diff --git a/.github/workflows/evals/test_ci_failure_scan_search.py b/.github/workflows/evals/test_ci_failure_scan_search.py deleted file mode 100644 index a9b8d92c2703fb..00000000000000 --- a/.github/workflows/evals/test_ci_failure_scan_search.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python3 - -import json -import os -from pathlib import Path -import subprocess -import tempfile -import unittest - - -WORKFLOWS = Path(__file__).resolve().parents[1] -SEARCH_SCRIPT = WORKFLOWS / "shared/search-kbe.sh" - - -def issue(number=42, **overrides): - return { - "number": number, - "title": "[ci-scan] Test failure", - "state": "open", - "user": {"login": "github-actions[bot]", "type": "Bot"}, - "labels": [{"name": "Known Build Error"}], - "html_url": f"https://github.com/dotnet/runtime/issues/{number}", - **overrides, - } - - -def response(items=None, **overrides): - items = [] if items is None else items - return { - "items": items, - "total_count": len(items), - "incomplete_results": False, - **overrides, - } - - -class SearchTests(unittest.TestCase): - def run_search(self, value, *, fail=False, raw=False, status="candidates", - kind="issues", query='is:issue is:open "Missing BBF_PROF_WEIGHT flag"'): - with tempfile.TemporaryDirectory() as directory: - temp = Path(directory) - fixture = value if raw else json.dumps(value) - (temp / "fixture").write_text(fixture) - output = temp / "lookup" - output.mkdir() - (output / "summary.json").write_text('{"status":"no_match"}') - env = { - **os.environ, - "TEST_DIRECTORY": directory, - "TEST_FAIL": "1" if fail else "0", - } - mock = """ -github() { - printf '%s\\0' "$@" > "$TEST_DIRECTORY/arguments" - cat > "$TEST_DIRECTORY/request.json" - cat "$TEST_DIRECTORY/fixture" - if [ "$TEST_FAIL" = "1" ]; then - echo "Simulated search failure" >&2 - return 1 - fi -} -""" - result = subprocess.run( - ["bash", "-c", mock + SEARCH_SCRIPT.read_text(), "search-kbe", - kind, query, str(output)], - env=env, capture_output=True, text=True, - ) - self.assertEqual( - (temp / "arguments").read_text().split("\0")[:-1], - ["search_issues" if kind == "issues" else "search_pull_requests", "."], - ) - request = json.loads((temp / "request.json").read_text()) - self.assertEqual(request["owner"], "dotnet") - self.assertEqual(request["repo"], "runtime") - self.assertEqual( - request["fields"], - ["number", "title", "state", "user", "labels", "html_url"], - ) - self.assertEqual(request["perPage"], 100) - self.assertIn("repo:dotnet/runtime", request["query"]) - self.assertIn('"Missing BBF_PROF_WEIGHT flag"', request["query"]) - self.assertEqual((output / "response.json").read_text(), fixture) - summary = json.loads((output / "summary.json").read_text()) - self.assertEqual(summary["status"], status, result.stderr) - self.assertEqual(json.loads(result.stdout), summary) - if status == "blocked": - self.assertNotEqual(result.returncode, 0) - self.assertTrue(summary["reason"]) - self.assertTrue(result.stderr) - else: - self.assertEqual(result.returncode, 0, result.stderr) - return summary - - def test_preserves_candidate_metadata(self): - candidates = [issue(134292), issue(134169)] - summary = self.run_search(response(candidates)) - self.assertEqual(summary["items"], candidates) - self.assertEqual(summary["total_count"], 2) - - def test_only_complete_empty_response_is_no_match(self): - summary = self.run_search(response(), status="no_match") - self.assertEqual(summary["items"], []) - - def test_uses_pull_request_search_for_fix_queries(self): - for state in ("open", "closed"): - with self.subTest(state=state): - candidate = issue(state=state, html_url="https://github.com/dotnet/runtime/pull/42") - summary = self.run_search( - response([candidate]), kind="pull-requests", - query=f'is:pr is:{state} "Missing BBF_PROF_WEIGHT flag"', - ) - self.assertEqual(summary["items"], [candidate]) - - def test_filtered_results_never_become_no_match(self): - for value in ( - response(["[Filtered]"]), - response([issue(), "[Filtered]"]), - response([], filtered="[Filtered]"), - {"content": [{"type": "text", "text": "[Filtered]"}]}, - response([], warning="[DIFC-FILTERED]"), - ): - with self.subTest(value=value): - summary = self.run_search(value, status="blocked") - self.assertEqual(summary["reason"], "integrity-filtered candidate") - - def test_accepts_mcp_text_envelope(self): - for value in (response(), response([issue()])): - with self.subTest(value=value): - summary = self.run_search( - {"content": [{"type": "text", "text": json.dumps(value)}]}, - status="candidates" if value["items"] else "no_match", - ) - self.assertEqual(summary["items"], value["items"]) - - def test_accepts_cli_proxy_content_array(self): - for value in (response(), response([issue()])): - with self.subTest(value=value): - summary = self.run_search( - [value], - status="candidates" if value["items"] else "no_match", - ) - self.assertEqual(summary["items"], value["items"]) - self.run_search([response(["[Filtered]"])], status="blocked") - self.run_search([response(), response()], status="blocked") - - def test_blocks_incomplete_and_malformed_results(self): - for value in ( - response(incomplete_results=True), - response(total_count=1), - response([issue()], total_count=1000), - response([issue(user=None)]), - response([issue(user={})]), - response([issue(number="42")]), - response([issue(labels=None)]), - {"items": []}, - {"message": "API rate limit exceeded"}, - {"isError": True, "content": [{"type": "text", "text": "Request failed"}]}, - {"content": []}, - {"content": [{"type": "text", "text": "not JSON"}]}, - None, - [], - ): - with self.subTest(value=value): - self.run_search(value, status="blocked") - - def test_blocks_non_json_and_multiple_documents(self): - for value in ("", "Sign in", "{}\n{}", "null\n", "truncated {"): - with self.subTest(value=value): - self.run_search(value, raw=True, status="blocked") - - def test_tool_failure_cannot_reuse_previous_no_match(self): - self.run_search(response(), fail=True, status="blocked") - - def test_workflow_uses_guarded_searches(self): - workflow = (WORKFLOWS / "ci-failure-scan.md").read_text() - self.assertIn(" cli-proxy: true\n", workflow) - self.assertIn(".github/workflows/shared/search-kbe.sh", workflow) - tally = workflow.split("Recognized values:", 1)[1] - self.assertIn("`lookup incomplete, needs human review`", tally) - instructions = (WORKFLOWS / "shared/create-kbe.instructions.md").read_text() - self.assertIn("If any lookup returns a `[Filtered]` marker", instructions) - self.assertNotIn("linked-tracker: integrity-filtered", instructions) - - def test_eval_uses_production_github_server(self): - prefix = "# gh-aw-manifest: " - manifest = next( - line[len(prefix):] - for line in (WORKFLOWS / "ci-failure-scan.lock.yml").read_text().splitlines() - if line.startswith(prefix) - ) - image_prefix = "ghcr.io/github/github-mcp-server:" - production_image = next( - container["pinned_image"] - for container in json.loads(manifest)["containers"] - if container["image"].startswith(image_prefix) - ) - eval_image = next( - line.strip()[2:] - for line in (WORKFLOWS / "evals/ci-failure-scan.eval.yaml").read_text().splitlines() - if line.strip().startswith("- " + image_prefix) - ) - self.assertEqual(eval_image, production_image) - - def test_invalid_arguments_fail_before_lookup(self): - for args in ( - [], - [""], - ["issues", "", "/tmp/unused-kbe-search"], - ["issues", "query", ""], - ["invalid", "query", "/tmp/unused-kbe-search"], - ): - with self.subTest(args=args): - result = subprocess.run( - ["bash", str(SEARCH_SCRIPT), *args], - capture_output=True, text=True, - ) - self.assertNotEqual(result.returncode, 0) - self.assertIn("Usage:", result.stderr) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/shared/create-kbe.instructions.md b/.github/workflows/shared/create-kbe.instructions.md index 8a7d72dabc4705..ad700d1a3b1b55 100644 --- a/.github/workflows/shared/create-kbe.instructions.md +++ b/.github/workflows/shared/create-kbe.instructions.md @@ -44,48 +44,44 @@ these variations in order and inspect every returned candidate. Narrow overly broad queries instead of truncating results. GitHub best-match ranking can place noisier hits above the correct one. -For every `search_issues` or `search_pull_requests` call, include `user` in the requested -`fields`, even when the author is not otherwise needed. The integrity gateway -uses `user.login` to recognize trusted bots before filtering search results. +Use `search_issues` for issues and `search_pull_requests` for open or merged +fix PRs. For every call, specify `owner: "dotnet"`, `repo: "runtime"`, and +`fields: ["number", "title", "state", "user", "labels", "html_url"]`. +Never omit `user`, even when the author is not otherwise needed. The integrity +gateway uses `user.login` to recognize trusted bots before filtering results. ### Preserve lookup results -When using the workflow's `github` CLI proxy, run the checked-in helper for -every issue or PR search in this file, including open, recently closed, and -merged searches. Run it from the repository root, using a separate evidence -directory for each query: - -```bash -bash .github/workflows/shared/search-kbe.sh issues \ - 'is:issue is:open label:"Known Build Error" "distinctive assertion text"' \ - /tmp/gh-aw/agent/kbe-search/signature-1-open -``` - -Use `pull-requests` instead of `issues` for open or merged fix-PR queries. -The two MCP search tools enforce different issue-type qualifiers. - -The helper supplies the author and label fields, preserves `request.json` and -the complete `response.json`, and emits `summary.json`. It continues to use the -integrity-gated proxy, not `gh` or a direct GitHub API request. - -- `candidates` means inspect the returned issues with the full candidate - verification below. A search hit alone does not prove a duplicate. -- `no_match` means only this query returned a complete, valid empty result. - Continue the remaining required search variations before deciding to file. -- `blocked` or any nonzero exit means this lookup is inconclusive, not empty. - Do not emit a KBE for that signature. Narrow an overbroad query and rerun; - otherwise record `skipped: integrity-filtered candidate, needs human review` - for a filtered response or `skipped: lookup incomplete, needs human review` - for a failed, malformed, or incomplete response. +Apply these rules to every issue and PR search below, including open, +recently closed, merged, and unlabeled-tracker searches. Use the available +integrity-gated GitHub tools and preserve each query and its complete response. Never pipe lookup output through `grep`, `head`, or a projection that discards filtered markers, errors, author metadata, or result counts. Never replace a -failed lookup with an empty array or bypass integrity filtering. - -Callers using native MCP tools instead of the CLI proxy must request -`fields: ["number", "title", "state", "user", "labels", "html_url"]` and apply -the same result checks before interpreting the response. A filtered candidate -body or comments read is also inconclusive, not evidence of a different failure. +failed lookup with an empty array, or bypass integrity filtering with `gh` +or direct GitHub API reads. + +Before recording that a query has no match, verify that the tool succeeded, +the response is valid, and `incomplete_results` is `false`. If `total_count` +exceeds the returned candidate count, fetch the remaining pages or narrow an +overbroad query and rerun it. Do not treat an uninspected page as empty. +Missing or inconsistent counts make the response inconclusive. + +- A successful, complete response with zero candidates means only this query + has no match. Continue all remaining search variations before deciding to file. +- For a nonempty response, inspect every returned candidate using the full + candidate verification below. A search hit alone does not prove a duplicate. +- A failed, malformed, or incomplete response, or a candidate missing requested + metadata, is inconclusive. Do not emit a KBE for that signature unless the + lookup succeeds on retry. Otherwise record + `skipped: lookup incomplete, needs human review`. + +Any `[Filtered]` or `[DIFC-FILTERED]` marker makes the lookup inconclusive, even +when visible candidates do not match. Record +`skipped: integrity-filtered candidate, needs human review` and do not file. +Failed or filtered candidate body and comments reads also stop filing. +Record the corresponding incomplete or filtered skip reason. An unreadable +candidate is not evidence of a different failure. ### Search variations @@ -154,8 +150,8 @@ If two candidate KBEs share more than 70% of their `ErrorMessage` / `ErrorPattern` tokens, do **not** guess: record `skipped: ambiguous dup #/#, needs human review` and stop. -If any lookup returns a `[Filtered]` marker, treat it as a possible -existing-KBE hit and record +If any lookup returns a `[Filtered]` or `[DIFC-FILTERED]` marker, treat it as +a possible existing-KBE hit and record `skipped: integrity-filtered candidate, needs human review` instead of creating a fresh KBE. @@ -296,10 +292,10 @@ hit, record `existing-PR #`. ### Integrity-filtered PR candidate -If any PR search above returns a `[Filtered]` marker for a candidate whose -title, source symbol, or assertion slice overlaps the failing signature, do -**not** assume no fix exists and file a fresh KBE. The filter hides a real PR -you are not permitted to read, and it may already handle this failure. Record +If any PR search above returns a `[Filtered]` or `[DIFC-FILTERED]` marker, do +**not** assume no fix exists and file a fresh KBE. Do not require visible +title, source-symbol, or assertion overlap before stopping, since filtering +may hide those fields. The hidden PR may already handle this failure. Record `skipped: integrity-filtered candidate, needs human review` and stop for this signature. A human can confirm whether the hidden PR fixes the failure; filing a duplicate KBE that is immediately closed as "fixed by" the hidden PR is a diff --git a/.github/workflows/shared/search-kbe.sh b/.github/workflows/shared/search-kbe.sh deleted file mode 100644 index 9cdac678679ad6..00000000000000 --- a/.github/workflows/shared/search-kbe.sh +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -usage="Usage: search-kbe.sh " -if [ "$#" -ne 3 ] || [ -z "$2" ] || [ -z "$3" ]; then - echo "$usage" >&2 - exit 1 -fi - -case "$1" in - issues) tool="search_issues" ;; - pull-requests) tool="search_pull_requests" ;; - *) echo "$usage" >&2; exit 1 ;; -esac - -directory="$3" -mkdir -p "$directory" -jq -n '{status: "blocked", reason: "lookup did not complete"}' > "$directory/summary.json" -jq -n --arg query "repo:dotnet/runtime $2" '{ - owner: "dotnet", - repo: "runtime", - query: $query, - fields: ["number", "title", "state", "user", "labels", "html_url"], - perPage: 100 -}' > "$directory/request.json" - -if ! github "$tool" . < "$directory/request.json" > "$directory/response.json"; then - echo "KBE lookup failed; do not create an issue from this search." >&2 - cat "$directory/summary.json" - exit 1 -fi - -if ! jq -s ' - def filtered: - any(.. | strings; test("\\[Filtered\\]|\\[DIFC-FILTERED\\]"; "i")); - def blocked($reason): - {status: "blocked", reason: $reason}; - def nonempty_string: - type == "string" and length > 0; - def candidate: - type == "object" and - (.number | type == "number" and . > 0 and floor == .) and - (.title | nonempty_string) and - (.state == "open" or .state == "closed") and - (.user.login | nonempty_string) and - (.labels | type == "array") and - (.html_url | nonempty_string); - - if length != 1 then - blocked("expected one JSON response") - elif filtered then - blocked("integrity-filtered candidate") - else - .[0] | - if type == "array" and length == 1 then .[0] else . end | - if type != "object" then error("expected an object") - elif .isError == true or has("error") then error("tool reported an error") - elif has("content") then - if (.content | type == "array" and length == 1) and - .content[0].type == "text" then - .content[0].text | fromjson - else error("expected one MCP text result") - end - else . - end | - if filtered then - blocked("integrity-filtered candidate") - elif type != "object" or .isError == true or has("error") then - blocked("invalid search result") - elif .incomplete_results != false or (.items | type != "array") or - (.total_count | type != "number") then - blocked("incomplete or invalid search result") - elif .total_count != (.items | length) then - blocked("not all candidates returned; narrow the query") - elif all(.items[]; candidate) | not then - blocked("candidate metadata is missing or invalid") - else - { - status: (if .total_count == 0 then "no_match" else "candidates" end), - total_count, - items - } - end - end -' "$directory/response.json" > "$directory/summary.json"; then - jq -n '{status: "blocked", reason: "malformed search response"}' > "$directory/summary.json" -fi - -cat "$directory/summary.json" -if ! jq -e '.status == "no_match" or .status == "candidates"' "$directory/summary.json" > /dev/null; then - echo "KBE lookup is inconclusive; do not create an issue from this search." >&2 - exit 1 -fi From 2555ac9780af49bd159e450f642e8403dddd01c8 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 22 Sep 2026 15:48:40 +0200 Subject: [PATCH 3/5] Keep KBE deduplication instructions focused Preserve lookup evidence and follow duplicate links to the original KBE. Respect caller tool policies and avoid introducing a skip reason that scanner feedback does not recognize. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 853fa924-7d9d-4d5b-9990-f7ac0d9d5e88 --- .../shared/create-kbe.instructions.md | 61 +++++++------------ 1 file changed, 21 insertions(+), 40 deletions(-) diff --git a/.github/workflows/shared/create-kbe.instructions.md b/.github/workflows/shared/create-kbe.instructions.md index ad700d1a3b1b55..254624f4fa5481 100644 --- a/.github/workflows/shared/create-kbe.instructions.md +++ b/.github/workflows/shared/create-kbe.instructions.md @@ -44,46 +44,21 @@ these variations in order and inspect every returned candidate. Narrow overly broad queries instead of truncating results. GitHub best-match ranking can place noisier hits above the correct one. -Use `search_issues` for issues and `search_pull_requests` for open or merged -fix PRs. For every call, specify `owner: "dotnet"`, `repo: "runtime"`, and -`fields: ["number", "title", "state", "user", "labels", "html_url"]`. -Never omit `user`, even when the author is not otherwise needed. The integrity -gateway uses `user.login` to recognize trusted bots before filtering results. - -### Preserve lookup results - -Apply these rules to every issue and PR search below, including open, -recently closed, merged, and unlabeled-tracker searches. Use the available -integrity-gated GitHub tools and preserve each query and its complete response. - -Never pipe lookup output through `grep`, `head`, or a projection that discards -filtered markers, errors, author metadata, or result counts. Never replace a -failed lookup with an empty array, or bypass integrity filtering with `gh` -or direct GitHub API reads. - -Before recording that a query has no match, verify that the tool succeeded, -the response is valid, and `incomplete_results` is `false`. If `total_count` -exceeds the returned candidate count, fetch the remaining pages or narrow an -overbroad query and rerun it. Do not treat an uninspected page as empty. -Missing or inconsistent counts make the response inconclusive. - -- A successful, complete response with zero candidates means only this query - has no match. Continue all remaining search variations before deciding to file. -- For a nonempty response, inspect every returned candidate using the full - candidate verification below. A search hit alone does not prove a duplicate. -- A failed, malformed, or incomplete response, or a candidate missing requested - metadata, is inconclusive. Do not emit a KBE for that signature unless the - lookup succeeds on retry. Otherwise record - `skipped: lookup incomplete, needs human review`. - -Any `[Filtered]` or `[DIFC-FILTERED]` marker makes the lookup inconclusive, even -when visible candidates do not match. Record -`skipped: integrity-filtered candidate, needs human review` and do not file. -Failed or filtered candidate body and comments reads also stop filing. -Record the corresponding incomplete or filtered skip reason. An unreadable -candidate is not evidence of a different failure. - -### Search variations +Use the lookup tools required by the caller; this shared file does not change +its tool policy. For GitHub MCP lookups, use `search_issues` for issues and +`search_pull_requests` for PRs. When supplying a `fields` filter, include +`user` and `labels` alongside `number`, `title`, and `state`; otherwise retain +the full response. The integrity gateway uses `user.login` to recognize +trusted bots before filtering results. + +Preserve the complete lookup response before extracting candidate numbers or +titles. Never pipe it through `grep`, `head`, or a projection that discards +filtered markers, errors, author metadata, or result counts. A failed, +malformed, incomplete, or unreadable lookup is not an empty result. Report the +retrieval failure to the caller and do not create a KBE for that signature +while the lookup remains inconclusive. +These rules also apply to candidate body and comments reads. Do not switch +retrieval paths to work around an integrity-filtered or denied read. 1. Full `[FAIL]` line. 2. Assertion text. @@ -193,6 +168,12 @@ search misses, also search recently closed KBEs with the same pair: Apply the closed-candidate timing and full candidate-verification rules below to any pair match. +If a candidate's body or comments identify it as a duplicate, read the linked +original through the same permitted tools and apply the full candidate +verification to it. Reuse a matching open KBE rather than filing a recurrence +against its closed duplicate. A duplicate closure does not establish that the +failure was fixed. + On a closed-candidate hit, compare the failing AzDO build's `finishTime` (read it from the build metadata, not the queue time) against the issue's `closed_at`: From 5698091056119f04aaff278166e53ff5f2805c37 Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 22 Sep 2026 18:32:14 +0200 Subject: [PATCH 4/5] Resolve canonical KBE identities before recurrence checks Follow duplicate chains to the original issue and stop on cycles or inconclusive reads. Use the verified original for closure timing and count it only once in recurring-signature checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 853fa924-7d9d-4d5b-9990-f7ac0d9d5e88 --- .../shared/create-kbe.instructions.md | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/shared/create-kbe.instructions.md b/.github/workflows/shared/create-kbe.instructions.md index 254624f4fa5481..c36f2e7a65fc81 100644 --- a/.github/workflows/shared/create-kbe.instructions.md +++ b/.github/workflows/shared/create-kbe.instructions.md @@ -168,14 +168,21 @@ search misses, also search recently closed KBEs with the same pair: Apply the closed-candidate timing and full candidate-verification rules below to any pair match. -If a candidate's body or comments identify it as a duplicate, read the linked -original through the same permitted tools and apply the full candidate -verification to it. Reuse a matching open KBE rather than filing a recurrence -against its closed duplicate. A duplicate closure does not establish that the -failure was fixed. - -On a closed-candidate hit, compare the failing AzDO build's `finishTime` (read -it from the build metadata, not the queue time) against the issue's `closed_at`: +If a candidate is identified as a duplicate, follow the linked issues through +the same permitted tools until reaching an original that is not itself a +duplicate. Track visited issues. If a link is missing or ambiguous, the chain +is cyclic, or any read is inconclusive, report the incomplete lookup and do +not file. + +Apply the full candidate verification to the original. Use only its issue +number, state, and `closed_at` for the timing and recurring-signature rules +below, counting each original once. Reuse a matching open KBE rather than +filing a recurrence against its closed duplicate. A duplicate closure does +not establish that the failure was fixed. + +On a verified closed-original hit, compare the failing AzDO build's `finishTime` +(read it from the build metadata, not the queue time) against that original's +`closed_at`: - Closed **after** the failing build finished, or closed within the last 7 days: the failure is already handled or under active triage. Record From 67bfb5263b25733fa3cef260b5eb89bb4574486b Mon Sep 17 00:00:00 2001 From: Milos Kotlar Date: Tue, 29 Sep 2026 11:51:00 +0200 Subject: [PATCH 5/5] Require comment reads when resolving KBE duplicates Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 06bd27cd-bf15-4fd8-9f4d-911464827963 --- .github/workflows/shared/create-kbe.instructions.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/shared/create-kbe.instructions.md b/.github/workflows/shared/create-kbe.instructions.md index c36f2e7a65fc81..28796fab351648 100644 --- a/.github/workflows/shared/create-kbe.instructions.md +++ b/.github/workflows/shared/create-kbe.instructions.md @@ -168,7 +168,9 @@ search misses, also search recently closed KBEs with the same pair: Apply the closed-candidate timing and full candidate-verification rules below to any pair match. -If a candidate is identified as a duplicate, follow the linked issues through +Read each candidate's body and comments before applying the recurrence rules. +If a candidate is identified as a duplicate, follow only explicit duplicate +links and read each linked issue's body and comments through the same permitted tools until reaching an original that is not itself a duplicate. Track visited issues. If a link is missing or ambiguous, the chain is cyclic, or any read is inconclusive, report the incomplete lookup and do