Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 82 additions & 32 deletions .github/workflows/discussion-draft.lock.yml

Large diffs are not rendered by default.

90 changes: 88 additions & 2 deletions .github/workflows/discussion-draft.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,68 @@ permissions:
copilot-requests: write
imports:
- shared/read-only-analysis.md
safe-outputs:
report-missing-tool: false
noop:
# One trusted scan signal plus the two existing model no-ops.
max: 3
report-as-issue: false
threat-detection:
continue-on-error: false
report-as-issue: false
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
with:
ref: ${{ github.sha }}
persist-credentials: false
- name: Download candidate for detection
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: typechat-discussion-candidate-${{ github.run_attempt }}
path: ${{ runner.temp }}/typechat-detection-candidate
Comment thread
robgruen marked this conversation as resolved.
Resolved
- name: Stage the exact candidate for detection
id: stage_candidate
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
TYPECHAT_DETECTION_DIRECTORY: /tmp/gh-aw/threat-detection
with:
github-token: ${{ github.token }}
retries: 0
script: |
const path = require("node:path");
const { pathToFileURL } = require("node:url");
const helper = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, "tools", "discussion-draft.mjs")).href);
const directory = path.join(process.env.RUNNER_TEMP, "typechat-detection-candidate");
const hash = helper.stageCandidateForDetection(directory, process.env.TYPECHAT_DETECTION_DIRECTORY);
core.setOutput("sha256", hash);
post-steps:
- name: Record the candidate scan
id: candidate_scan
if: success() && steps.stage_candidate.outcome == 'success' && steps.detection_agentic_execution.outcome == 'success'
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
TYPECHAT_CANDIDATE_SHA256: ${{ steps.stage_candidate.outputs.sha256 }}
TYPECHAT_SCAN_OUTCOME: ${{ steps.detection_agentic_execution.outcome }}
with:
github-token: ${{ github.token }}
retries: 0
script: |
const path = require("node:path");
const { pathToFileURL } = require("node:url");
const helper = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, "tools", "discussion-draft.mjs")).href);
const scan = helper.createCandidateScan(process.env.TYPECHAT_CANDIDATE_SHA256, process.env.TYPECHAT_SCAN_OUTCOME, context.runId, process.env.GITHUB_RUN_ATTEMPT);
const directory = path.join(process.env.RUNNER_TEMP, "typechat-candidate-scan");
helper.writeJson(directory, "candidate-scan.json", scan);
core.setOutput("path", path.join(directory, "candidate-scan.json"));
- name: Upload candidate scan binding
if: success() && steps.candidate_scan.outcome == 'success'
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: typechat-discussion-scan-${{ github.run_attempt }}
path: ${{ steps.candidate_scan.outputs.path }}
if-no-files-found: error
overwrite: false
retention-days: 7
checkout: false
network: defaults
timeout-minutes: 60
Expand Down Expand Up @@ -88,7 +150,10 @@ jobs:
seal:
name: Seal discussion draft
needs: [prepare, agent, detection]
if: needs.agent.result == 'success' && needs.detection.result == 'success'
if: >-
needs.agent.result == 'success' && needs.detection.result == 'success' &&
needs.detection.outputs.run_detection == 'true' &&
needs.detection.outputs.detection_conclusion == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
Expand All @@ -112,6 +177,11 @@ jobs:
with:
name: typechat-discussion-candidate-${{ github.run_attempt }}
path: ${{ runner.temp }}/typechat-seal-candidate
- name: Download candidate scan binding
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
name: typechat-discussion-scan-${{ github.run_attempt }}
path: ${{ runner.temp }}/typechat-seal-scan
- name: Validate and seal the draft
id: seal
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
Expand All @@ -123,7 +193,8 @@ jobs:
const { pathToFileURL } = require("node:url");
const helper = await import(pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, "tools", "discussion-draft.mjs")).href);
const prepared = helper.readJson(path.join(process.env.RUNNER_TEMP, "typechat-seal-context"), "context.json", helper.MAX_CONTEXT_BYTES);
const candidate = helper.readJson(path.join(process.env.RUNNER_TEMP, "typechat-seal-candidate"), "discussion-draft.json", helper.MAX_DRAFT_BYTES);
const scan = helper.readJson(path.join(process.env.RUNNER_TEMP, "typechat-seal-scan"), "candidate-scan.json", 1024);
const candidate = helper.readDetectedDraft(path.join(process.env.RUNNER_TEMP, "typechat-seal-candidate"), scan, context.runId, process.env.GITHUB_RUN_ATTEMPT);
const bundle = await helper.sealDraft(github, context, process.env, prepared, candidate);
const directory = path.join(process.env.RUNNER_TEMP, "typechat-sealed-draft");
helper.writeJson(directory, "sealed-draft.json", bundle);
Expand Down Expand Up @@ -158,6 +229,21 @@ steps:
artifact-ids: ${{ needs.prepare.outputs.context_artifact_id }}
merge-multiple: true
path: .typechat-draft
pre-agent-steps:
# The pinned detection guard only sees native output types, not file artifacts.
- name: Require candidate threat detection
id: require_candidate_detection
env:
GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
shell: node {0}
run: |
const fs = require("node:fs");
const path = require("node:path");
fs.mkdirSync(path.dirname(process.env.GH_AW_SAFE_OUTPUTS), { recursive: true });
fs.appendFileSync(process.env.GH_AW_SAFE_OUTPUTS, JSON.stringify({
type: "noop",
message: "This read-only workflow requires threat detection of its candidate artifact before sealing."
}) + "\n");
post-steps:
- name: Verify candidate draft file
if: success()
Expand Down
37 changes: 31 additions & 6 deletions AUTOMATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -466,6 +466,8 @@ The workflow check also runs the contract tests in `tools/` (`npm ci --ignore-sc
by `npm test`), including checks that the runtime pin matches the compilation
command, generated drafting jobs cannot write to issues or pull requests, and
the separate publisher has only its target-specific comment permission.
The discussion tests execute the compiled detection guard with Bash (Git Bash
on Windows), including a candidate with no model-emitted safe outputs or patch.

The assessment itself remains report-only and does not change existing
maintenance and deployment workflows. The separate
Expand Down Expand Up @@ -516,11 +518,27 @@ recorded requirements, suggested criteria, and a readiness recommendation.
Thread context is bounded at 1 MiB and 1,000 comments, including replies;
incomplete or oversized context is an error, not silent truncation.

After successful agent execution and threat detection, a separate trusted job
validates the structured output and downloads the **original** context again.
It checks for changes, renders the comment, and seals its exact bytes and
provenance. Model output cannot choose the publishing target, owner, workflow
identity, or approval record.
Before model execution, a trusted step adds a read-only no-op to the native
safe-output stream. This makes the pinned runtime's detection guard run even
when the model writes only the candidate file and emits no safe outputs or patch.
The no-op limit reserves one slot for that signal in addition to the two existing
model no-ops; no-op issue reporting remains disabled.

The detector downloads the attempt-specific, immutable candidate artifact.
Trusted code adds its complete JSON text to the detector's agent-output input,
preserving any native safe outputs. This includes file-only candidates, not just
text the model sends to safe-output tools. Sealing requires an executed,
successful scan with a successful verdict; skipped scans, warnings, and failures
cannot authorize sealing.

After detection, a separate trusted job downloads that same candidate artifact
and checks its SHA-256 against an immutable scan-binding artifact from the same
run and attempt. The binding records the bytes staged for scanning and the
detector step's successful execution; the job's final verdict must also succeed.
The sealing job validates the structured output and downloads the **original**
context again. It checks for changes, renders the comment, and seals its exact
bytes and provenance. Model output cannot choose the publishing target, owner,
workflow identity, or approval record.

The [publisher](./.github/workflows/publish-discussion-draft.yml) runs no model.
Its Issue job has `issues: write`; its Discussion job has `discussions: write`.
Expand Down Expand Up @@ -548,7 +566,7 @@ Publishing invokes no model. A **1,000-credit implementation budget** is a
future owner-approved implementation-stage setting only; this pilot enables
neither that spending nor a build command.

Context, candidate, and sealed draft artifacts are retained for seven days.
Context, candidate, scan-binding, and sealed draft artifacts are retained for seven days.
Approval validity ends at the earlier of artifact expiry or seven days after
artifact creation. Artifacts and Actions summaries follow repository visibility;
they are not a private channel and must not contain secrets or sensitive data.
Expand All @@ -559,6 +577,13 @@ must permit that retention; an upload or retention check failure prevents the
write. The resulting receipt is also retained for 30 days. Do not delete
publication runs or intent artifacts while the associated draft remains valid.

Intent names are keyed by the sealed draft's artifact ID, not the publishing run
or attempt; the intent document still records both. Each history check makes one
repository-artifact request filtered by that exact name and reads at most two
matches. Only one match can be the current upload, so a second match always blocks
another write. Missing, malformed, or incomplete history fails closed. This avoids
enumerating unrelated workflow runs while finding intents across runs and reruns.

An existing exact comment from the verified Actions bot makes a retry a no-op.
A prior intent without a confirmed matching comment blocks another write,
including reruns and new publish commands for that artifact. A timeout may mean
Expand Down
120 changes: 65 additions & 55 deletions tools/discussion-draft.mjs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { isUtf8 } from "node:buffer";
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
Expand Down Expand Up @@ -561,15 +562,21 @@ export async function sealDraft(github, context, env, prepared, candidate, clock
return bundle;
}

export function readJson(directory, filename, maximum) {
function readArtifact(directory, filename, maximum) {
const entries = fs.readdirSync(directory);
requireValue(entries.length === 1 && entries[0] === filename, "Unexpected files in the downloaded artifact.");
const file = path.join(directory, filename);
const info = fs.lstatSync(file);
requireValue(info.isFile() && !info.isSymbolicLink() && info.size > 0 && info.size <= maximum,
"The artifact must contain one bounded regular JSON file.");
const bytes = fs.readFileSync(file);
requireValue(isUtf8(bytes), "The artifact must contain UTF-8 JSON.");
return bytes.toString("utf8");
}

function parseJson(text) {
try {
return JSON.parse(fs.readFileSync(file, "utf8"));
return JSON.parse(text);
}
catch (error) {
if (error instanceof SyntaxError) {
Expand All @@ -579,6 +586,41 @@ export function readJson(directory, filename, maximum) {
}
}

export function readJson(directory, filename, maximum) {
return parseJson(readArtifact(directory, filename, maximum));
}

export function stageCandidateForDetection(directory, detectionDirectory) {
const text = readArtifact(directory, "discussion-draft.json", MAX_DRAFT_BYTES);
validateDraft(parseJson(text));
const outputPath = path.join(detectionDirectory, "agent_output.json");
const output = fs.existsSync(outputPath) ? parseJson(fs.readFileSync(outputPath, "utf8")) : { items: [] };
requireValue(output && Array.isArray(output.items), "The detector's agent output is malformed.");
output.items.push({ type: "discussion-draft", content: text });
fs.mkdirSync(detectionDirectory, { recursive: true });
fs.writeFileSync(outputPath, JSON.stringify(output) + "\n");
return digest(text);
}

export function createCandidateScan(hash, outcome, runId, attempt) {
requireValue(typeof hash === "string" && SHA256.test(hash), "Missing candidate threat-scan hash.");
requireValue(outcome === "success", "The candidate threat scan did not execute successfully.");
return {
version: 1, sha256: hash, outcome,
runId: integer(Number(runId), "Invalid candidate scan run ID."),
attempt: integer(Number(attempt), "Invalid candidate scan attempt."),
};
}

export function readDetectedDraft(directory, scan, runId, attempt) {
exactKeys(scan, ["version", "sha256", "outcome", "runId", "attempt"], "Missing or malformed candidate threat-scan binding.");
requireValue(equal(scan, createCandidateScan(scan.sha256, scan.outcome, runId, attempt)),
"The candidate threat-scan binding belongs to a different run or attempt.");
const text = readArtifact(directory, "discussion-draft.json", MAX_DRAFT_BYTES);
requireValue(digest(text) === scan.sha256, "The candidate differs from the threat-scanned artifact. Request a new draft.");
return parseJson(text);
}

export function writeJson(directory, filename, value) {
fs.mkdirSync(directory, { recursive: true });
fs.writeFileSync(path.join(directory, filename), JSON.stringify(value, null, 2) + "\n", { flag: "wx" });
Expand Down Expand Up @@ -664,60 +706,28 @@ export function publishedComment(thread, bundle) {
return matches[0];
}

async function listArtifacts(github, runId) {
const artifacts = [];
let total;
for (let page = 1; page <= 10; page++) {
const data = (await github.rest.actions.listWorkflowRunArtifacts({ ...API_REPO, run_id: runId, per_page: 100, page })).data;
requireValue(Array.isArray(data.artifacts) && Number.isSafeInteger(data.total_count) &&
data.total_count >= 0 && data.total_count <= 1000, "Artifact history is unavailable or exceeds the pilot limit.");
requireValue(total === undefined || total === data.total_count, "Artifact history changed during pagination.");
total = data.total_count;
artifacts.push(...data.artifacts);
if (artifacts.length === total) {
return artifacts;
}
requireValue(artifacts.length < total && data.artifacts.length === 100, "Artifact history is incomplete.");
}
throw new Error("Artifact history exceeds the pilot limit. Human inspection is required.");
}

async function priorIntent(github, publication, excludeId) {
const prefix = `${INTENT_PREFIX}${publication.artifact.id}-`;
const current = await listArtifacts(github, publication.request.runId);
const existing = current.find(artifact => artifact.id !== excludeId && artifact.name.startsWith(prefix));
if (existing) {
return { runId: publication.request.runId, artifactId: existing.id };
}
const name = `${INTENT_PREFIX}${publication.artifact.id}`;
// At most one result can be this run's new intent; any other match blocks publication.
const data = (await github.rest.actions.listArtifactsForRepo({ ...API_REPO, name, per_page: 2, page: 1 })).data;
requireValue(data && Array.isArray(data.artifacts) && Number.isSafeInteger(data.total_count) &&
data.total_count >= 0 && data.artifacts.length === Math.min(data.total_count, 2),
"Publication intent history is unavailable or incomplete.");
const seen = new Set();
let total;
for (let page = 1; page <= 10; page++) {
const data = (await github.rest.actions.listWorkflowRuns({
...API_REPO, workflow_id: path.posix.basename(PUBLISH_WORKFLOW), actor: OWNER.login,
created: `>=${publication.artifact.created_at}`, per_page: 100, page,
})).data;
requireValue(Array.isArray(data.workflow_runs) && Number.isSafeInteger(data.total_count) &&
data.total_count >= 0 && data.total_count <= 1000, "Publication history is unavailable or exceeds the pilot limit.");
requireValue(total === undefined || total === data.total_count, "Publication history changed during pagination.");
total = data.total_count;
for (const run of data.workflow_runs) {
integer(run.id, "Invalid publication history run ID.");
requireValue(!seen.has(run.id), "Publication history changed during pagination.");
seen.add(run.id);
if (run.id === publication.request.runId) {
continue;
}
const found = (await listArtifacts(github, run.id)).find(artifact => artifact.name.startsWith(prefix));
if (found) {
return { runId: run.id, artifactId: found.id };
}
}
if (seen.size === total) {
return null;
}
requireValue(seen.size < total && data.workflow_runs.length === 100, "Publication history is incomplete.");
for (const artifact of data.artifacts) {
requireValue(artifact && Number.isSafeInteger(artifact.id) && artifact.id > 0 &&
artifact.name === name && !seen.has(artifact.id) &&
artifact.workflow_run?.repository_id === REPOSITORY.id &&
Number.isSafeInteger(artifact.workflow_run.id) && artifact.workflow_run.id > 0,
"Publication intent history contains invalid or duplicate artifacts.");
seen.add(artifact.id);
}
const prior = data.artifacts.find(artifact => artifact.id !== excludeId);
if (prior) {
return { runId: prior.workflow_run.id, artifactId: prior.id };
}
throw new Error("Publication history exceeds the pilot limit. No comment will be posted.");
requireValue(excludeId === undefined || seen.has(excludeId), "The current publication intent is missing from artifact history.");
return null;
}

async function verifyBundleContext(github, publication, bundle) {
Expand Down Expand Up @@ -746,7 +756,7 @@ export async function checkPublication(github, context, env, bundle, clock = ()
requireValue(!prior, "A previous publication intent exists without a confirmed comment. Inspect that run; automatic retry is blocked.");
return {
shouldPost: true,
intentName: `${INTENT_PREFIX}${publication.artifact.id}-${publication.request.attempt}`,
intentName: `${INTENT_PREFIX}${publication.artifact.id}`,
intent: {
version: 1, artifactId: publication.artifact.id, bodyHash: bundle.bodyHash,
target: bundle.target, approvalComment: publication.request.commentId,
Expand All @@ -763,7 +773,7 @@ export async function publishDraft(github, context, env, bundle, intentId, clock
}
const intent = (await github.rest.actions.getArtifact({ ...API_REPO, artifact_id: integer(Number(intentId), "Missing durable publication intent.") })).data;
requireValue(intent.id === Number(intentId) &&
intent.name === `${INTENT_PREFIX}${publication.artifact.id}-${publication.request.attempt}` &&
intent.name === `${INTENT_PREFIX}${publication.artifact.id}` &&
intent.workflow_run?.id === publication.request.runId &&
intent.workflow_run.repository_id === REPOSITORY.id && intent.workflow_run.head_sha === context.sha &&
intent.expired === false &&
Expand Down
Loading
Loading