From 97128324e6746b5d836f380dd89b3ac14d25465b Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Mon, 14 Sep 2026 16:14:11 +0800 Subject: [PATCH 1/4] test: enable evidence-based autotest analysis --- .github/workflows/e2e-autotest.yml | 36 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/.github/workflows/e2e-autotest.yml b/.github/workflows/e2e-autotest.yml index 3ad27a0d..b89d69be 100644 --- a/.github/workflows/e2e-autotest.yml +++ b/.github/workflows/e2e-autotest.yml @@ -146,8 +146,10 @@ jobs: distribution: temurin java-version: 21 - - name: Install autotest CLI - run: npm install -g @vscjava/vscode-autotest + - name: Install latest autotest CLI + run: | + npm install -g @vscjava/vscode-autotest@latest + npm list -g @vscjava/vscode-autotest --depth=0 - name: Setup virtual display (Linux) if: runner.os == 'Linux' @@ -342,7 +344,12 @@ jobs: AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} run: | - $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml") + $autotestArgs = @( + "run", + "test-plans/${{ matrix.plan }}.yaml", + "--analysis-mode", + "case" + ) if (Test-Path vsix) { $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" | Sort-Object @{ Expression = { if ($_.Name -like "vscode-java-pack*") { 0 } else { 1 } } }, Name | @@ -352,9 +359,10 @@ jobs: # PR/manual runs test the branch-built pack VSIX first, then any # supplied VSIX overrides such as a vscode-java Actions run artifact. - # LLM verification activates automatically when AZURE_OPENAI_* secrets - # are available (e.g. internal PRs); fork PRs without secret access - # simply skip the LLM step (LLMClient.isConfigured() returns false). + # Case mode always captures the evidence bundle. LLM verification and + # case analysis activate when AZURE_OPENAI_* secrets are available; + # runs without secret access retain the evidence and record the + # unavailable analysis without changing the test verdict. # Scheduled & manual runs default to --pre-release unless explicitly disabled. $isPR = "${{ github.event_name }}" -eq "pull_request" if (-not $isPR -and "${{ inputs.pre_release }}" -ne "false") { @@ -374,7 +382,7 @@ jobs: # ── Job 3: Aggregate analysis ─────────────────────────── analyze: - if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }} + if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' }} needs: e2e-test runs-on: ubuntu-latest @@ -384,8 +392,10 @@ jobs: with: node-version: 22 - - name: Install autotest CLI - run: npm install -g @vscjava/vscode-autotest + - name: Install latest autotest CLI + run: | + npm install -g @vscjava/vscode-autotest@latest + npm list -g @vscjava/vscode-autotest --depth=0 - name: Download all results uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 @@ -414,7 +424,7 @@ jobs: AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} - run: autotest analyze test-results --output test-results + run: autotest analyze test-results --output test-results --analysis-mode case --report-only - name: Write Job Summary if: always() @@ -428,5 +438,9 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: e2e-aggregate-summary - path: test-results/summary.md + path: | + test-results/summary.md + test-results/*/results.json + test-results/*/analysis/ + test-results/*/evidence/manifest.json retention-days: 30 From e9b968383b81a0292a42a49ba6a79863b4b52709 Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Mon, 14 Sep 2026 16:38:09 +0800 Subject: [PATCH 2/4] test: limit aggregate analysis to full matrix --- .github/workflows/e2e-autotest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-autotest.yml b/.github/workflows/e2e-autotest.yml index b89d69be..0affd0e9 100644 --- a/.github/workflows/e2e-autotest.yml +++ b/.github/workflows/e2e-autotest.yml @@ -382,7 +382,7 @@ jobs: # ── Job 3: Aggregate analysis ─────────────────────────── analyze: - if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' }} + if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }} needs: e2e-test runs-on: ubuntu-latest From 2fd283a47a914e4c2dcebaf9e91b2510fa90ad7a Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Tue, 15 Sep 2026 10:17:27 +0800 Subject: [PATCH 3/4] test: clarify JDK dropdown state verification --- test-plans/java-webview-migration.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-plans/java-webview-migration.yaml b/test-plans/java-webview-migration.yaml index ec1eeeb1..2e09575f 100644 --- a/test-plans/java-webview-migration.yaml +++ b/test-plans/java-webview-migration.yaml @@ -178,7 +178,11 @@ steps: - id: "verify-jdk-dropdown" action: "wait 2 seconds" - verify: "screenshot shows the JDK dropdown expanded with at least one runtime entry" + verify: > + After waiting, the JDK dropdown is expanded with at least one visible + runtime entry. This checks the current state, not an opening transition. + If the dropdown was already expanded before waiting, unchanged screenshots + are expected and should pass, provided a runtime entry is still visible. - id: "close-java-runtime" action: "run command View: Close All Editors" From 144ad704ba5833acb65df354b34b685e446f905f Mon Sep 17 00:00:00 2001 From: wenytang-ms Date: Tue, 15 Sep 2026 11:25:56 +0800 Subject: [PATCH 4/4] test: share one AutoTest version across each workflow run --- .github/workflows/e2e-autotest.yml | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-autotest.yml b/.github/workflows/e2e-autotest.yml index 0affd0e9..7676ac01 100644 --- a/.github/workflows/e2e-autotest.yml +++ b/.github/workflows/e2e-autotest.yml @@ -65,10 +65,28 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.scan.outputs.matrix }} + autotest_version: ${{ steps.autotest.outputs.version }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + + - name: Resolve AutoTest version for this run + id: autotest + shell: bash + run: | + version=$(npm view @vscjava/vscode-autotest@latest version) + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then + echo "::error::Invalid AutoTest version returned by npm: $version" + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "Using AutoTest $version for all jobs in this run" + - name: Resolve test plan matrix id: scan shell: bash @@ -146,9 +164,9 @@ jobs: distribution: temurin java-version: 21 - - name: Install latest autotest CLI + - name: Install resolved AutoTest CLI run: | - npm install -g @vscjava/vscode-autotest@latest + npm install -g "@vscjava/vscode-autotest@${{ needs.discover.outputs.autotest_version }}" npm list -g @vscjava/vscode-autotest --depth=0 - name: Setup virtual display (Linux) @@ -382,8 +400,8 @@ jobs: # ── Job 3: Aggregate analysis ─────────────────────────── analyze: - if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }} - needs: e2e-test + if: ${{ always() && needs.discover.result == 'success' && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }} + needs: [discover, e2e-test] runs-on: ubuntu-latest steps: @@ -392,9 +410,9 @@ jobs: with: node-version: 22 - - name: Install latest autotest CLI + - name: Install resolved AutoTest CLI run: | - npm install -g @vscjava/vscode-autotest@latest + npm install -g "@vscjava/vscode-autotest@${{ needs.discover.outputs.autotest_version }}" npm list -g @vscjava/vscode-autotest --depth=0 - name: Download all results