test: enable evidence-based AutoTest analysis

.github/workflows/e2e-autotest.yml

@@ -65,10 +65,28 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.scan.outputs.matrix }}
+      autotest_version: ${{ steps.autotest.outputs.version }}
     steps:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: 22
+
+      - name: Resolve AutoTest version for this run
+        id: autotest
+        shell: bash
+        run: |
+          version=$(npm view @vscjava/vscode-autotest@latest version)
+          if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then
+            echo "::error::Invalid AutoTest version returned by npm: $version"
+            exit 1
+          fi
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+          echo "Using AutoTest $version for all jobs in this run"
+
       - name: Resolve test plan matrix
         id: scan
         shell: bash
@@ -146,8 +164,10 @@ jobs:
           distribution: temurin
           java-version: 21
 
-      - name: Install autotest CLI
-        run: npm install -g @vscjava/vscode-autotest
+      - name: Install resolved AutoTest CLI
+        run: |
+          npm install -g "@vscjava/vscode-autotest@${{ needs.discover.outputs.autotest_version }}"
+          npm list -g @vscjava/vscode-autotest --depth=0
 
       - name: Setup virtual display (Linux)
         if: runner.os == 'Linux'
@@ -342,7 +362,12 @@ jobs:
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
         run: |
-          $autotestArgs = @("run", "test-plans/${{ matrix.plan }}.yaml")
+          $autotestArgs = @(
+            "run",
+            "test-plans/${{ matrix.plan }}.yaml",
+            "--analysis-mode",
+            "case"
+          )
           if (Test-Path vsix) {
             $vsixFiles = (Get-ChildItem vsix -Filter "*.vsix" |
               Sort-Object @{ Expression = { if ($_.Name -like "vscode-java-pack*") { 0 } else { 1 } } }, Name |
@@ -352,9 +377,10 @@ jobs:
 
           # PR/manual runs test the branch-built pack VSIX first, then any
           # supplied VSIX overrides such as a vscode-java Actions run artifact.
-          # LLM verification activates automatically when AZURE_OPENAI_* secrets
-          # are available (e.g. internal PRs); fork PRs without secret access
-          # simply skip the LLM step (LLMClient.isConfigured() returns false).
+          # Case mode always captures the evidence bundle. LLM verification and
+          # case analysis activate when AZURE_OPENAI_* secrets are available;
+          # runs without secret access retain the evidence and record the
+          # unavailable analysis without changing the test verdict.
           # Scheduled & manual runs default to --pre-release unless explicitly disabled.
           $isPR = "${{ github.event_name }}" -eq "pull_request"
           if (-not $isPR -and "${{ inputs.pre_release }}" -ne "false") {
@@ -374,8 +400,8 @@ jobs:
 
   # ── Job 3: Aggregate analysis ───────────────────────────
   analyze:
-    if: ${{ always() && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }}
-    needs: e2e-test
+    if: ${{ always() && needs.discover.result == 'success' && needs.e2e-test.result != 'skipped' && github.event_name != 'pull_request' && inputs.test_plan == '' }}
+    needs: [discover, e2e-test]
     runs-on: ubuntu-latest
 
     steps:
@@ -384,8 +410,10 @@ jobs:
         with:
           node-version: 22
 
-      - name: Install autotest CLI
-        run: npm install -g @vscjava/vscode-autotest
+      - name: Install resolved AutoTest CLI
+        run: |
+          npm install -g "@vscjava/vscode-autotest@${{ needs.discover.outputs.autotest_version }}"
+          npm list -g @vscjava/vscode-autotest --depth=0
 
       - name: Download all results
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
@@ -414,7 +442,7 @@ jobs:
           AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
           AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
           AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }}
-        run: autotest analyze test-results --output test-results
+        run: autotest analyze test-results --output test-results --analysis-mode case --report-only
 
       - name: Write Job Summary
         if: always()
@@ -428,5 +456,9 @@ jobs:
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-aggregate-summary
-          path: test-results/summary.md
+          path: |
+            test-results/summary.md
+            test-results/*/results.json
+            test-results/*/analysis/
+            test-results/*/evidence/manifest.json
           retention-days: 30

test-plans/java-webview-migration.yaml

@@ -178,7 +178,11 @@ steps:
 
   - id: "verify-jdk-dropdown"
     action: "wait 2 seconds"
-    verify: "screenshot shows the JDK dropdown expanded with at least one runtime entry"
+    verify: >
+      After waiting, the JDK dropdown is expanded with at least one visible
+      runtime entry. This checks the current state, not an opening transition.
+      If the dropdown was already expanded before waiting, unchanged screenshots
+      are expected and should pass, provided a runtime entry is still visible.
 
   - id: "close-java-runtime"
     action: "run command View: Close All Editors"