Skip to content

Commit cd34ab6

Browse files
committed
ci(macrobenchmark): Resolve the device from the catalog before opening a session
The first run reached the API but matched zero devices: Real Device Access descriptors are not the ids used in .sauce/*.yml, which carry an OS version and a region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1). Fetch the catalog unfiltered and match client-side against both descriptor id and name, then open the session with the resolved concrete id rather than a pattern. On no match, print every available Android device so a naming mismatch is diagnosable from a single run -- the endpoint's own deviceId filter cannot distinguish "no devices on this account" from "your pattern matched nothing". Also run the no-match check before --probe-only returns. It previously came after, so the probe reported success and the real run failed on it minutes later.
1 parent f7c928e commit cd34ab6

2 files changed

Lines changed: 61 additions & 18 deletions

File tree

‎.github/workflows/integration-tests-macrobenchmark.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ on:
1919
device-name:
2020
description: 'Sauce device id or regex'
2121
required: false
22-
default: 'Google_Pixel_9_Pro_XL_15_real.*'
22+
default: 'Google_Pixel_9_Pro_XL'
2323
# Temporary scaffolding: workflow_dispatch cannot target a workflow that does not exist on
2424
# the default branch yet, so trigger on pushes to this branch while we validate the flow.
2525
# Remove before merge.
@@ -43,7 +43,7 @@ jobs:
4343
GRADLE_ENCRYPTION_KEY: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
4444
# Defaults apply to the push trigger too, where `inputs` is empty.
4545
MODE: ${{ inputs.mode || 'smoke' }}
46-
DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL_15_real.*' }}
46+
DEVICE_NAME: ${{ inputs.device-name || 'Google_Pixel_9_Pro_XL' }}
4747

4848
steps:
4949
- name: Git checkout

‎scripts/macrobenchmark-sauce.py‎

Lines changed: 59 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import argparse
3535
import json
3636
import os
37+
import re
3738
import sys
3839
import time
3940
from pathlib import Path
@@ -53,7 +54,10 @@
5354
# Shell-owned scratch space for the instrumentation's own stdout and exit code.
5455
SHELL_SCRATCH_DIR = "/data/local/tmp/macrobenchmark"
5556

56-
DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL_15_real.*"
57+
# Matched as a regex against descriptor ids and names. Kept loose on purpose: Real Device
58+
# Access descriptors are not the ids used in .sauce/*.yml, which carry an OS version and a
59+
# region suffix (Google_Pixel_9_Pro_XL_15_real_sjc1).
60+
DEFAULT_DEVICE = "Google_Pixel_9_Pro_XL"
5761

5862

5963
class SauceError(Exception):
@@ -79,14 +83,14 @@ def _request(self, method, url, expect=(200,), **kwargs):
7983

8084
# --- entitlement probe -------------------------------------------------
8185

82-
def probe(self, device_name):
83-
"""Confirms the account can reach the API, and that the device exists."""
84-
response = requests.get(
85-
f"{self.rda}/devices",
86-
params={"deviceId": device_name},
87-
auth=self.auth,
88-
timeout=60,
89-
)
86+
def device_catalog(self):
87+
"""Returns the full device catalog, confirming the account can reach the API.
88+
89+
Deliberately unfiltered: the endpoint's own deviceId filter gives no way to tell
90+
"this account has no devices" apart from "your pattern matched nothing", and the
91+
descriptors here are not the ids saucectl uses (no region suffix).
92+
"""
93+
response = requests.get(f"{self.rda}/devices", auth=self.auth, timeout=60)
9094
if response.status_code in (401, 403):
9195
raise SauceError(
9296
"Real Device Access API rejected these credentials "
@@ -97,7 +101,7 @@ def probe(self, device_name):
97101
raise SauceError(
98102
f"GET /devices returned {response.status_code}: {response.text[:500]}"
99103
)
100-
return [d for d in response.json() if d.get("os") == "ANDROID"]
104+
return response.json()
101105

102106
# --- app storage -------------------------------------------------------
103107

@@ -118,6 +122,8 @@ def upload_app(self, apk):
118122
# --- session lifecycle -------------------------------------------------
119123

120124
def open(self, device_name, duration="PT1H"):
125+
# device_name is a concrete descriptor id resolved from the catalog, not a pattern,
126+
# so we do not depend on the server's regex semantics.
121127
body = {
122128
"device": {"deviceName": device_name, "os": "android"},
123129
"configuration": {"sessionDuration": duration},
@@ -276,6 +282,39 @@ def pull_results(device, out_dir):
276282
return pulled
277283

278284

285+
def android_devices(catalog):
286+
return [d for d in catalog if str(d.get("os", "")).upper() == "ANDROID"]
287+
288+
289+
def matching_devices(catalog, pattern):
290+
"""Android descriptors whose id or human-readable name matches `pattern`."""
291+
regex = re.compile(pattern, re.IGNORECASE)
292+
return [
293+
d
294+
for d in android_devices(catalog)
295+
if regex.search(d.get("id", "")) or regex.search(d.get("name", ""))
296+
]
297+
298+
299+
def describe_catalog(catalog, pattern):
300+
android = android_devices(catalog)
301+
lines = [
302+
f"Real Device Access API reachable: {len(catalog)} device(s) in the catalog, "
303+
f"{len(android)} Android."
304+
]
305+
matches = matching_devices(catalog, pattern)
306+
lines.append(f"{len(matches)} match {pattern!r}: {[d['id'] for d in matches[:10]]}")
307+
if not matches:
308+
# Print the catalog so a naming mismatch is diagnosable from one run: RDA
309+
# descriptors have no region suffix, unlike the ids in .sauce/*.yml.
310+
lines.append("Available Android devices:")
311+
lines += [
312+
f" {d.get('id')} ({d.get('name')}, {d.get('os')} {d.get('osVersion')})"
313+
for d in sorted(android, key=lambda d: d.get("id", ""))
314+
]
315+
return "\n".join(lines)
316+
317+
279318
def print_summary(benchmark_data):
280319
data = json.loads(benchmark_data.read_text())
281320
context = data["context"]
@@ -323,24 +362,28 @@ def main():
323362
device = RealDeviceSession(args.region, username, access_key)
324363

325364
try:
326-
matches = device.probe(args.device_name)
365+
catalog = device.device_catalog()
327366
except SauceError as error:
328367
sys.exit(f"Real Device Access API probe failed: {error}")
329368

330-
print(f"Real Device Access API reachable; {len(matches)} Android device(s) match "
331-
f"{args.device_name!r}: {[d['id'] for d in matches[:5]]}")
332-
if args.probe_only:
333-
return
369+
print(describe_catalog(catalog, args.device_name))
370+
matches = matching_devices(catalog, args.device_name)
371+
# Checked before --probe-only returns, so the probe fails loudly on a device name that
372+
# matches nothing rather than passing and letting the real run discover it.
334373
if not matches:
335374
sys.exit(f"No Android device matches {args.device_name!r}")
375+
device_id = matches[0]["id"]
376+
print(f"Using device {device_id}")
377+
if args.probe_only:
378+
return
336379
if not args.app or not args.test_app:
337380
sys.exit("--app and --test-app are required unless --probe-only is given")
338381

339382
app_reference = device.upload_app(args.app)
340383
test_app_reference = device.upload_app(args.test_app)
341384

342385
try:
343-
device.open(args.device_name)
386+
device.open(device_id)
344387
device.install(app_reference)
345388
device.install(test_app_reference)
346389
device.disable_animations()

0 commit comments

Comments
 (0)