Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
271 changes: 271 additions & 0 deletions scripts/ax-chat-caret.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,271 @@
// Reads whether the Chat composer paints a blinking text caret (#891).
//
// The accessibility tree cannot answer this. A text field reports
// AXSelectedTextRange whether or not anything is drawn, so the composer
// looked healthy to AX for the whole life of the bug. The only witness is the
// screen: a caret is a narrow column that toggles on and off about twice a
// second, so a burst of stills of the field either changes or it does not.
//
// Needs an Accessibility grant and a Screen Recording grant for whatever runs
// it (System Settings > Privacy & Security). Caller: verify-chat-caret-macos.sh.
//
// This takes the pointer's place in the window order: it brings ai-buddy to
// the front for the length of the burst. It posts no click and no keystroke.

import AppKit
import ApplicationServices
import Foundation

let args = Array(CommandLine.arguments.dropFirst())

func die(_ message: String) -> Never {
FileHandle.standardError.write("ax-chat-caret: \(message)\n".data(using: .utf8)!)
exit(2)
}

guard args.count >= 3, let pid = pid_t(args[0]) else {
die("usage: ax-chat-caret <pid> <window title> <out dir> [frames]")
}
let wantedTitle = args[1]
let outDir = args[2]
let frames = args.count >= 4 ? (Int(args[3]) ?? 16) : 16

guard AXIsProcessTrusted() else {
die("""
no Accessibility grant. Add the terminal running this to
System Settings > Privacy & Security > Accessibility, then run again.
""")
}

let app = AXUIElementCreateApplication(pid)

func attribute(_ element: AXUIElement, _ name: String) -> CFTypeRef? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else {
return nil
}
return value
}

func string(_ element: AXUIElement, _ name: String) -> String? {
attribute(element, name) as? String
}

func children(_ element: AXUIElement) -> [AXUIElement] {
attribute(element, kAXChildrenAttribute) as? [AXUIElement] ?? []
}

func waitFor<T>(_ seconds: Double, _ produce: () -> T?) -> T? {
let deadline = Date().addingTimeInterval(seconds)
while Date() < deadline {
if let value = produce() { return value }
usleep(120_000)
}
return produce()
}

func frame(_ element: AXUIElement) -> CGRect? {
guard let position = attribute(element, kAXPositionAttribute),
let size = attribute(element, kAXSizeAttribute)
else { return nil }
var origin = CGPoint.zero
var extent = CGSize.zero
AXValueGetValue(position as! AXValue, .cgPoint, &origin)
AXValueGetValue(size as! AXValue, .cgSize, &extent)
return CGRect(origin: origin, size: extent)
}

func window(titled title: String) -> AXUIElement? {
let windows = attribute(app, kAXWindowsAttribute) as? [AXUIElement] ?? []
return windows.first { string($0, kAXTitleAttribute) == title }
}

/// Every element under `root` in tree order, which is render order.
func flatten(_ root: AXUIElement, _ depth: Int = 0) -> [AXUIElement] {
guard depth < 30 else { return [] }
return [root] + children(root).flatMap { flatten($0, depth + 1) }
}

/// The first editable text area in tree order. Whether the value can be
/// written is the only signal a frozen field gives: the Prompt tab draws two
/// read-only layers beside the one you can type into, and a read-only field
/// paints no caret however healthy the window is.
func editableTextArea(_ root: AXUIElement) -> AXUIElement? {
flatten(root).first { element in
guard string(element, kAXRoleAttribute) == "AXTextArea" else { return false }
var settable: DarwinBoolean = false
let read = AXUIElementIsAttributeSettable(
element, kAXValueAttribute as CFString, &settable)
return read == .success && settable.boolValue
}
}

/// A WKWebView publishes its tree only on request, and the real subtree lands
/// a moment after the first ask. `ax-settings.swift` carries the long version.
func settled(_ window: AXUIElement) -> Bool {
guard let area = flatten(window).first(where: { string($0, kAXRoleAttribute) == "AXWebArea" })
else { return false }
return !children(area).isEmpty
}

/// The window the caret question is about, front-most so WebKit paints it as
/// active. A window that is not key draws no caret however the page is styled,
/// which is the whole reason this script exists.
func activate() {
NSRunningApplication(processIdentifier: pid)?.activate()
usleep(400_000)
}

func focusedRole() -> String {
var reference: CFTypeRef?
guard
AXUIElementCopyAttributeValue(app, kAXFocusedUIElementAttribute as CFString, &reference)
== .success, let focused = reference,
CFGetTypeID(focused) == AXUIElementGetTypeID()
else { return "none" }
return string(focused as! AXUIElement, kAXRoleAttribute) ?? "?"
}

func keyState(_ window: AXUIElement) -> String {
let main = (attribute(window, kAXMainAttribute) as? Bool).map(String.init) ?? "-"
let focused = (attribute(window, kAXFocusedAttribute) as? Bool).map(String.init) ?? "-"
return "main=\(main) key=\(focused)"
}

/// RGBA of one still, or nil when the file never landed.
func pixels(_ path: String) -> (width: Int, height: Int, bytes: [UInt8])? {
guard let image = NSImage(contentsOfFile: path),
let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil)
else { return nil }
let width = cgImage.width
let height = cgImage.height
var bytes = [UInt8](repeating: 0, count: width * height * 4)
guard
let context = CGContext(
data: &bytes, width: width, height: height, bitsPerComponent: 8,
bytesPerRow: width * 4, space: CGColorSpaceCreateDeviceRGB(),
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue)
else { return nil }
context.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height))
return (width, height, bytes)
}

struct Burst {
let name: String
let rect: String
let differingFrames: Int
let columns: Int
let rows: Int
let columnSpan: String

/// A caret, told apart from a repaint: something toggled, and what toggled
/// is a narrow column taller than it is wide. Retina doubles the width, so
/// the bound is in device pixels rather than points.
var isCaret: Bool { differingFrames > 0 && columns <= 8 && rows >= 8 }
}

/// Take `frames` stills of `box` and say what changed between them.
func burst(_ name: String, _ box: CGRect) -> Burst {
let rect = "\(Int(box.origin.x)),\(Int(box.origin.y)),\(Int(box.width)),\(Int(box.height))"
var paths: [String] = []
for index in 0..<frames {
let path = "\(outDir)/\(name)-\(String(format: "%02d", index)).png"
let capture = Process()
capture.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
capture.arguments = ["-x", "-o", "-R", rect, path]
try? capture.run()
capture.waitUntilExit()
paths.append(path)
usleep(110_000)
}

guard let first = pixels(paths[0]) else {
die("no still at \(paths[0]) - is Screen Recording granted?")
}
var changedColumns = Set<Int>()
var changedRows = Set<Int>()
var differing = 0
for path in paths.dropFirst() {
guard let still = pixels(path), still.width == first.width, still.height == first.height
else { continue }
var differs = false
for y in 0..<first.height {
for x in 0..<first.width {
let offset = (y * first.width + x) * 4
// 24 clears JPEG-ish noise and leaves a caret, which is drawn
// at the ink colour against the field fill.
let moved = (0..<3).contains {
abs(Int(still.bytes[offset + $0]) - Int(first.bytes[offset + $0])) > 24
}
if moved {
changedColumns.insert(x)
changedRows.insert(y)
differs = true
}
}
}
if differs { differing += 1 }
}
let span =
changedColumns.isEmpty
? "-" : "\(changedColumns.min()!)..\(changedColumns.max()!)"
return Burst(
name: name, rect: rect, differingFrames: differing, columns: changedColumns.count,
rows: changedRows.count, columnSpan: span)
}

try? FileManager.default.createDirectory(atPath: outDir, withIntermediateDirectories: true)

guard let chat = waitFor(15, { window(titled: wantedTitle) }) else {
die("no window titled \(wantedTitle)")
}
activate()
guard waitFor(8, { settled(chat) ? chat : nil }) != nil else {
die("the Chat webview never published its tree")
}

// The composer is the text area inside the form landmark; the Prompt tab's
// field is the control. Both live in this one window, so they share its key
// state and its webview. If only the composer stays dark the cause is in the
// page; if both do, it is the window.
let tree = flatten(chat)
guard let composer = editableTextArea(chat), let composerBox = frame(composer)
else { die("no composer text area in \(wantedTitle)") }

print("window \(keyState(chat)) focused=\(focusedRole())")
let composerBurst = burst("composer", composerBox)
print("window \(keyState(chat)) focused=\(focusedRole())")

// The Prompt tab, opened through its accessibility action rather than a click.
var controlBurst: Burst?
if let tab = tree.first(where: {
string($0, kAXRoleAttribute) == "AXRadioButton" && string($0, kAXTitleAttribute) == "Prompt"
}) {
if AXUIElementPerformAction(tab, kAXPressAction as CFString) == .success {
usleep(600_000)
if let field = editableTextArea(chat), let box = frame(field) {
controlBurst = burst("prompt", box)
}
}
}

func report(_ burst: Burst) {
print(
"\(burst.name): rect=\(burst.rect) differing=\(burst.differingFrames)/\(frames - 1) "
+ "columns=\(burst.columns) rows=\(burst.rows) span=\(burst.columnSpan) "
+ "caret=\(burst.isCaret)")
}

report(composerBurst)
if let control = controlBurst { report(control) } else { print("prompt: not reached") }

if composerBurst.isCaret {
print("PASS composer paints a blinking caret")
exit(0)
}
if let control = controlBurst, control.isCaret {
print("FAIL composer draws no caret while the Prompt field does - the cause is in the page")
} else {
print("FAIL neither field draws a caret - the cause is the window, not the page")
}
exit(1)
97 changes: 97 additions & 0 deletions scripts/verify-chat-caret-macos.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
#!/usr/bin/env bash
# The Chat composer draws a blinking text caret (#891).
#
# TAKES THE SCREEN for about half a minute. Ask whoever is at the machine
# first. It posts no click and no keystroke: AI_BUDDY_OPEN_CHAT opens the Chat
# surface and an accessibility action opens the Prompt tab.
#
# Covers pixels and nothing else. AX has no caret — a focused field reports
# AXSelectedTextRange whether or not one is painted, which is why this was
# filed against a window every AX assertion called healthy. So it proves that
# something narrow blinks where the caret belongs, and not its colour, its
# place, or that it follows an arrow key.
#
# Usage:
# ./scripts/verify-chat-caret-macos.sh
# AI_BUDDY_VERIFY_BIN=path/to/ai-buddy ./scripts/verify-chat-caret-macos.sh
#
# Needs an Accessibility grant and a Screen Recording grant. CI does not run it.

set -uo pipefail

root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$root" || exit 1

bin="${AI_BUDDY_VERIFY_BIN:-$root/target/debug/ai-buddy}"
out="$root/.verify/chat-caret-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$out"

app_pid=""

info() { printf '\033[36m[INFO]\033[0m %s\n' "$*"; }
pass() { printf '\033[32m[PASS]\033[0m %s\n' "$*"; }
fail() { printf '\033[31m[FAIL]\033[0m %s\n' "$*"; }

# Inline: shellcheck cannot see that trap calls a function.
trap '[ -n "$app_pid" ] && kill "$app_pid" 2> /dev/null' EXIT

[ -x "$bin" ] || {
fail "no binary at $bin - cargo build -p ai-buddy first, or set AI_BUDDY_VERIFY_BIN"
exit 1
}

info "Output: $out"
probe="$out/ax-chat-caret"
swiftc -O "$root/scripts/ax-chat-caret.swift" -o "$probe" || {
fail "could not compile scripts/ax-chat-caret.swift"
exit 1
}

# A scratch HOME, so a sitting never edits the settings of whoever ran it.
home="$out/home"
mkdir -p "$home"
for link in .claude .claude.json .codex .config; do
[ -e "$HOME/$link" ] && ln -sfn "$HOME/$link" "$home/$link"
done

log="$out/app.log"
# The key skips the Keychain modal that otherwise holds startup; without it the
# app traces no frames and reads as a broken build. Capturable, or the stills
# come back without the window.
HOME="$home" \
AI_BUDDY_DIRECTOR_API_KEY="${AI_BUDDY_DIRECTOR_API_KEY:-verify}" \
AI_BUDDY_CAPTURABLE=1 \
AI_BUDDY_OPEN_CHAT=1 \
AI_BUDDY_CHARACTER="${AI_BUDDY_CHARACTER:-timber-wolf}" \
AI_BUDDY_CHARACTERS="${AI_BUDDY_CHARACTERS:-$root/characters}" \
"$bin" > "$log" 2>&1 &
app_pid=$!

waited=0
while [ "$waited" -lt 40 ]; do
grep -qE '^overlay:' "$log" 2> /dev/null && break
if ! kill -0 "$app_pid" 2> /dev/null; then
fail "ai-buddy exited before overlay:"
tail -40 "$log" || true
exit 1
fi
sleep 0.5
waited=$((waited + 1))
done

# The Chat window carries the Instance name, which is the Character's unless
# the roster renamed it. The log line says which Instances came up.
title="${AI_BUDDY_VERIFY_CHAT_TITLE:-$(sed -n 's/.*sprite [0-9]*x[0-9]*; \([^,]*\) as \(.*\)$/\2/p' "$log" | head -1)}"
[ -n "$title" ] || title="Timber Wolf"
info "Chat window title: $title"

"$probe" "$app_pid" "$title" "$out" 16 > "$out/caret.txt" 2>&1
rc=$?
cat "$out/caret.txt"

if [ "$rc" -eq 0 ]; then
pass "composer caret proven - stills under $out"
exit 0
fi
fail "composer caret not proven - stills and reasoning under $out"
exit 1
Loading
Loading