diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2e7a0d3..510a7489 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,20 @@ jobs: with: components: rustfmt, clippy + # Reclaim runner disk before building. This job compiles the workspace + # several times (debug + two release builds) plus the whole test suite, and + # the release profile keeps full debuginfo (`debug = true`), so the target + # tree is large; a full GitHub-hosted runner can otherwise exhaust its disk + # mid-link (a linker `Bus error`/SIGBUS). Removing preinstalled SDKs we do + # not use frees ~20 GB with no third-party action. + - name: Free disk space (Linux) + if: runner.os == 'Linux' + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/local/graalvm || true + sudo docker image prune --all --force > /dev/null 2>&1 || true + df -h / + # Cache Cargo registry and target directory for faster builds - name: Cache Cargo registry and target directory uses: Swatinem/rust-cache@v2 @@ -114,9 +128,11 @@ jobs: - name: Build LSP run: cargo build -p wfl-lsp --verbose - # Run Clippy for code quality + # Run Clippy for code quality. `--all-features` matches the binding gate in + # testing.md (the only features are opt-in dhat profiling, so this just + # compiles the feature-gated code for linting — it never runs it). - name: Run Clippy - run: cargo clippy --all-targets -- -D warnings + run: cargo clippy --all-targets --all-features -- -D warnings # Cross-platform integration test verification integration-tests: @@ -135,6 +151,19 @@ jobs: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + # Reclaim runner disk before building: this job builds the (debuginfo-heavy) + # release tree AND every integration test binary (`cargo test --test '*'`), + # which together can exhaust a full runner's disk mid-link (linker + # `Bus error`/SIGBUS). Freeing unused preinstalled SDKs gives ~20 GB of + # headroom. Linux only — the Windows runner is not affected. + - name: Free disk space (Linux) + if: runner.os == 'Linux' + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /opt/hostedtoolcache/CodeQL /usr/local/share/boost /usr/local/graalvm || true + sudo docker image prune --all --force > /dev/null 2>&1 || true + df -h / + # Cache Cargo registry and target directory for faster builds - name: Cache Cargo registry and target directory uses: Swatinem/rust-cache@v2 @@ -164,13 +193,47 @@ jobs: } Write-Host "✓ Release binary found: target/release/wfl.exe" - # Run integration tests specifically - - name: Run Integration Tests - run: cargo test --test split_functionality --verbose + # Run the DOCUMENTED WFL integration gate (testing.md): the same script a + # contributor runs locally. It executes the integration test binaries + # (`cargo test --test '*'`) AND the TestPrograms end-to-end programs — + # crucially including the intentional-error programs, which it asserts exit + # nonzero (previously those assertions lived only in this script and the + # script was never invoked in CI, so they never ran). Uses the release + # binary built above. Run on BOTH OSes so the declared Windows integration + # command is actually exercised, not merely documented. + - name: Run Integration Gate (Unix) + if: runner.os != 'Windows' + run: ./scripts/run_integration_tests.sh + + - name: Run Integration Gate (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: ./scripts/run_integration_tests.ps1 + + # Validate that documentation examples still parse/analyze/lint against the + # current release binary (testing.md requires docs validation in CI). + # `--force` ignores the committed cache so CI always re-validates rather + # than trusting a stale cached result. + - name: Validate Docs Examples (Unix) + if: runner.os != 'Windows' + run: python3 scripts/validate_docs_examples.py --ci --force + + - name: Validate Docs Examples (Windows) + if: runner.os == 'Windows' + run: python scripts/validate_docs_examples.py --ci --force + + # Web-server integration tests: start real WFL servers and exercise them + # over HTTP (testing.md requires web tests in CI for the R3 web/streaming + # surface). Uses the release binary built above. Both OSes are covered so + # Windows web-server behavior does not go unvalidated. + - name: Run Web Server Tests (Unix) + if: runner.os != 'Windows' + run: ./scripts/run_web_tests.sh - # Run all integration tests to ensure comprehensive coverage - - name: Run All Integration Tests - run: cargo test --test '*' --verbose + - name: Run Web Server Tests (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: ./scripts/run_web_tests.ps1 # Database integration tests against live PostgreSQL and MariaDB servers. # SQLite database tests need no services and already run everywhere via diff --git a/AGENTS.md b/AGENTS.md index eda3e7a6..f5b5197f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,12 +12,13 @@ Binding community and contribution policy lives at the **repo root** (not only u | `AI_POLICY.md` | **AI-assisted work is welcome** — WFL was built with AI; do not discriminate against AI use; human author remains accountable | | `CONTRIBUTING.md` | How to contribute; **Contributor application** process (Discussion or email) | | `SECURITY.md` | Private vulnerability reporting only — never file security bugs as public issues | +| `testing.md` | **Binding Logbie Testing Policy + WFL testing profile** — Red→Green TDD evidence, required test layers, risk classes, and merge/release gates (see **Testing Guidelines** below) | **Agent implications (already in force via governance):** - **AI is first-class** — use coding agents freely; same quality bar as hand-written work (tests, docs, compatibility, reviewability). - **Backward compatibility is sacred** — never break existing WFL programs without the documented deprecation path. -- **TDD mandatory** — failing tests first (`tests/`, `TestPrograms/`). +- **TDD mandatory** — failing tests first (`tests/`, `TestPrograms/`), governed by the binding **Logbie Testing Policy** in root `testing.md`: auditable **Red→Green** evidence for every behavioral change, coverage at the lowest useful layer plus every affected higher layer. - **Docs ship with the feature** — same change; validate examples; Dev Diary for non-trivial work. - **Quality gates** — `cargo fmt`, `clippy -D warnings`, `cargo test`; conventional commits. - **Do not invent maintainer identity or process** — Contributor status is by application; Maintainers own merges and releases unless those responsibilities are **explicitly delegated**. Prefer first name **Brad** only if referring to the primary maintainer in docs (no last name). @@ -121,12 +122,40 @@ Source Code → Lexer → Parser → Analyzer → Type Checker → Interpreter - Constants: `SCREAMING_SNAKE_CASE` ## Testing Guidelines -- **TDD is mandatory**: Write failing tests FIRST for any feature or bug fix. + +**Binding policy:** root `testing.md` holds the **Logbie Testing Policy** and the +WFL testing profile. It governs every behavioral change. Non-negotiables an agent +MUST follow: + +- **Red → Green → Refactor → Broaden → Record** — write the smallest useful test + FIRST, run it, confirm it **fails for the intended reason**, then make it pass; + a defect fix reproduces the defect. Keep auditable Red evidence (a Red commit + that is an ancestor of Green, or a timestamped CI artifact). A test first + observed after the code already passed is **not** a valid Red step. (§3, §6) +- **Classify risk first (R0–R3)** — concurrency, cancellation, lifecycle, + streaming, untrusted input, crypto/secrets, and backward compatibility are + **R3** and require negative/failure-path plus §11 risk-triggered tests. Risk is + never lowered to dodge a gate. (§5, §11) +- **Real boundaries, real assertions** — don't mock the boundary under test; + assert outcomes + side effects (not "didn't crash"); use negative assertions + for cancellation, writes-after-close, denial. (§7, §8.3) +- **No manufactured green** — never retry/skip/quarantine a required test to go + green; a flaky required test is failing. Non-executable docs programs use the + runner's `// CI-SKIP:` first-line directive and stay statically validated. (§8.2) +- **Concurrency/streaming/lifecycle (§11.3)** — for this repo's async/web/ + streaming work, prove races/ordering, cancellation, timeouts, disconnects, + bounded queues/backpressure, resource limits, clean shutdown, writes-after- + close, and that one slow/failed handler doesn't block unrelated work. +- **PR evidence (§15)** — record risk class, acceptance criteria → tests, Red + evidence, layers run, and residual risk (template in `testing.md`). + +### Mechanics - **Locations**: - Rust Unit/Integration: `tests/` - WFL End-to-End: `TestPrograms/` (must pass with release build) - WFL Test Framework: Use `describe`/`test` blocks, run with `wfl --test ` - **Conventions**: feature‑oriented names (`*_test.rs`, `*.test.wfl`), keep perf benches under `benches/`. +- **Commands & profile**: one command per layer + the "run all presubmit" block are in root `testing.md`. - **Testing Guide**: See `Docs/guides/testing-guide.md` for WFL testing framework documentation. ## Commit & Pull Request Guidelines diff --git a/CLAUDE.md b/CLAUDE.md index 789db63a..b123e166 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,12 +14,13 @@ Binding community and contribution policy lives at the **repo root** (not only u | `AI_POLICY.md` | **AI-assisted work is welcome** — WFL was built with AI; do not discriminate against AI use; human author remains accountable | | `CONTRIBUTING.md` | How to contribute; **Contributor application** process (Discussion or email) | | `SECURITY.md` | Private vulnerability reporting only — never file security bugs as public issues | +| `testing.md` | **Binding Logbie Testing Policy + WFL testing profile** — Red→Green TDD evidence, required test layers, risk classes, and merge/release gates (see **Testing Policy** below) | **Agent implications (already in force via governance):** - **AI is first-class** — use coding agents freely; same quality bar as hand-written work (tests, docs, compatibility, reviewability). - **Backward compatibility is sacred** — never break existing WFL programs without the documented deprecation path. -- **TDD mandatory** — failing tests first (`tests/`, `TestPrograms/`). +- **TDD mandatory** — failing tests first (`tests/`, `TestPrograms/`). Governed by the binding **Logbie Testing Policy** in root `testing.md` (see **Testing Policy** below): every behavioral change needs auditable **Red→Green** evidence and coverage at the lowest useful layer plus every affected higher layer. - **Docs ship with the feature** — same change; validate examples; Dev Diary for non-trivial work. - **Quality gates** — `cargo fmt`, `clippy -D warnings`, `cargo test`; conventional commits. - **Do not invent maintainer identity or process** — Contributor status is by application; Maintainers own merges and releases unless those responsibilities are **explicitly delegated**. Prefer first name **Brad** only if referring to the primary maintainer in docs (no last name). @@ -159,13 +160,49 @@ Source Code → Lexer → Parser → Analyzer → Type Checker → Interpreter - Types/Traits: `CamelCase` - Constants: `SCREAMING_SNAKE_CASE` -## Testing Guidelines -- **TDD is mandatory**: Write failing tests FIRST for any feature or bug fix. +## Testing Policy (binding — root `testing.md`) + +WFL adopts the **Logbie Testing Policy** (full text + the WFL testing profile in +root `testing.md`). It is binding for every behavioral change; the highlights an +agent MUST follow: + +- **Red → Green → Refactor → Broaden → Record.** Write the smallest useful test + FIRST and run it to confirm it **fails for the intended reason**, then make it + pass. A defect fix MUST reproduce the defect. Keep auditable evidence (a Red + test-only commit that is an ancestor of the Green commit, or a timestamped CI + artifact) — a test first observed *after* the code already passed does **not** + establish Red. (§3, §6) +- **Risk class first.** Classify R0–R3 before implementing; when ambiguous, the + higher class applies, and it MUST NOT be lowered to dodge a gate. Anything + touching **concurrency, cancellation, lifecycle, streaming, untrusted input, + crypto/secrets, or backward compatibility is R3** and needs negative/ + failure-path + the §11.3/§11.1 risk-triggered tests. (§5, §11) +- **Real boundaries.** A test MUST NOT mock the boundary it claims to verify; + "end-to-end" means the real binary/socket/file. Assert outcomes and side + effects, not "did not crash." Use negative assertions where absence matters + (cancellation, writes-after-close, denial). (§7, §8.3) +- **No manufactured green.** Required tests are never made green via retries, + skips, ignores, quarantine, or relaxed assertions; a flaky required test is a + failing test. Non-executable docs examples use the runner's `// CI-SKIP:` + first-line directive and are still validated statically. (§8.2) +- **Concurrency/streaming/lifecycle (§11.3) — always required for this repo's + async/web/streaming work:** prove races/ordering, cancellation, timeouts, + disconnects, bounded queues/backpressure, resource limits, clean shutdown, and + writes-after-close, and that one slow/failed handler does not block unrelated + work. +- **PR evidence (§15).** Every behavioral PR records risk class, acceptance + criteria → tests, Red evidence, the layers run, and residual risk (template in + `testing.md`). +- **Same bar for AI work.** AI-authored code/tests get the same verification — + "the model said it works" is not evidence. + +### Testing mechanics - **Locations**: - Rust Unit/Integration: `tests/` - WFL End-to-End: `TestPrograms/` (must pass with release build) - WFL Test Framework: Use `describe`/`test` blocks, run with `wfl --test ` - **Conventions**: feature‑oriented names (`*_test.rs`, `*.test.wfl`), keep perf benches under `benches/`. +- **Commands & profile**: one command per layer + the "run all presubmit" block are in root `testing.md`. - **Testing Guide**: See `Docs/guides/testing-guide.md` for WFL testing framework documentation. ## Commit & Pull Request Guidelines diff --git a/Dev diary/2026-07-22-concurrent-handler-run-state-isolation.md b/Dev diary/2026-07-22-concurrent-handler-run-state-isolation.md new file mode 100644 index 00000000..663a5667 --- /dev/null +++ b/Dev diary/2026-07-22-concurrent-handler-run-state-isolation.md @@ -0,0 +1,72 @@ +# Dev Diary — 2026-07-22 — Per-handler run-state isolation for `main loop concurrently:` + +## Context + +PR #641 shipped `main loop concurrently:` — opt-in cooperative concurrency for +HTTP request handlers, driven by a `FuturesUnordered` on the single interpreter +thread. Review (maintainer P1 #1, echoed by Copilot) flagged a soundness gap: +the concurrent loop isolated each handler's **environment** (variables), but the +interpreter's **run-state** — the count-loop variable and its flag +(`current_count` / `in_count_loop`), the live recursion depth (`call_depth`), +the diagnostic call stack (`call_stack`), and the current block's overload-dup +set — still lived on the shared `Interpreter` behind `RefCell`/`Cell`. + +Under serial execution that state is never contended. Under +`main loop concurrently:` several handler futures interleave on one thread, so at +every `await` one handler's run-state was visible to — and overwritable by — +whichever sibling was polled next. A handler that yielded *inside a `count` loop* +would resume and read a `count` set by another handler. + +## The bug, concretely + +`count` does not resolve through the environment while a count loop is active; +`try_evaluate_variable_sync` short-circuits on `in_count_loop` and reads +`self.current_count` directly. Both fields are global, so two concurrent count +loops share one `current_count`. A handler counting `1..5` that yields mid-loop +could come back reading `100..104` from a sibling. + +## Fix — a poll-swap wrapper (no `Rc`→`Arc`, no threads) + +The interpreter core stays single-threaded and `Rc`-based (a hard constraint). +Rather than thread a per-handler execution context through every `&self` method, +each handler owns a `RunState` snapshot and an `IsolatedHandler` future wraps the +handler: + +- On **each `poll`**, `swap_run_state` swaps the handler's `RunState` into the + interpreter's live fields (a field-by-field `mem::swap`, its own inverse). +- The inner handler future is polled. +- The instant `poll` returns — `Ready` **or** `Pending` — the state is swapped + back out into the handler's `RunState`. + +So the interpreter's run-state fields become effectively poll-local: exactly one +handler's state is installed at a time, and a suspended handler's state is parked +in its own `RunState` where no sibling can touch it. Each handler starts from +`RunState::fresh(base_call_depth)`. The inner future is already wrapped in +`catch_unwind`, so a panic surfaces as `Ready` and the swap-back still runs, +leaving the scratch fields clean for the next sibling. + +Serial execution is completely untouched — `IsolatedHandler` is used only by +`execute_concurrent_main_loop`. + +## Testing (Red → Green) + +`tests/concurrent_main_loop_test.rs::test_concurrent_handlers_do_not_share_count_loop_state`: +two concurrent handlers each run a `count` loop over a **disjoint** range +(`1..5` vs `100..104`), yielding via `wait for` mid-iteration and then reading +`count`. With isolation each handler observes only its own range. + +- **Red** (isolation bypassed — plain handler pushed to `FuturesUnordered`): + `/a` returned `100-101-102-103-104-`, i.e. it observed the *other* handler's + entire count range. `assertion left == right failed`. +- **Green** (isolation restored): `/a` → `1-2-3-4-5-`, `/b` → + `100-101-102-103-104-`. + +Risk class **R3** (concurrency + lifecycle). The test asserts a concrete wrong +outcome under sharing, not merely "did not crash". + +## Follow-ups still open from the review + +Larger P1 items remain and are tracked in +`Docs/development/concurrency-phase-plan.md`: immediate-500 on pre-respond +failure, browser-disconnect/504 cancellation threaded into `wait for` and +upstream reads, and an absolute total-stream deadline. diff --git a/Dev diary/2026-07-22-concurrent-request-handlers.md b/Dev diary/2026-07-22-concurrent-request-handlers.md new file mode 100644 index 00000000..64a1ae49 --- /dev/null +++ b/Dev diary/2026-07-22-concurrent-request-handlers.md @@ -0,0 +1,94 @@ +# Dev Diary — 2026-07-22 — Concurrent request handlers (`main loop concurrently:`) + +## Context + +Final piece of the five-capability streaming request (item 4). With outbound and +server streaming shipped, a handler can proxy a slow upstream to the browser — +but on the **serial** `main loop`, that slow handler blocks every other request +(login, history, health, other chats). This adds opt-in concurrent handling so a +slow stream no longer stalls its siblings. + +This maps onto **Phase 1** of `Docs/development/concurrency-phase-plan.md` — a +maintainer-locked, gated plan. I followed its hard rules (locked marker `main +loop concurrently:`, no `Rc→Arc`/`Send` rewrite of the interpreter core, plain +`main loop` stays serial, TDD, `panic = "unwind"` gate already in CI). Phase 1 +landed in one change rather than the staged 1a→1b→1c; **this is the plan's +maintainer STOP/review point.** + +## What shipped + +```wfl +listen on port 8080 as server +main loop concurrently: + wait for request comes in on server as req + // a slow handler here (e.g. streaming a slow upstream) no longer blocks siblings + respond to req with "Hello!" +end loop +``` + +Plain `main loop` is unchanged (strictly serial, byte-compatible). Adding +`concurrently` is the only way to opt in — no silent semantics swap. + +## Design & mechanism + +- **AST:** `MainLoop` gained a `concurrent: bool` (default false). `concurrently` + is a contextual identifier parsed only right after `main loop`, so it stays + usable as a variable name elsewhere. +- **Execution:** `execute_concurrent_main_loop` keeps up to + `CONCURRENT_HANDLER_LIMIT` (256) iterations of the body in flight via a + `FuturesUnordered` of `!Send`, non-`'static`, `&self`-borrowing handler + futures — cooperative concurrency on the one interpreter thread, exactly as the + plan's 1a spike prescribes (no `spawn_local`, no `Send`/`Arc` across the core). + Each iteration runs in a fresh `Environment::new_child_env` (isolation by + default). The set is refilled to the cap, so with cap ≥ 1 it is never empty — + avoiding the `Ready(None)` busy-spin trap. +- **Containment:** each handler future is wrapped in + `AssertUnwindSafe(...).catch_unwind()`. A panicking handler is caught, its + request is answered 500 by the existing `ResponseCompletion` drop guard, and + siblings keep running. A handler that returns a `RuntimeError` is likewise + logged and contained. +- **Ops defaults reused:** 503 (bounded transport queue), 504 (per-request + response deadline), and 500 (drop guard) already existed at the transport + layer, so the concurrent loop inherits them — it adds handler-level + concurrency, not a parallel ops stack. +- **Why `wait for request` allows this:** it holds the server's receiver mutex + only to dequeue one request, then releases it before the handler body runs — so + concurrent iterations hand off requests one at a time and then handle them + concurrently. No `Rc`/`RefCell` is held across an `.await` (the crate-wide + `#![deny(clippy::await_holding_refcell_ref)]` backstop enforces this). + +## Concurrent, not parallel + +Cooperative concurrency on a single thread: handlers interleave at their await +points (`wait for`, outbound HTTP, stream read/write, `respond`). A tight +CPU-bound handler with no await still holds the thread — documented as the yield +cliff. This is the honest model, not multicore parallelism (Phase 3, deferred). + +## Files + +- `src/parser/ast.rs` (`MainLoop.concurrent`), `src/parser/stmt/control_flow.rs` + (parse `concurrently`), `src/interpreter/mod.rs` + (`execute_concurrent_main_loop`, `CONCURRENT_HANDLER_LIMIT`, MainLoop branch). +- Docs: `Docs/04-advanced-features/web-servers.md` ("Concurrent request + handling") + validated example + `TestPrograms/docs_examples/web_servers/concurrent_main_loop.wfl`; + `concurrency-phase-plan.md` tracker updated. + +## Tests + +`tests/concurrent_main_loop_test.rs`: +- `main loop concurrently:` parses concurrent; plain `main loop:` stays serial. +- Concurrent: a 500 ms handler does not block a fast sibling (fast < 300 ms). +- Serial: the same slow handler *does* block the next request (fast > 300 ms) — + proving no silent upgrade. +- Handler-error containment: an erroring handler doesn't kill the server. + +`fmt`, `clippy -D warnings`, the 618 lib tests, and the existing web-server / +streaming suites are all green. + +## Out of scope (Phase 2+, per the plan) + +- Structured nursery / `wait for all|any of` / `change shared`. +- Multicore (Phase 3; deferred, needs profiling). +- Request-ID *structured* logging scheme and a written per-site eval-core audit + (mechanically enforced by the clippy backstop for now). diff --git a/Dev diary/2026-07-22-immediate-500-on-unanswered-request.md b/Dev diary/2026-07-22-immediate-500-on-unanswered-request.md new file mode 100644 index 00000000..eacfbe28 --- /dev/null +++ b/Dev diary/2026-07-22-immediate-500-on-unanswered-request.md @@ -0,0 +1,60 @@ +# Dev Diary — 2026-07-22 — Immediate 500 when a handler ends without responding + +## Context + +PR #641 review (maintainer P1 #3, echoed by CodeRabbit) flagged that the +`ResponseCompletion` drop guard does **not** cover every pre-response path. The +guard is armed only once a handler *reaches* a `respond` / `start streaming +response` statement (it takes the request's sender out of `pending_responses` +into the guard). A handler that dequeues a request with `wait for request comes +in` and then ends **before** responding — a runtime error, a `break`, or simply +returning without `respond` — leaves the sender parked in `pending_responses`. +The client then waits out the request timeout instead of getting a prompt 500. + +## Fix — arm the fallback at dequeue, disarm at respond + +Each handler now tracks the request ids it dequeued but has not answered, in +per-handler run-state (`open_pending_requests`, part of the `RunState` swapped +per poll — the same mechanism that isolates `count`/recursion state and tracks +open response streams). On exit, any still-unanswered request is answered 500: + +- `wait for request comes in` pushes the request id. +- `respond` / `start streaming response` remove the id from the map + (`pending_responses.remove`) *and* disarm the tracking. Because a responded + request is gone from the map, the exit-time sweep is naturally idempotent — it + only 500s ids still present. +- On handler exit (any path) the tracked ids are swept: + `fail_unanswered_requests` removes each from `pending_responses`, and if the + sender is still live (`try_lock` + `oneshot::send`, fully synchronous) sends a + 500. Wired at every boundary, mirroring the stream close-on-exit: + `IsolatedHandler`'s `Drop` (concurrent), the serial `main loop`'s per-iteration + drain, and program exit. + +The sweep is synchronous so it runs from `Drop`. The sender's mutex is only held +during `respond`, which a finished handler is no longer inside, so `try_lock` +succeeds; the transport request timeout remains the backstop for the impossible +case where it does not. + +## Testing (Red → Green) + +`tests/concurrent_main_loop_test.rs::test_handler_that_never_responds_gets_immediate_500`: +a `main loop concurrently:` server whose `/drop` path dequeues the request and +ends without responding. The client must get a **prompt 500**, and the server +must keep serving (`/ok` still works). + +- **Red** (500-on-exit disabled): the `/drop` request hangs — the client waits + out the transport timeout, exceeding even the 120s test harness timeout, i.e. + exactly the "client left waiting" symptom. +- **Green**: `/drop` returns 500 in well under a second; `/ok` still returns + `ok`. + +Risk class **R3** (lifecycle). The test asserts both the status *and* that it +arrives before the timeout, so a regression to "eventually times out" fails. + +## Also in this change (from review) + +- The response-byte-ceiling and client-disconnect paths now also untrack the + stream id from `open_response_streams`, so a handler that catches the error and + continues keeps no stale ids. +- Typechecker: HTTP/response header type hints widened to `map[text, any]` + (values may be text, numbers, or booleans, converted to text). diff --git a/Dev diary/2026-07-22-outbound-response-streaming.md b/Dev diary/2026-07-22-outbound-response-streaming.md new file mode 100644 index 00000000..91bc87f5 --- /dev/null +++ b/Dev diary/2026-07-22-outbound-response-streaming.md @@ -0,0 +1,100 @@ +# Dev Diary — 2026-07-22 — Generic outbound response streaming + +## Context + +A downstream app (a browser chat UI talking to a model endpoint) needs the WFL +runtime to proxy a slow upstream to the browser without buffering. That request +came in as five items: (1) outbound response streaming, (2) incremental +chunk/line reads, (3) streamed *server* responses, (4) concurrent request +handlers, and (5) lifecycle guarantees (timeouts, backpressure, cancellation, +catchable errors, close-on-every-exit). + +Items 3 and 4 are large and, importantly, item 4 (concurrent handlers) is +**already governed** by `Docs/development/concurrency-phase-plan.md` — a +maintainer-locked, gated plan (locked marker `main loop concurrently:`, "no +Rc→Arc rewrite of the interpreter core", TDD-first, stop-for-review between +phases). This entry covers the first shippable slice: **items 1, 2, and the +streaming-relevant parts of item 5 (outbound/client side)**, which are net-new +and do *not* touch the locked concurrency core. Server streaming (3) and the +concurrent loop (4) are separate follow-on changes. + +## What shipped + +New surface, mirroring the existing `open url` client: + +```wfl +open url at "" [with method .. and headers .. and body ..] and stream response as upstream +wait for next line from upstream as line // Text, or nothing at clean EOF +wait for next chunk from upstream as chunk // Binary, or nothing at clean EOF +close upstream // cancels the in-flight upstream +``` + +`stream response as` returns as soon as the status/headers arrive — **without +buffering the body** — and binds an object exposing `status`, `ok`, `headers`, +and an internal `_stream` id. The body stays parked in the interpreter and is +pulled incrementally. + +## Design notes + +- **No new lexer tokens.** `stream`, `next`, `chunk`, `line`, `upstream` are all + contextual identifiers; `response`/`from`/`as` are existing keywords. The + lexer's identifier-merging means `next chunk`/`next line` arrive as a single + token, handled in `parse_wait_for_statement`. +- **Handle model follows the existing pattern.** Open resources in WFL are + opaque ids into side-tables on `IoClient` (files, DB pools, processes). Added + `stream_handles: Mutex>`. `HttpStreamHandle` + holds a `Pin>> + Send>>` (from + `response.bytes_stream()`), a leftover-byte buffer for line splitting, a + `done` flag, and a running `bytes_read` total. +- **No lock held across the network await.** `next_chunk`/`next_line` *take* the + handle out of the map, await, then put it back — so a slow read on one stream + never blocks map access for another (this matters once concurrent handlers + land). +- **Lifecycle (item 5, client side).** The head phase and each per-chunk read go + through the existing `run_http_with_budget` select, so connect/read timeouts, + the response-byte ceiling (`web_server_max_response_size`, enforced + incrementally on the running total, not just `Content-Length`), and + cooperative cancellation all apply. Mid-stream network errors surface as + catchable `RuntimeError`s from the `wait for next ...` statement. Dropping the + handle — on clean EOF, error, explicit `close`, or interpreter teardown — + drops the reqwest body future and cancels the upstream. Reading a + closed/drained handle is a predictable catchable error. +- **`close` extended, not duplicated.** `close ` already closed files; + it now also accepts a streaming-response object and closes its stream. +- **EOF semantics.** A final unterminated line is delivered before EOF; the + handle is re-inserted (drained, `done`) so the *next* read returns `nothing` + once, then the handle is freed — the `check if line is nothing: break` loop + works and handles don't leak across many streamed requests. + +## Files + +- AST: `HttpStreamStatement`, `WaitForNextChunkStatement`, + `WaitForNextLineStatement` (`src/parser/ast.rs`). +- Parser: `stream response as` clause (`src/parser/stmt/io.rs`), `wait for next + chunk|line from` (`src/parser/stmt/processes.rs`). +- Interpreter: `IoClient::{open_http_stream, next_chunk, next_line, + close_stream}` + helpers, three statement arms, `close` extension + (`src/interpreter/mod.rs`). +- Analyzer/typechecker/transpiler: variable-binding registration and an explicit + "not supported in JS transpilation" arm. +- Docs: `Docs/04-advanced-features/interoperability.md` (new "Streaming a + response incrementally" section) + validated example + `TestPrograms/docs_examples/interoperability/streaming_response.wfl`. + +## Tests + +`tests/http_stream_test.rs` — parser tests for all three statements, and +offline runtime tests against a local one-shot TCP server: status/headers +available immediately, `next line` yields lines then `nothing`, a final +unterminated line is delivered, `next chunk` yields binary, and reading a closed +stream is an error. `cargo fmt`, `clippy -D warnings`, and the existing +`http_request_*` / `http_outbound_budget` suites are green. + +## Not in this change (follow-ons) + +- **Server-side streaming** (`write chunk`/`flush`/`close` on a response) — + needs the `oneshot` reply path reworked into a chunked body + channel through warp. +- **Concurrent request handlers** — Phase 1 of the concurrency plan + (`main loop concurrently:`); the keystone that makes a slow upstream stream + not stall other requests. Follows the gated plan, not this change. diff --git a/Dev diary/2026-07-22-server-response-streaming.md b/Dev diary/2026-07-22-server-response-streaming.md new file mode 100644 index 00000000..f66621f5 --- /dev/null +++ b/Dev diary/2026-07-22-server-response-streaming.md @@ -0,0 +1,87 @@ +# Dev Diary — 2026-07-22 — Streamed server responses + +## Context + +Follow-on to the same-day outbound response streaming work. This adds the +**server** half (item 3 of the five-capability request): a WFL handler can now +send a response whose body is produced progressively — status/headers first, +then body pieces — which is what a browser chat UI needs to read newline- +delimited JSON off a `fetch()` response as it arrives. + +## What shipped + +```wfl +start streaming response to req with status 200 and content type "application/x-ndjson" as out +write line json_text to out // frames a line (newline appended) +write chunk raw_bytes to out // raw bytes/text, verbatim +flush out // advisory +close out // ends the response body +``` + +`start streaming response` returns immediately after sending the head and binds +a stream handle (`{ _server_stream, status }`). Combined with the client-side +`stream response as upstream` + `wait for next line`, a handler can proxy a slow +upstream to the browser line-by-line without buffering either side. + +## Design & mechanism + +- **Reply payload is now an enum.** The per-request `oneshot` carries a + `HandlerReply` = `Buffered(WflHttpResponse)` | `Streaming { status, + content_type, headers, body: mpsc::Receiver> }`. `respond` sends + `Buffered`; `start streaming response` sends `Streaming` with the receiving end + of a bounded body channel. +- **Transport reply became `Body`-typed.** The warp route's final closure now + returns `Response`. Key simplification: warp's `.recover()` + unifies reply types via `Either`, so only that one closure changed — the five + `Response>` helper functions and `handle_overloaded` were left alone; + the closure wraps their returns with `.map(Body::from)`. The streaming arm + builds `Body::wrap_stream(futures_util::stream::unfold(rx, ...))` — no new + dependency. +- **Backpressure & disconnect.** The body channel is bounded + (`RESPONSE_STREAM_BUFFER = 64`), so a slow client backpressures the handler's + `write` (it awaits a free slot). When the client disconnects, hyper drops the + body, dropping the receiver; the handler's next `write` then fails with a + catchable error — that is how a browser disconnect propagates to the handler + (which can then `close` the upstream it is proxying). An explicit `close out` + drops the sender, ending the response; and as a safety net the stream is + auto-closed when the handler ends on any path (see the follow-up entry + `2026-07-22-stream-auto-close-on-handler-exit.md`), so a forgotten `close` + never hangs the client. +- **`start` is a keyword, `streaming`/`flush`/`line`/`chunk` are identifiers.** + `start streaming response` dispatches on `Token::KeywordStart`; `flush ` + and `write line|chunk to ` handle the lexer's identifier-merging + (`flush out`, `line payload`) the same way the websocket-message statements do. +- **`close` unified.** `close ` now closes a file (`Text`), a client upstream + (`_stream`), or a server response stream (`_server_stream`). + +## Files + +- Types + transport + exec: `src/interpreter/mod.rs` (`HandlerReply`, + `Body`-typed closure, `server_response_streams` map, three statement arms, + `close` extension). +- AST: `StartStreamingResponseStatement`, `StreamWriteStatement`, + `FlushStreamStatement`. +- Parser: `parse_start_streaming_response`, `parse_flush_stream` + (`src/parser/stmt/web.rs`), `write line|chunk` branch + (`src/parser/stmt/io.rs`), `KeywordStart`/`flush` dispatch (`parser/mod.rs`). +- Analyzer/typechecker/transpiler arms. +- Docs: `Docs/04-advanced-features/web-servers.md` ("Streaming a response", + incl. an upstream-proxy example) + validated example + `TestPrograms/docs_examples/web_servers/streaming_response.wfl`. + +## Tests + +`tests/http_server_streaming_test.rs` — parser tests for all four statements, +plus end-to-end runtime tests that stand up a WFL streaming server and read it +back with reqwest: status/headers arrive with `write line` framing +(`alpha\nbeta\ngamma\n`), and `write chunk` is verbatim (`onetwo`). The existing +web-server suites (query/binary/content-length/queue-bound/admission) still pass +after the transport reply-type change; `fmt`, `clippy -D warnings`, and the 618 +lib tests are green. + +## Still open + +- **Item 4 — concurrent handlers** (`main loop concurrently:`), Phase 1 of the + locked concurrency plan: today a streamed handler runs to completion before the + next request is served, so per-request isolation between a slow stream and + other requests still awaits Phase 1. diff --git a/Dev diary/2026-07-22-stream-auto-close-on-handler-exit.md b/Dev diary/2026-07-22-stream-auto-close-on-handler-exit.md new file mode 100644 index 00000000..e7f171a2 --- /dev/null +++ b/Dev diary/2026-07-22-stream-auto-close-on-handler-exit.md @@ -0,0 +1,66 @@ +# Dev Diary — 2026-07-22 — Auto-close server response streams on handler exit + +## Context + +PR #641's streamed server responses (`start streaming response` / `write` / +`flush` / `close out`) parked the body-channel **sender** in a long-lived +interpreter table, `server_response_streams`, keyed by an opaque `respstream*` +id. The transport turns the matching receiver into the chunked response body via +`Body::wrap_stream`, which only ends once **every** sender is dropped. + +Review (Devin 🐛, Copilot, CodeRabbit) flagged the consequence: the sender was +dropped in only two places — an explicit `close out` and the write-after- +disconnect path. A handler that started a stream and ended **without** `close out` +(normal return, a caught error, a `break`) left its sender in the table forever: + +- the client's chunked body was never terminated → **the client hangs**, and +- the table grew one dead entry per streamed request → **a memory leak**. + +This also contradicted the shipped docs and design doc, which promised +"close-on-exit," and the original streaming spec's item 5 lifecycle guarantee: +*all streams close on every exit path.* + +## Fix — tie each stream's lifetime to its handler + +Each handler now tracks the `respstream*` ids it opens and closes them when it +ends, on **every** path: + +- New per-handler field `open_response_streams` lives in the interpreter and is + part of the `RunState` swapped in/out per poll (the same poll-local mechanism + that isolates `count`/recursion state under `main loop concurrently:`), so each + handler tracks only its own streams even while interleaved. +- `start streaming response` pushes the new id; an explicit `close out` removes it + (keeping the list bounded to genuinely-open streams). +- `close_response_streams(ids)` removes each id from `server_response_streams`, + dropping its sender and ending the body. It is idempotent — an id already closed + is a no-op — so closing a handler's whole opened-list on exit is always safe. + +Close-on-exit is wired at every boundary: + +- **Concurrent handlers:** `IsolatedHandler`'s `Drop` closes the handler's + `state.open_response_streams`. Drop runs whether the handler returned, errored, + panicked (contained by `catch_unwind`), or was cancelled as the loop tore down. +- **Serial `main loop`:** each iteration drains and closes after `execute_block`, + on the normal *and* error paths. +- **Top level:** `interpret_inner` drains at program exit for streams opened + outside any loop, and clears the tracking on run entry (REPL reuse). + +## Testing (Red → Green) + +`tests/http_server_streaming_test.rs::test_stream_auto_closes_when_handler_ends_without_close`: +a handler starts a stream, writes one line, and ends **without** `close out`. The +client reads the body under a 5s `tokio::time::timeout`. + +- **Red** (auto-close drain disabled): the body never finishes — + `timeout ... Elapsed(())`; the read hangs exactly as a real client would. +- **Green** (auto-close restored): body reads back `"hello\n"` and completes. + +Risk class **R3** (lifecycle/streaming). The negative outcome (a hang) is turned +into a deterministic failure by the timeout rather than a stuck test. + +## Docs + +`web-servers.md`, `response-streaming-design.md`, and the server-streaming dev +diary updated to describe the shipped close-on-exit behavior. Explicit `close out` +is still recommended to finalize promptly (and free the connection sooner); +auto-close is the safety net, not a substitute. diff --git a/Dev diary/2026-07-22-write-line-file-write-backcompat.md b/Dev diary/2026-07-22-write-line-file-write-backcompat.md new file mode 100644 index 00000000..14ca7249 --- /dev/null +++ b/Dev diary/2026-07-22-write-line-file-write-backcompat.md @@ -0,0 +1,58 @@ +# Dev Diary — 2026-07-22 — `write line/chunk` preserves the classic file write + +## Context + +The new streamed-response verbs `write line to ` / +`write chunk to ` share a surface with the pre-existing file write +`write to `. Because WFL identifiers can be space-separated, the +lexer merges `line payload` into a single `Identifier("line payload")` token. The +first cut of the parser always split such a token into a `line` marker plus a +value, so `write line payload to out` was unconditionally parsed as a stream +write — silently breaking any pre-existing program that wrote a variable literally +named `line payload` to a file. Review (Copilot, twice) flagged this as a +backward-compatibility break. + +Backward compatibility is sacred, and the two readings genuinely cannot be told +apart at parse time: `write line to ` (the primary NDJSON use case) +and `write line to ` (a variable named `line `) both use a bare +variable. The only correct disambiguation is on the **runtime target type**. + +## Fix — carry both readings, decide at runtime + +- **AST/parser.** `StreamWriteStatement` gained `fallback_content: + Option>`. For the ambiguous merged form the parser now records + both the stream value (`Variable("payload")`) and the classic file-write + content (`Variable("line payload")`). Unambiguous forms — a literal value, or a + bare marker directly before `to` — set `None` (they were never valid file + writes). +- **Interpreter.** `StreamWriteStatement` evaluates the target first. If it is a + server response stream, it does the stream write. Otherwise, if a + `fallback_content` is present, it performs the classic `write to + ` file write; if not, it errors as before. +- **Static analysis.** For the ambiguous form the live reading (and thus which + variable must exist) is unknown until runtime, so semantic analysis defers + definedness for it instead of rejecting the file-write reading. The + unused-variable pass counts **both** candidate variables as used, so a variable + named `line ` written to a file is not falsely reported unused. + +The other statements (`analyze`, typechecker, transpiler) already matched with +`..`; the transpiler still rejects streaming statements outright. + +## Testing (Red → Green) + +`tests/write_line_backcompat_test.rs`: + +1. `test_write_line_multiword_variable_parses_with_fallback` — the merged form + parses with `fallback_content: Some`; the literal form with `None`. +2. `test_write_multiword_line_variable_to_file_still_works` — runs the full + analyzer + interpreter on `store line note as "…"` / `write line note to + ""`, asserting analysis accepts it and the file receives the **variable's + value**, not the token `note`. + +- **Red** (analyzer analyzing the stream value unconditionally): semantic analysis + rejects the program with `Variable 'note' is not defined`. +- **Green**: analysis accepts it and the file contains the variable's value. + +Risk class **R3** (backward compatibility). The existing streaming tests +(`write line "alpha" to out`, bare `write line to out`) continue to pass, so the +stream write and the classic file write both work through the shared surface. diff --git a/Dev diary/2026-07-24-concurrent-loop-ordering-and-header-key-types.md b/Dev diary/2026-07-24-concurrent-loop-ordering-and-header-key-types.md new file mode 100644 index 00000000..f3bd7e2b --- /dev/null +++ b/Dev diary/2026-07-24-concurrent-loop-ordering-and-header-key-types.md @@ -0,0 +1,81 @@ +# Dev Diary — 2026-07-24: concurrent-loop ordering guard + header-key type validation + +Two review-driven hardening changes on the runtime-streaming branch, each with +Red→Green evidence at the lowest useful layer (semantic analysis / type check). + +## 1. `main loop concurrently:` must begin with `wait for request` + +**Problem (reviewer, CodeRabbit on `src/interpreter/mod.rs`):** the concurrent +main loop refills its handler set by starting `execute_block(body, …)` for every +slot up to the concurrency cap. Each future runs the body from the top. If the +body has any statement *before* the first `wait for request`, that statement runs +once per slot — speculatively, before a single request has been dequeued. For a +body that starts with `wait for request` the future simply parks on the request +channel, so nothing runs early; the hazard only exists for out-of-order bodies. + +**Fix:** rather than restructure the loop into a heavier dequeue-then-run engine, +enforce the invariant the well-formed case already satisfies — a +`main loop concurrently:` body must begin with `wait for request`. Semantic +analysis now rejects a concurrent loop whose first statement is anything else, +with an actionable message that tells the author to move setup above the loop. +Serial `main loop` is unaffected (it runs one iteration at a time, so there is no +speculative fan-out). This is new surface (`concurrently` shipped on this branch), +so no existing program is affected; every example and test already starts with +`wait for request`. + +- Code: `src/analyzer/mod.rs` — split the merged `ForeverLoop | MainLoop` arm so + the concurrent case is checked; extracted the shared body walk into + `analyze_loop_body`. The ordering error is pushed *before* the body is analyzed + so it is not swept up by the handler-body error→warning demotion. +- **Risk class R3** (concurrency/lifecycle). +- **Red→Green:** `test_concurrent_main_loop_requires_wait_for_request_first` + (inline in `src/analyzer/mod.rs`) parses real WFL source and asserts: (a) a + concurrent loop with `store … ` before `wait for request` is rejected naming + the ordering rule; (b) the *same* body under serial `main loop` is accepted; + (c) a concurrent loop that starts with `wait for request` is accepted. Red was + confirmed by neutralizing the guard (test failed), then restored (test passed). +- Docs: `Docs/04-advanced-features/web-servers.md` states the requirement; the + existing `TestPrograms/docs_examples/web_servers/concurrent_main_loop.wfl` + already begins with `wait for request`, so it needed no change. + +## 2. HTTP header maps must have text keys + +**Problem (reviewer, Copilot on `src/typechecker/mod.rs`):** the header type +checks for outbound HTTP (`http … with headers`), streaming responses, and +`respond … and headers` all accepted any `Map<_, _>`. HTTP header names must be +text, so `Map` passed typechecking even though it can never be a valid +header set — and the error message already promised "header names." + +**Fix:** a single `is_valid_header_map_type` helper, used at all four sites, that +accepts a map only when its key type is `Text` (or `Unknown`/`Any`/`Error`, so a +header set the checker cannot fully resolve — map literals often infer an unknown +key — is never falsely flagged) and rejects a map with a concrete non-text key. + +- Code: `src/typechecker/mod.rs` — helper + four call sites collapsed onto it. +- **Red→Green:** `test_header_map_type_requires_text_keys` covers accepted + (`Map`, loose-key maps, `Unknown`/`Any`) and rejected (`Map`, + `Map`, non-map) cases. Because map literals infer `Map` + keys today, a concretely non-text-keyed header map is not reachable from source + — this is a defensive guard, so the honest evidence is a unit test on the + boundary that changed. Red confirmed by broadening the key match, then restored. + +## 3. CI: validate Windows too; ignore the docs cache + +Also on this branch (reviewer, Copilot on `.github/workflows/ci.yml`): the +Integration Tests job ran docs-example validation and the web-server integration +tests on Linux only, and trusted the committed validation cache. Now: + +- Docs validation runs on both OSes with `--force` (ignores the cache so CI + always re-validates). +- The web-server suite runs on Windows too via the existing + `scripts/run_web_tests.ps1` (`shell: pwsh`), matching the testing profile's + requirement that web tests run in CI rather than leaving Windows unvalidated. + +## Residual risk + +- The concurrent-loop guard is a static structural rule, not a runtime rewrite: + it removes the speculative-side-effect footgun by construction, but the loop + still starts its slots eagerly (each parked on `wait for request`). A full + dequeue-then-run engine remains future work. +- Enabling the Windows web-test suite may surface pre-existing Windows-only + behavior; if it does, that is a real signal to fix, not to re-hide. diff --git a/Dev diary/2026-07-24-issue-642-completion.md b/Dev diary/2026-07-24-issue-642-completion.md new file mode 100644 index 00000000..19ca3705 --- /dev/null +++ b/Dev diary/2026-07-24-issue-642-completion.md @@ -0,0 +1,177 @@ +# Dev Diary — 2026-07-24: PR #641 / issue #642 re-review repair + +Issue [#642](https://github.com/WebFirstLanguage/wfl/issues/642) requested a +fresh review of PR #641 from reviewed head +`8e8be0fcde944d0d7b357b94d5951497af5ff0b7`. This pass repaired every newly +confirmed product defect in auditable test-only Red → later Green commits, +replaced false-positive R3 tests with causal tests, and recorded the older +evidence gap without rewriting history. + +No merge or PR comment is part of this work. The previously preserved green +Actions run is `30142079511`; it is not final evidence for the repaired +candidate. Final local gates and a new complete Linux/Windows Actions matrix are +required after the documentation and characterization commit. + +## Risk, compatibility, and gate status + +- **Risk class:** R3. +- **Triggers:** concurrency, cancellation, HTTP lifecycle, streaming, resource + ownership, bounded retention, async control flow, and backward-compatible WFL + grammar/typechecking. +- **Compatibility:** no WFL syntax or public `ErrorKind` variant was removed. + The parser fixes restore ordinary-expression parity and legacy `flush` + behavior; typechecker fixes accept every runtime-viable classic/streaming + branch without changing runtime binding rules. +- **External state:** none. Rollback is a source revert; no data migration is + involved. +- **Policy gate:** unresolved pending maintainer approval of the Section 17 + exception for pre-existing work that lacks retained Red chronology. New + defects found during this re-review do have valid Red ancestry. + +## Implemented behavior + +1. Streaming response `status` parses the full clause-aware expression grammar + without consuming `headers`, `content type`, or `as`. +2. Seeded write/response/flush operands resume postfix composition after `of` + calls, matching ordinary expressions. +3. Bare `type` is no longer treated as a nonexistent response clause boundary. +4. Same-line unmerged `flush` operands reach stream parsing while genuinely + bare legacy bindings/actions keep their old meaning. +5. Repeat, try, and count bodies receive checker child scopes; conditional and + possibly-zero-iteration control flow conservatively joins all runtime-viable + binding types. +6. Locally opened files are recreated as `Custom("File")` when analyzer scope + reconstruction leaves no current checker symbol. +7. A final unterminated outbound line is followed by clean EOF even after the + former absolute deadline. +8. Expired unread streams release the live body, reaper, and handler ownership. + Typed terminal results use at most 64 lightweight records with a 60-second + TTL and are consumed by the next read. +9. The complete buffered/streaming response precommit phase—including the + request operand, actions it calls, all response fields, ownership precheck, + sender take, and transport commit—observes disconnects as + `ErrorKind::Cancelled`. +10. Cancellation drops the active future before restoring action/loop state and + closes only resources opened by that response attempt. Ordinary expression + failures and duplicate/forged response errors retain their prior behavior. + +## Auditable Red → Green ledger + +Every Red below is a test-only ancestor of its Green implementation. Fixture +corrections and Green-first characterization commits are listed separately and +are not represented as Red evidence. + +| Behavior | Affected base | Test-only Red | Green implementation | Focused command | +|---|---|---|---|---| +| Full streaming status operands | `8e8be0fcde944d0d7b357b94d5951497af5ff0b7` | `09115f88b0ba1bcf8ecbdba3ca81ab62eaa07e40` | `99353201518917b350009822554c7d41f6662582` | `cargo test --test write_web_postfix_test -- --nocapture --test-threads=1` | +| Post-`of` postfix continuation | `f23fb6bc0c3b2b77cf1f9eeab567b38032710f9c` | `d97f15b6d9a05be7f35d54b1bbf3d627472ea7d6` | `764685c081f62123a56cf2bbe11aa2b4617d2711` | `cargo test --test write_web_postfix_test -- --nocapture --test-threads=1` | +| Remove false bare-`type` boundary | `764685c081f62123a56cf2bbe11aa2b4617d2711` | `c8cfa08c0352555bd4d302fd4ded21b827e5ceca` | `485bc34b4daad1354b838a74e59938b5041c0db5` | `cargo test --test write_web_postfix_test -- --nocapture --test-threads=1` | +| Reach unmerged flush targets | `485bc34b4daad1354b838a74e59938b5041c0db5` | `55f3d507c741f44576afce24affbf643ee7d258e` | `4a838459bf985611338e69f81253b2a6eee0e269` | `cargo test --test flush_action_backcompat_test -- --nocapture --test-threads=1` and `cargo test --test http_server_streaming_test -- --nocapture --test-threads=1` | +| Checker child scopes | `4a838459bf985611338e69f81253b2a6eee0e269` | `8b10f8bff36fba1df4e6bae0eda07e9f05c16721` | `a1bdd9d75bd3c8134cb0fb49dc601ff209d6c26f` | `cargo test --test typechecker_response_stream_scope_test -- --nocapture --test-threads=1` | +| Conditional/loop type joins | `7bafc6da8682de19886bb3c47cc14e67c5d2b9e2` | `24f57d63dcd7018a1ea31d1f14c63c2e4069a982` | `046b012e8fd9d79a34ee2032fd2ae36da405816d` | `cargo test --test typechecker_response_stream_join_test -- --nocapture --test-threads=1` | +| Recreate local File symbols | `046b012e8fd9d79a34ee2032fd2ae36da405816d` | `a30fe4f50f8beff3d3b3af67aa234723f6d858fb` | `370073e4431af2e3cbad7273bace3ee0ff307e9d` | `cargo test --test open_file_local_type_test -- --nocapture --test-threads=1` | +| Stable clean EOF after final line | `0e98fe35415abe1e067293edfaa47a4509446303` | `5bef23578d0c315dd12b5613e63f6c9192d4e79a` | `af800a7dfe44a6188b591aebfd4f8211d51719e8` | `cargo test --lib interpreter::outbound_stream_deadline_tests::final_unterminated_line_survives_deadline_after_clean_eof -- --nocapture --test-threads=1` | +| Bounded expired-stream state | `af800a7dfe44a6188b591aebfd4f8211d51719e8` | `5d8fa3d6775145f8f63a4684f365f5f2e95c55c4` | `c7f57b9594a7286d57692efa066502fd6c08c16e` | `cargo test --lib interpreter::outbound_stream_deadline_tests::unread_expired_stream_metadata_and_ownership_are_bounded -- --nocapture --test-threads=1` | +| Cancel buffered content and streaming-head evaluation | `00a2a3fa5f60bf414ac211b2c76d546f784b0d49` | `4c45f1617097cbd39f92183bbe9dcbd986cea41d` | `0d4b26b23bcd356bd62fc4de6abf89e062e0279c` | `cargo test --lib interpreter::response_expression_disconnect_tests -- --nocapture --test-threads=1` | +| Cancel request operands; clean precheck and commit races | `edb8ce89c3d693015f655daac012c73cbc12d293` | `3bc38c668a91229e213a10d8eaebdba3789556a9` | `c73260ff61a32694c5ecfe72ab8749810033de0d` | `cargo test --lib interpreter::response_expression_disconnect_tests -- --nocapture --test-threads=1` and `cargo test --lib interpreter::response_disconnect_result_tests -- --nocapture --test-threads=1` | + +The intended Red failures included incomplete/misbounded ASTs, unreachable flush +forms, leaked checker types, unknown local File types, stale-deadline Timeout, +unbounded live stream/owner populations, response evaluation that remained +pending after its client disconnected, stale pending-response ownership, and +upstream streams retained after commit-time cancellation. + +Post-Green fixture corrections were +`f23fb6bc0c3b2b77cf1f9eeab567b38032710f9c`, +`7bafc6da8682de19886bb3c47cc14e67c5d2b9e2`, +`f0dc05db5b2c6722a3a400e629544295a3b07609`, and +`85768c2384b2e414fe66c26751b28f12f2614890`. They correct or broaden +test fixtures; none is claimed as a new Red. + +## R3 characterization and preservation evidence + +- `0e98fe35415abe1e067293edfaa47a4509446303` proves the classic write fallback + with a real opened File handle. +- `90a225d4de36fc74a0b17e4312889c2fa3511c93` makes simultaneous body/expiry + arbitration deterministic. +- `edb8ce89c3d693015f655daac012c73cbc12d293` replaces timing-only lifecycle + coverage with active-read close, spawned-reaper, exact disconnect + classification, zero/fractional timeout, backpressure, and real client + disconnect tests. The real TCP test covers buffered content plus streaming + status/content-type/headers, asserts upstream EOF, and proves `/ping` + remains serviceable. +- `tests/concurrent_disconnect_paths_burst_test.rs` uses causal release markers + and iteration barriers. Each 256-client wave is fully consumed before the + next wave or `/ping`; fixed handler sleeps are not used as proof. +- Green-first breadth checks cover builtin status operands, ordinary/seeded AST + parity and runtime behavior after `of`, a genuinely bare non-callable + `flush` binding, outer Text/File scope reconstruction, and ordinary + expression/error preservation. + +Focused preservation commands run on the repaired tree include: + +```text +cargo test --lib interpreter::response_expression_disconnect_tests -- --nocapture --test-threads=1 +cargo test --lib interpreter::response_disconnect_result_tests -- --nocapture --test-threads=1 +cargo test --lib interpreter::request_wait_timeout_tests -- --nocapture --test-threads=1 +cargo test --lib interpreter::outbound_stream_deadline_tests -- --nocapture --test-threads=1 +cargo test --test response_expression_disconnect_runtime_test -- --nocapture --test-threads=1 +cargo test --test concurrent_disconnect_paths_burst_test -- --nocapture --test-threads=1 +cargo clippy --lib -- -D warnings +``` + +All completed focused commands passed without retry, skip, quarantine, +weakened assertions, or replacement with timing-only assertions. + +## Historical evidence gap + +The original PR work before reviewed head `8e8be0fc` does not have retained +test-only Red ancestors for every behavioral change. Actions run `30106107011` +is the only located durable pre-Green Red artifact for that earlier work. +Writing passing tests now, reverting finished code, or rewriting commit history +would not establish the missing chronology. + +The repository therefore contains a narrowly scoped Section 17 exception draft +under `Docs/development/testing-policy-exceptions/`. It records the exact +missing rule/scope, reason, compensating verification, residual risk, +containment, rollback, owner, repair deadline, and seven-day R3 expiry. It is +explicitly **PENDING MAINTAINER APPROVAL**. Until approved, the testing-policy +merge/release gate remains unresolved; the exception does not turn missing +evidence into a pass. + +## Required final verification + +The final candidate must run these exact commands after all code, tests, and +documentation are committed: + +```text +cargo fmt --all -- --check +git diff --check +cargo clippy --all-targets --all-features -- -D warnings +cargo build --release +cargo test --all --verbose --jobs 2 +scripts/run_integration_tests.sh +python3 scripts/validate_docs_examples.py --ci --force +scripts/run_web_tests.sh +``` + +After push, the new GitHub Actions run must finish successfully across Linux +and Windows, including the integration gate, TestPrograms, docs validation, web +tests, TLS, PostgreSQL, MariaDB, and fuzz-target compilation. Those results +belong in the final handoff rather than being preclaimed here. + +## Residual risk and recovery + +- Recent typed stream terminals are deliberately bounded to 64 records and 60 + seconds. A much later read, or a read after capacity eviction, receives the + documented unknown/closed-handle result rather than retaining metadata + indefinitely. +- A response request operand is raced against the stable set of requests owned + when it begins; normal handlers own one request. Newly accepted resources are + treated as work created by the attempt and are cleaned if it is cancelled. +- The Section 17 approval is an explicit unresolved governance risk, not a + product-test failure. +- No deployment or persistent state changed. A rollback returns the complete + affected change set to its recorded base and reruns the gate; it must not + claim the reverted behavior remains repaired. Forward repair is preferred + for any later race or platform defect. diff --git a/Dev diary/2026-07-24-issue-642-p1-followups.md b/Dev diary/2026-07-24-issue-642-p1-followups.md new file mode 100644 index 00000000..484895c8 --- /dev/null +++ b/Dev diary/2026-07-24-issue-642-p1-followups.md @@ -0,0 +1,61 @@ +# Dev Diary — 2026-07-24: issue #642 PR #641 follow-up P1s + +> **Superseded:** this records the first implementation pass, not the final +> verified state. See `2026-07-24-issue-642-completion.md` for the subsequent +> correctness fixes and strengthened R3 evidence. + +Follow-up to the exact-head re-review of #641 (`b25aed57`). CI was green but five +P1 lifecycle/compatibility blockers remained. Risk class **R3** (concurrency, +cancellation, lifecycle, streaming, compatibility). + +## P1.1 — request-local failures must not stop the concurrent server + +- Sticky `accepted_request` on `RunState` / `IsolatedHandler` output. +- Concurrent loop only feeds structural pre-request failures into the 256-breaker. +- `Cancelled`, `Timeout` (finite `wait for request`), and any post-accept error/panic + are non-structural. +- `wait for request ... with timeout` expiry uses `ErrorKind::Timeout`. +- Missing pending while the handler still owns the request → `Cancelled` (sibling + prune of a disconnected client); duplicate respond when not owned stays General. +- Tests: strengthened `concurrent_disconnect_paths_burst_test` (assert connected + count, 15s drain, pre-streaming-head path, wait-timeout survival). + +## P1.2 — outbound hard-lifetime reaper ownership + +- `StreamSlot` with handle / deadline / expired tombstone / `AbortHandle`. +- Reaper marks expired and drops parked handles; mid-read `put_stream` refuses + reinsertion and returns `Timeout`. +- EOF/error/close abort the reaper timer (bounded open/close cost). +- `outbound_stream_deadline` clamps extreme `u64` config (no Instant panic). +- Tests: `outbound_stream_reaper_race_test` (active-read near deadline + rapid + open/close). + +## P1.3 — ambiguous `write line|chunk` soundness + +- Typechecker validates definedness + payload for the concrete branch; gradual + targets validate **both** branches. +- `main loop` / `forever` typecheck push a scope; `start streaming response` + always binds `ResponseStream`. +- Analyzer walks `PropertyAccess` in the ambiguous-write parallel walker. +- Tests: one-sided undefined classic lead, main-loop list payload, property access. + +## P1.4 — merged operands match ordinary expression grammar + +- `parse_trailing_postfix` gains direct-integer and `at` indexing. +- Shared `parse_merged_operand_from_lead` for write, `content type`, and `headers`. +- Tests: `at` / integer indexing, `content type mime_type of path`. + +## P1.5 — full `flush` expression-statement fallback + +- Full merged name bound as any value (zero-arg action, non-zero-arg function, + overloaded without zero-arg, non-callable) → old expression-statement behavior. +- Only unbound names fall through to stream flush. +- Analyzer/typechecker stay aligned. +- Tests: non-callable binding + parameterized action without zero-arg. + +## R3 test strength + +- Backpressure test asserts error kind + lower timing bound. +- Open-expiry test asserts setup success + lower timing bound. +- `dropped_interpret_server_cleanup_test` covers pending-request 500 without + streaming head. diff --git a/Dev diary/2026-07-24-issue-642-rereview-fixes.md b/Dev diary/2026-07-24-issue-642-rereview-fixes.md new file mode 100644 index 00000000..8197b03c --- /dev/null +++ b/Dev diary/2026-07-24-issue-642-rereview-fixes.md @@ -0,0 +1,55 @@ +# Dev Diary — 2026-07-24: issue #642 re-review fixes + +> **Superseded:** this records an intermediate branch state. The completion +> audit found remaining typed-timeout, EOF, parser, container-property, and test +> evidence gaps. See `2026-07-24-issue-642-completion.md` for the corrected +> design and final evidence. + +Follow-up to the maintainer checklist on the #642 round. + +## CI + +- `cargo fmt --all` unblocked the formatting gate (commit `a2620362`). + +## Runtime lifecycle + +| Item | Fix | +|------|-----| +| Close during active read | `StreamCancel` (AtomicBool + Notify); reads select against it; close/reaper cancel + remove slot | +| Map cleanup from Drop | `std::sync::Mutex` (no silent `try_lock` abandon) | +| Reaper before insert | Insert slot then arm reaper under same lock | +| Tombstones | Finish always **removes** the slot | +| Final unterminated line | Finish after emitting (no parked done+reaper) | +| Respond/stream eval disconnect | `ensure_pending_response_owned` then evaluate; take sender only at commit | +| Stall vs disconnect | Stall write → `ErrorKind::Timeout`; disconnect → `Cancelled` | +| Breaker Timeout exemption | Only messages starting with `Timeout waiting for request` | +| Fractional wait timeout | Reject `0 < ms < 1` | + +## Parser / typechecker + +| Item | Fix | +|------|-----| +| Classic write to `open file` | Accept `Custom("File")` as classic-file target | +| content type / headers clauses | `parse_clause_operand_from_lead` stops at clause connectives | +| flush postfix legacy | Full expression AST fallback on phrase + postfix | +| Parameterized flush | ExpressionStatement semantics → arity error | +| ResponseStream binding | `define_or_replace` in current scope only (shadow) | +| Write definedness | Analyzer `name_is_defined_for_write` shared with typechecker | + +## Red→Green note + +The original #642 landing was a single mixed commit (`5e01e446`, +1399/−271). +Rewriting that history on the already-pushed branch would require a force-push; +this re-review is a **new** Green commit on top with targeted regressions. +Auditable Red-first history for a future rework can soft-reset and re-land as +test-only then fix commits if the maintainer prefers a rewritten PR stack. + +## Tests run (local) + +``` +cargo test --test flush_action_backcompat_test --test write_web_postfix_test \ + --test ambiguous_write_branch_typecheck_test --test concurrent_disconnect_paths_burst_test \ + --test outbound_stream_reaper_race_test --test response_stream_backpressure_test \ + --test outbound_stream_close_during_read_test +``` +All green. diff --git a/Dev diary/2026-07-24-outbound-stream-lifecycle-p1.md b/Dev diary/2026-07-24-outbound-stream-lifecycle-p1.md new file mode 100644 index 00000000..a53dc0b7 --- /dev/null +++ b/Dev diary/2026-07-24-outbound-stream-lifecycle-p1.md @@ -0,0 +1,73 @@ +# Dev Diary — 2026-07-24: outbound-stream lifecycle P1s (deadline + ownership + disconnect) + +The two remaining P1 blockers from the maintainer's streaming re-review, each +closed with a **real-socket** Red→Green regression (mock upstream via +`tokio::TcpListener` + the WFL interpreter), which is the boundary evidence the +Testing Policy requires for R3 concurrency/streaming/lifecycle work. + +## P1-B — the absolute stream deadline must bound an ACTIVE read + +**Bug:** `stream_pull` computed the right per-read bound — +`min(idle_timeout, remaining_absolute)` — and handed it to `run_http_with_budget` +as `configured_timeout`. But `run_http_with_budget`/`outbound_http_deadline` +derived the operation timeout purely from the run/budget duration and *discarded* +`configured_timeout` (using it only as a fallback). So with `timeout_seconds = 10` +and `outbound_stream_max_seconds = 1`, a head-then-stall upstream let +`wait for next chunk` wait ~10s instead of ~1s. The absolute clock also started +only after the head arrived, excluding connect/header time. + +**Fix:** compose the operation deadline as `MIN(configured_timeout, budget)` and +report the stream `Timeout` vs the budget `Deadline` depending on which bound +fired; `configured_timeout` is always finite, so the read is bounded even with no +run-wide budget deadline. Start the absolute clock at request initiation and bound +the head phase by it too. + +**Test:** `outbound_stream_deadline_test` — mock sends head then stalls; read now +fails at ~1s (was ~10s, verified Red). + +## P1-A — outbound streams are handler-owned, and disconnect cancels a blocked read + +Two parts: + +**Part 1 — ownership / close-on-every-exit.** Outbound `httpstream*` handles lived +only in the interpreter-wide `IoClient.stream_handles`; `RunState`/ +`IsolatedHandler::drop` closed downstream response streams and pending requests +but not upstream handles, so an abandoned proxy read leaked the upstream until the +whole interpreter tore down. Now `RunState.open_http_streams` tracks them +per-handler (swapped per poll); added on open, untracked on EOF/error/explicit +close, and dropped from the map on every handler exit (`IsolatedHandler::drop` +for the concurrent loop; `close_open_http_streams()` at the serial-loop/program +cleanup sites) — dropping the reqwest stream cancels the upstream. +*Test:* `outbound_stream_ownership_test` — a run ends without `close` while the +interpreter is still alive; the mock sees its client disconnect only because +handler-exit cleanup cancelled the upstream (verified Red by disabling cleanup). + +**Part 2 — disconnect cancels a blocked upstream read.** A proxy handler blocked +in `wait for next line|chunk` on the upstream had no disconnect signal — it only +noticed at the next downstream `write` (or, after P1-B, the absolute deadline). +The downstream response stream's `mpsc::Sender::closed()` resolves when hyper +drops the client's body receiver, so we clone the handler's open-response-stream +senders (no `RefCell` borrow across the await) and `select!` the upstream read +against "any downstream disconnected". On disconnect the read future is dropped +(cancelling the upstream), the handle is closed, and a catchable `Cancelled` error +unwinds the handler (whose owned cleanup then runs). +*Test:* `outbound_stream_disconnect_test` — a WFL concurrent proxy relays a mock +upstream that stalls after one chunk; the client reads the first chunk and +disconnects while the handler is blocked; the mock observes its own connection +close within the window only because the blocked read was cancelled (verified Red +by disabling the `select!`). + +## Risk class & residual risk + +- **R3** (concurrency/cancellation/lifecycle/streaming). Real-boundary tests, + negative assertions (connection actually closes / read actually fails), and + Red evidence for each. +- Part 2 selects against the handler's currently-open response streams; a handler + with no downstream stream (a pure client-side reader) keeps the plain read path + (`pending` disconnect branch), so there is no behavior change there. +- `close_http_streams` in `Drop` is best-effort via `try_lock`; if the async lock + is momentarily held the handles are reclaimed at interpreter teardown (they no + longer leak *past* that, and in practice the lock is free at handler exit). +- Docs (`interoperability.md`, `response-streaming-design.md`) updated to state the + now-accurate read bound (min idle/absolute), handler-exit release, and the + proactive disconnect cancellation. diff --git a/Dev diary/2026-07-24-streaming-rereview-p1-blockers.md b/Dev diary/2026-07-24-streaming-rereview-p1-blockers.md new file mode 100644 index 00000000..973deb75 --- /dev/null +++ b/Dev diary/2026-07-24-streaming-rereview-p1-blockers.md @@ -0,0 +1,240 @@ +# Dev Diary — 2026-07-24: streaming re-review P1 blockers + +The maintainer's re-review of the streaming/concurrency PR raised a set of P1 +merge blockers on the runtime lifecycle. Each was closed with a **real-boundary** +(real-socket / real-binary) regression under the R3 profile, committed **Red +first** (a failing test-only commit that is an ancestor of the fix commit) per the +Testing Policy. + +## #2 — a client disconnect is a normal cancellation, not a handler failure + +The disconnect branch of a blocked upstream read returned a generic budget error +that the concurrent `main loop` fed into its single global consecutive-failure +breaker (backoff after every failure, break the whole loop at 256). A burst of +256 browser disconnects therefore tore the loop down — an ordinary "client hung +up," repeated, became a denial of service. + +Fix: a distinct `HttpClientError::Disconnected` mapped to a new +`ErrorKind::Cancelled` (still catchable). The concurrent loop recognizes a +`Cancelled` handler outcome as an expected cancellation — it releases the handler +(its owned streams are already closed on unwind) without touching the failure +counter or backing off. Internal budget-cancellation keeps its `ResourceLimit` +kind, so only a real downstream disconnect is exempt. +*Test:* `concurrent_disconnect_burst_test` — 270 disconnects, then `/ping` is +still served (was refused; ~13 s → ~0.9 s). + +## #3 — `outbound_stream_max_seconds` is a TRUE absolute lifetime + +`next_line`/`next_chunk` served locally-buffered bytes before consulting the +handle's absolute deadline (only `stream_pull` checked it), so a proxy that pulled +a multi-line chunk kept draining the buffer past the stream's absolute lifetime. +Fix: `check_stream_deadline` runs before serving buffered bytes; on expiry the +handle is dropped (cancelling the upstream). An empty-buffer read already expired +via `stream_pull`'s identical check. +*Test:* `outbound_stream_absolute_lifetime_test`. + +## #1 — per-request cancellation valid BEFORE and after the head + +The disconnect signal came only from an open downstream response stream, which +does not exist during the upstream HEAD phase (`open url ... and stream response`, +before `start streaming response`). A browser that disconnected while the upstream +withheld its head was not noticed until the head timeout. Fix: a second disconnect +signal from the request's oneshot — the transport drops the receiver when the +client goes away, so the parked sender reports `is_closed()` +(`any_pending_request_disconnected`, polled). `any_client_disconnected` races both +signals; the head open **and** both body reads select against it, so a disconnect +cancels the handler in either phase (dropping the head future aborts the upstream +connection). Empirically hyper drops the pending route future's receiver on a +pre-head disconnect, so the poll observes it. +*Test:* `outbound_stream_head_disconnect_test` — upstream withholds its head, the +client disconnects, the upstream closes promptly, and an unrelated `/ping` is +still served. + +## #4 — a dropped `interpret()` future still closes outbound handles + +Handler-exit / program-cleanup sites close outbound handles on normal exits, but a +dropped/cancelled `interpret()` future runs none of them, leaking a parked +(opened, not mid-read) upstream until the interpreter itself is dropped. Fix: an +RAII `OutboundStreamCleanup` guard held for the whole `interpret_inner` body, +sharing `open_http_streams` and the `IoClient` via `Rc`, so its `Drop` closes the +tracked handles even as the future unwinds and the interpreter stays alive. On a +normal run the exit sites drain the list first, so the guard is a no-op. +*Test:* `dropped_interpret_cleanup_test`. + +## #5 — a bracket index composes after a `.property` / `.method` access + +`store ct as upstream.headers["content-type"]` mis-parsed into two statements +(`store ct as upstream.headers` + a stray `["content-type"]` list literal), +silently dropping the lookup. The identifier property-access / method-call fast +paths returned before the postfix loop could consume the `[...]`. Fix: route both +through `parse_trailing_bracket_index`, folding any chained `[...]` onto the base +(`grid.rows[0][1]`); the shared postfix loop and the static-member `.` arm are +untouched. +*Tests:* `property_index_access_test` (AST + runtime), and the strengthened dot +test in `stream_handle_type_test` (previously a false green — type-checking alone +passed on the split). + +## #6 — drop a span-mismatched classic-write fallback + +The ambiguous `write line|chunk ... to ` form kept the classic +file-write fallback whenever it merely parsed, even if it consumed a **different** +span than the stream reading — so `write line min with a: 1 and b: 2 to ` +retained a partial `line min with a` fallback (the classic reading stops at the +`:` the builtin-call stream reading consumes as a named arg), corrupting the file +write. Fix: keep the fallback only when it consumed exactly to the stream +reading's end checkpoint; otherwise there is no valid classic interpretation and +the non-stream target is a clean error. +*Test:* `write_line_backcompat_test` (two new cases; the matching-span back-compat +cases still pass). + +## #7 (P2) — analyze the shared continuation of a desugared ambiguous write + +The analyzer deferred ALL non-simple ambiguous `write line|chunk` values to +runtime, so an undefined variable in an operator continuation +(`write line value plus missing_suffix to ...`) went unflagged. Replaced the +simple-lead check with `analyze_ambiguous_write`, a parallel walk of the stream +and classic readings (parsed from the same tokens, differing only at the leftmost +leaf): a subtree identical under both readings is pure continuation and analyzed +normally; otherwise the walk recurses on BOTH children so a lead a desugaring +duplicated into the right operand (`is between`) is matched against the fallback's +copy instead of mis-flagged, and at a differing leaf reports undefined only when +NEITHER reading resolves. Call-based desugarings still defer. +*Tests:* `write_line_backcompat_test` (desugared-continuation flag + +operator-continuation no-false-positive), all prior back-compat cases intact. + +## #8 (P2) — streaming-operand type enforcement + flush operand postfix + +- **Operand types.** `wait for next chunk|line` now requires an `HttpStream` + source, and `write line|chunk` / `flush` a `ResponseStream` target (the + ambiguous merged write also accepts a text file-path target for its classic + reading). Unknown/Any/Error still pass for gradual typing, so only a concrete + non-stream operand is rejected — at typecheck instead of as a runtime surprise. + *Tests:* `stream_handle_type_test`. +- **`flush` operand postfix (review feedback).** The lexer merges `flush` with the + following identifier, so `flush streams["a"]` / `flush obj.out` left the accessor + tokens dangling. Generalized the property-then-index helper into + `parse_trailing_postfix` (folds both `[...]` index and `.field` property access + onto a lead) and route `flush`'s split-off lead through it. Also anchored that + helper's missing-`]` end-of-input diagnostic to the `[` token's byte span rather + than the file start (review feedback). *Tests:* `http_server_streaming_test`. + +## Test infrastructure / CI + +- Server integration tests now bind an OS-assigned free port + (`tests/common/free_tcp_port`) instead of a hardcoded constant, removing a + parallel-run flakiness class (review feedback). +- The heavy CI build jobs free ~20 GB of unused preinstalled SDKs on Linux before + building; the debuginfo-heavy release tree plus every integration test binary + was exhausting a runner's disk mid-link (linker `Bus error`/SIGBUS). + +## Risk class & residual risk + +- **R3** (concurrency / cancellation / lifecycle / streaming). Real-boundary + tests, negative assertions (the connection actually closes / the read actually + fails / a burst does not tear the loop down), Red evidence for each fix. +- The pre-head disconnect signal is **polled** (20 ms) because the request's + oneshot sender lives behind an `Arc>>` shared with the + transport, not an awaitable primitive; the downstream-response-stream signal + stays event-driven. A fully idle handler that opens an outbound stream and never + reads it is still reclaimed at handler exit rather than by a mid-idle timer — the + single-threaded, `!Send`-stream model has no wake point to close it earlier; + this is noted rather than claimed as instantaneous. + + +--- + +# Second re-review round — deeper P1 blockers (head `e8c9712`) + +Round of fixes for the maintainer's re-review at `e8c9712`. Risk class **R3** +(concurrency, cancellation, lifecycle, streaming, backward compatibility). Each +behavioral change has a Red→Green real-boundary test; the Red evidence is a +test-only commit that is an ancestor of the source commit (verified by running +each new test with the source fixes stashed). + +## Cancellation / disconnect + +- **`wait for next line` pre-response disconnect (P1-1).** The line read watched + only the downstream response stream, which does not exist before `start + streaming response`; it now races the same combined pending-request/downstream + signal as `wait for next chunk`, so a blocked pre-response line read is + cancelled the moment the client goes away. *Test:* + `wait_line_pre_response_disconnect_test`. +- **Sibling-prune cancellation race (P1-2).** `any_pending_request_disconnected` + treated an owned request id missing from `pending_responses` as "still + connected". But the only removal that leaves an id in `open_pending_requests` + is a sibling `wait for request`'s global prune, which deletes ONLY closed + (disconnected) senders — so a missing owned id is now treated as a terminal + disconnect, and a parked pre-head handler is no longer stranded until its idle + timeout. *Test:* `concurrent_prehead_prune_race_test`. +- **Classify every client disconnect as cancellation (P1-3).** The buffered + `respond`, streaming-head, and response-stream `write` send failures returned a + General runtime error, which fed the concurrent loop's structural-failure + breaker — so a burst of >256 disconnects at those paths tore the loop down. + They are now `ErrorKind::Cancelled`. *Test:* + `concurrent_disconnect_paths_burst_test` (buffered-respond and stream-write + bursts; `/ping` survives). + +## Lifecycle (P1-4) + +- **Absolute outbound lifetime is real-time (a).** `outbound_stream_max_seconds` + was only re-checked on the next read, so an opened-but-unread upstream outlived + the cap. `stream_handles` is now shared via `Arc` and each open spawns a reaper + that drops the handle (cancelling the upstream) when the deadline elapses. + *Test:* `outbound_stream_open_expiry_test`. *Docs:* configuration-reference + updated to state the cap is enforced in real time. +- **Dropped-run cleanup covers server streams + pending (b).** The + interpret-scoped guard covered only outbound streams; the server response + streams and pending requests (`Rc`-shared now) are also finalized on a dropped + `interpret()`, so a cancelled run does not leave a client body hanging on a + reused interpreter. *Test:* `dropped_interpret_server_cleanup_test` (holds the + interpreter alive after the drop to prove it is the guard, not interpreter + teardown, that closes the body). +- **Backpressured write is bounded (c).** `tx.send(bytes).await` past the 64-slot + channel could park forever against a connected-but-non-reading client (a `main + loop` is deadline-exempt). It is now capped by + `web_server_response_timeout_seconds`. *Test:* + `response_stream_backpressure_test` (a >send-buffer payload genuinely blocks; + the write fails at the cap instead of pinning). *Docs:* config reference notes + this timeout bounds streaming writes. + +## Backward compatibility / correctness + +- **Branch-aware ambiguous-write type check (P1-5).** `write line|chunk to + ` has a stream reading and a classic file-write reading; the checker + now validates the reading the runtime actually takes (by the target type), + instead of always checking the stream `value` — so a valid file write is no + longer rejected on the never-run stream branch, a broken file write is caught, + and a concrete non-streamable payload (Map/List/Nothing) to a real stream is a + static error. *Test:* `ambiguous_write_branch_typecheck_test`. +- **`flush` no longer steals a zero-arg action (P1-6).** `flush cache` used to + auto-invoke an action named `flush cache`; the streaming `flush` dispatch now + carries the full merged phrase and the interpreter/typechecker/analyzer prefer + a defined action of that name before treating the operand as a stream. *Test:* + `flush_action_backcompat_test`. +- **Postfix composition on write / web-clause operands (P1-9).** `write line + chunks[0] to out`, `write line upstream.status to out`, `headers + upstream.headers`, and `content type upstream.headers["content-type"]` compose + their trailing `[...]`/`.field` accessors instead of leaving them to dangle. + *Test:* `write_web_postfix_test`. +- **Analyzer walks call/pattern continuations (P1-10).** `analyze_ambiguous_write` + now recurses in parallel through `starts/ends with`, pattern, index, and + function/action/method-call shapes, so an undefined name in a shared + continuation is reported instead of reaching runtime. *Test:* + `ambiguous_write_analyzer_test`. + +## Test infrastructure / CI + +- Clippy runs `--all-features` (matching the binding gate in `testing.md`). +- The Integration Tests job runs the documented + `scripts/run_integration_tests.{sh,ps1}` on both OSes, so the intentional-error + TestPrograms are actually asserted (their assertions lived only in that script, + which CI never invoked). +- `run_integration_tests.ps1` redirects stdout/stderr to two distinct temp files + (PowerShell 7 rejects reusing a single `NUL` target, which left the Windows + integration command unrunnable). +- `run_web_tests.ps1` fails the run — not merely warns — when a server cannot be + killed/waited or a TLS temp dir leaks, since the pass is counted before the + `finally` cleanup. +- Streaming visibility coverage: `response_stream_backpressure_test` also proves + an early chunk is delivered on the wire ~2s before the late one (head/first + chunk visible before body completion, not buffered to close). diff --git a/Dev diary/2026-07-25-parser-streaming-compat-regressions.md b/Dev diary/2026-07-25-parser-streaming-compat-regressions.md new file mode 100644 index 00000000..f87ac18b --- /dev/null +++ b/Dev diary/2026-07-25-parser-streaming-compat-regressions.md @@ -0,0 +1,99 @@ +# Dev Diary — 2026-07-25: Parser streaming compatibility regressions + +This change repairs four backward-compatibility regressions introduced by the +response-streaming grammar: classic `write line`/`write chunk` expressions with +continuations, exact zero-argument actions named `flush`, display folds after +property access, and JavaScript transpilation of ambiguous classic file writes. + +## Risk and compatibility contract + +- **Risk class:** R3. +- **Triggers:** backward-compatible parsing, streaming dispatch, file writes, + action dispatch, and silent output changes. +- **Compatibility contract:** existing classic programs keep their pre-streaming + interpretation while unambiguous response-stream operations remain available. +- **External state:** none beyond temporary files created and removed by tests. + +## Acceptance criteria and coverage + +| Acceptance criterion | Regression coverage | +|---|---| +| A variable literally named `line` or `chunk` remains a classic file-write expression when followed by an operator, `at`, property access, or indexing | `tests/write_web_postfix_test.rs`; `TestPrograms/parser_streaming_compat_regression.test.wfl` | +| Direct integer stream targets such as `write line 0` remain streaming syntax | `tests/write_web_postfix_test.rs` and existing streaming parser tests | +| `flush (expr)` and same-line `flush call ...` invoke an exact zero-argument action named `flush` when one exists | `tests/flush_action_backcompat_test.rs`; `TestPrograms/parser_streaming_compat_regression.test.wfl` | +| Spaced values after property access remain display folds, while adjacent postfix indexing remains indexing | `tests/write_web_postfix_test.rs`; existing `tests/property_index_access_test.rs` | +| JavaScript transpilation uses the valid classic fallback for ambiguous `write line ... to ...`, but still rejects an unambiguous response-stream write | `tests/transpiler_test.rs` | + +## Red → Green record + +The tests were added and run before the corresponding production changes. +The observed Red failures were: + +- `cargo test --test flush_action_backcompat_test`: both new tests failed + because the parser/typechecker treated the target as a response stream + (`Expected a server response stream`). +- `cargo test --test write_web_postfix_test`: six new regressions failed, + reproducing the reported stream-target error, undefined `at`, bracket + misparse, display parse error, and runtime text-indexing error. +- `cargo test --test transpiler_test ambiguous_write_line_uses_classic_file_fallback`: + the transpiler rejected the statement as unsupported streaming. +- The standalone WFL E2E program failed on the same write and flush dispatch + errors. +- Additional direct-binary tests were observed Red for `line + ...` and direct + integer-target discrimination before their parser changes. + +The final Green focused command was: + +```text +cargo test --test write_web_postfix_test --test flush_action_backcompat_test --test transpiler_test --test property_index_access_test --test http_server_streaming_test +``` + +It passed all 123 focused tests. The standalone E2E command +`target\debug\wfl.exe --test TestPrograms\parser_streaming_compat_regression.test.wfl` +passed 4/4 tests. + +This working-tree session records the Red observations but does not claim a +test-only Red commit ancestor; preserving commit-level Red ancestry remains a +maintainer integration responsibility if these changes are committed. + +## Implementation + +The write parser now classifies exact bare `line`/`chunk` markers followed by a +classic expression continuation as classic file writes. For the still +ambiguous direct-integer form, it keeps a span-matched classic fallback while +preserving the response-stream interpretation. + +Exact `flush` forms with a parenthesized target or explicit same-line call now +carry the same exact-binding action fallback as the merged-identifier form. +The existing analyzer/runtime binding check decides between that legacy action +and response-stream flushing. + +Property-origin postfix parsing now requires source adjacency for brackets and +does not consume a spaced integer as an index. Direct and chained adjacent +property indexes continue to work. The JavaScript transpiler emits the classic +file-write fallback when the parser supplied one and continues rejecting +unambiguous streaming writes. + +## Verification + +Completed successfully: + +- `cargo fmt --all -- --check` +- `cargo clippy --all-targets --all-features -- -D warnings` +- `cargo test --all --jobs 2` (complete workspace and doctests) +- `cargo build --release` +- Focused 123-test parser, streaming, property, and transpiler command +- Standalone WFL E2E: 4 passed, 0 failed + +The official integration wrapper's split layer passed 11/11. Its unconstrained +parallel `cargo test --test '*'` then failed during compilation with Windows +error 1455 (paging file too small); no product test failed. The equivalent +complete workspace suite passed with compiler parallelism bounded to two jobs. +The wrapper also emits an existing read-only Cargo cache warning on this host. + +## Residual risk + +The discrimination is deliberately narrow: only exact marker/action bindings +receive compatibility fallbacks, and adjacent postfix syntax remains available. +CI should rerun the official integration and platform matrix on the final +committed candidate. No coverage percentage is claimed. diff --git a/Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md b/Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md new file mode 100644 index 00000000..16483404 --- /dev/null +++ b/Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md @@ -0,0 +1,204 @@ +# Dev Diary — 2026-07-25: Stream terminal and type-flow review fixes + +This pass resolves four R3 findings found while reviewing the PR #641 / issue +#642 candidate: clean EOF retained a live outbound stream indefinitely, +`try` handler state did not reach `finally` in the checker, loop backedges were +not rechecked under later-iteration types, and deferred event/WebSocket handler +types leaked into the registration scope. + +The affected repair base is +`c9c748ce850b7d106ffef90e299e4b7221411517`. The final executable candidate is +`de34e32e513d7d73634b0d7308c681930953a4db`. The latest executable-test +descendant is `81b25745e757671538a833ddf7bc837e19ad83c7`; it adds deterministic +test-only coverage and does not change production code. The documentation-only +commit containing this entry does not change either executable identity. + +## Risk, contracts, and compatibility + +- **Risk class:** R3. +- **Triggers:** streaming lifecycle, cancellation and reaper races, bounded + resource retention, control-flow joins, loop iteration, deferred callbacks, + and backward-compatible typechecking. +- **Lifecycle contract:** observing upstream EOF immediately removes the live + slot, body, owner, and reaper. Only one lightweight `CleanEof` result remains, + sharing the existing recent-terminal bound of 64 records and 60 seconds. +- **Type-flow contract:** `try` success, handler, `otherwise`, and unmatched + endpoints are conservatively joined before `finally`; error aliases remain + clause-local. Loop bodies are checked at a conservative header fixed point. + Deferred event and WebSocket bodies are checked in child scopes. +- **Compatibility:** no syntax, public error variant, or runtime binding rule + was removed. Runtime-viable gradual joins stay permissive, while concrete + invalid later-iteration branches now produce diagnostics. +- **External state:** none. There is no deployment, schema, data migration, + secret, or external-service mutation. + +## Acceptance criteria and exact regressions + +| Acceptance criterion | Exact regression tests | +|---|---| +| A final unterminated line releases all heavy stream state immediately, retains one bounded EOF result, and remains first-wins against later timeout/close cleanup | `final_unterminated_line_survives_deadline_after_clean_eof`; `unconsumed_clean_eof_records_are_bounded`; `observed_clean_eof_wins_over_a_later_deadline_claim`; `put_stream_restores_clean_eof_after_close_removed_the_live_slot`; `put_stream_deduplicates_clean_eof_after_reaper_removed_the_live_slot` | +| Handler and `otherwise` endpoint types reach `finally`, while temporary error aliases do not | `handler_response_stream_state_is_joined_before_finally`; `handler_error_aliases_remain_clause_local_before_finally`; `handler_created_binding_is_semantically_visible_in_finally`; `otherwise_created_binding_is_semantically_visible_in_finally`; `full_pipeline_error_alias_is_clause_local` | +| Later loop iterations are checked after a body changes a target from File/Text-compatible state to `ResponseStream` | `while_loop_rechecks_stream_lead_after_tail_response_stream_rebind`; `repeat_while_loop_rechecks_stream_lead_after_tail_response_stream_rebind` | +| Merely registering a deferred handler cannot overwrite the enclosing checker binding | `event_handler_body_types_do_not_leak_after_registration`; `websocket_handler_body_types_do_not_leak_after_registration` | + +The regressions live in `src/interpreter/mod.rs`'s unit-test module, +`tests/typechecker_response_stream_join_test.rs`, +`tests/typechecker_response_stream_scope_test.rs`, and +`tests/typechecker_try_finally_join_test.rs`. + +## Auditable Red → Green ledger + +Each Red commit below contains tests only and is an ancestor of its Green +implementation. + +| Repair | Red parent / affected base | Test-only Red | Green implementation | Intended Red observation | +|---|---|---|---|---| +| Immediate bounded clean-EOF terminalization | `c9c748ce850b7d106ffef90e299e4b7221411517` | `96d53052388f75bd809c2af42f12445944e8fc69` | `b32ff55fa76fd03b07e2ade7159d3719f2ac0642` | After the final line, the live-slot assertion observed one retained slot instead of zero. | +| Loop-header fixed points, joined `try` endpoints, and deferred-handler isolation | `96d53052388f75bd809c2af42f12445944e8fc69` | `68569b31b9fd969cb5adc3b8c0832ec604bb98e2` | `527b8fb184245e7df35fe5229b23e2a969c74520` | Later-iteration invalid branches were missed, valid `finally` cleanup was rejected, and deferred handler registration changed outer types. | +| First-wins EOF and analyzer/runtime `try`-scope parity | `527b8fb184245e7df35fe5229b23e2a969c74520` | `03966f06e78aec7c3bcdbd40feabc2bdff37a16d` | `de34e32e513d7d73634b0d7308c681930953a4db` | A later timeout displaced observed EOF; handler/`otherwise` bindings were absent in analyzer `finally`; aliases collided in the full pipeline. | + +Commit `81b25745e757671538a833ddf7bc837e19ad83c7` broadens the Green +evidence with deterministic close/reaper missing-slot tests. Those tests use +the real close helper and the reaper's critical-section behavior, then assert +zero live slots and owners, exactly one `CleanEof`, one `nothing` result, and a +subsequent closed/unknown-handle error. + +## Implementation + +`StreamTerminal::CleanEof` now uses the same bounded, expiring, one-shot recent +terminal queue as timeout records. Upstream EOF is the first-wins +linearization point. Returning the final buffered line no longer parks the +completed handle or clears its deadline; `put_stream` drops heavy state, +removes ownership, aborts the reaper, and deduplicates/restores the one +lightweight EOF record even if close or reaper cleanup already removed the +slot. + +The analyzer and typechecker now model the runtime's shared `try` child +environment. Ordinary endpoint bindings are promoted and joined before +`finally`, while the named error and `error_message` aliases are discarded +with their clause scope. `while` and `repeat while` widen entry and backedge +snapshots to a stable conservative header before their diagnostic pass. +Event and WebSocket callback bodies are checked under isolated child scopes, +matching runtime dispatch. + +## Focused and boundary verification + +The following completed without product-test retry, quarantine, assertion +weakening, or timing-only substitution: + +```text +cargo test --lib put_stream_ --jobs 1 -- --nocapture --test-threads=1 +cargo test --lib observed_clean_eof_wins_over_a_later_deadline_claim --jobs 1 -- --nocapture --test-threads=1 +cargo test --lib clean_eof --jobs 1 -- --nocapture --test-threads=1 +cargo test --lib interpreter::outbound_stream_deadline_tests --jobs 1 -- --nocapture --test-threads=1 +cargo test --test typechecker_try_finally_join_test --jobs 1 -- --nocapture +cargo test --test typechecker_response_stream_join_test --jobs 1 -- --nocapture +cargo test --test typechecker_response_stream_scope_test --jobs 1 -- --nocapture +cargo test --test nothing_reassign_widen_test --jobs 1 -- --nocapture +cargo test --test overload_alias_resolution_test --jobs 1 -- --nocapture +cargo test --test http_stream_test --jobs 1 -- --nocapture --test-threads=1 +cargo test --test stream_backpressure_test --jobs 1 -- --nocapture --test-threads=1 +cargo test --test open_file_local_type_test --jobs 1 -- --nocapture +``` + +The respective focused results were 2, 1, 3, 10, 5, 4, 5, 7, 1, 12, 2, +and 3 tests passed with zero failures. + +The official Windows integration runner completed all Rust integration targets +and then reported **110 WFL programs passed, 0 failed, 24 documented skips**. +The official web runner reported **2/2 passed**; its separate certificate-file +journey was explicitly skipped because OpenSSL is unavailable on this host. +The Rust integration suite's eight TLS server tests passed. Forced docs-example +validation reported **18 passed, 0 failed** across validation layers 1–5. +The validator also emitted existing manifest-schema-key warnings; they did not +represent failed examples. + +## Infrastructure interruption and test integrity + +The first `cargo test --all` invocation terminated during rustc compilation +while memory-mapping an rlib with Windows error 1455: the paging file was too +small. No test binary produced a product-test result. The complete unchanged +suite was then invoked with `cargo test --all --jobs 1`, altering only Cargo's +compiler parallelism to bound peak memory. This is an infrastructure rerun +under `testing.md` §8.2, not a product-test retry; no product failure was +retried. + +The initial script invocations inside the restricted process sandbox did not +run product tests: Windows execution policy blocked one integration-script +launch, and a later sandbox process launch could not access Cargo's artifact +database. The same repository scripts were then run once successfully through +PowerShell with `-ExecutionPolicy Bypass` outside that boundary. Existing +ignored Rust tests and the integration runner's documented WFL skips were not +introduced or changed by this repair. + +## Final local gate record + +The final local gate ran on 2026-07-25 against executable candidate +`de34e32e513d7d73634b0d7308c681930953a4db`, test-only descendant +`81b25745e757671538a833ddf7bc837e19ad83c7`, and the final documentation +working tree. The host reported Windows NT `10.0.26200.0`, rustc/cargo +`1.97.0`, and PowerShell `7.6.4`. The official `.ps1` runners executed under +Windows PowerShell `5.1.26100.8875`. + +| Gate | Result | +|---|---| +| `cargo fmt --all -- --check` | Passed. | +| `cargo clippy --all-targets --all-features --jobs 1 -- -D warnings` | Passed. | +| `cargo test --all --jobs 1` | Passed across the complete workspace, integration binaries, LSP, `wflpkg`, and doctests. Existing ignored tests remained reported. Pre-existing unused-code warnings were emitted only by `wfl-lsp` test fixtures; the strict Clippy gate passed. | +| `cargo build --release --jobs 1` | Passed. | +| `git diff --check` | Passed; Git emitted only the repository's Windows LF-to-CRLF working-copy notices for two Markdown files. | +| `$env:CARGO_BUILD_JOBS='1'; ... run_integration_tests.ps1 -TestOnly` | Passed all Rust integration targets and 110 WFL programs; 0 failed and 24 documented programs were skipped by the runner. | +| `... run_web_tests.ps1` | Passed 2/2 runnable journeys; the OpenSSL-dependent certificate-file journey was visibly skipped on this host. | +| `python scripts/validate_docs_examples.py --ci --force` | Passed 18/18 examples with 0 failures. | + +No changed-code warning, product-test failure, retry, quarantine, mute, or +weakened assertion was used to obtain this result. Exact-candidate CI still +must exercise the repository's required platform and service matrix, including +the TLS journey that the local web script could not generate without OpenSSL. + +## Coverage, review, and residual risk + +Coverage was not instrumented, so there is no numeric result. Root +`testing.md` records the absent automated coverage gate as a tracked +conformance gap; no percentage or threshold pass is claimed. This change adds +behavioral, real-boundary, negative, race, and higher-layer regression +coverage. + +An independent Codex review inspected `c9c748ce..de34e32e` for lifecycle and +concurrency correctness, type-flow soundness, compatibility, and test +integrity. Its three Important findings were deterministic close/reaper +missing-slot coverage, explicit 64-record/60-second clean-EOF documentation, +and current candidate chronology. Commit `81b25745` and the accompanying +documentation resolve them. Follow-up review reported no remaining Critical or +Important issue. This is review evidence, not maintainer or security-owner +approval. + +Recent `CleanEof` results intentionally expire after 60 seconds and share a +64-record cap with other terminal results. After expiry, eviction, or +consumption, another read reports an unknown/already-closed handle. Type joins +conservatively widen missing or path-disagreeing bindings to the project's +gradual `Any`/`Unknown` behavior. These are documented design choices. + +PR merge and release remain blocked until the exception record has the required +project/reliability and security-owner approvals, requester identity/date, +maximum-release and expiration acknowledgment, and a successful GitHub Actions +matrix on the final integrated PR head. That head must include the +`81b25745` evidence tests and this documentation; a run on production identity +`de34e32e` alone is insufficient. The older Actions run `30142079511` is Green +evidence for an earlier head, not the final candidate. + +## Rollback + +No external recovery is needed. Revert the complete post-base range beginning +after `c9c748ce850b7d106ffef90e299e4b7221411517`, including its test-only +commits, so intentionally failing Red tests are not left active. Preserve the +original Red and Green commits in repository history as evidence, then run the +complete gate on the rollback candidate. Prefer a forward repair if later work +depends on these compatibility or lifecycle corrections. + +Related durable records are +`Dev diary/2026-07-24-issue-642-completion.md`, +`Docs/development/response-streaming-design.md`, +`Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md`, +the implementation plan under `Docs/superpowers/plans/`, and root +`testing.md`. diff --git a/Docs/04-advanced-features/interoperability.md b/Docs/04-advanced-features/interoperability.md index a1abd312..479f24a2 100644 --- a/Docs/04-advanced-features/interoperability.md +++ b/Docs/04-advanced-features/interoperability.md @@ -105,6 +105,72 @@ request that is waiting on the remote peer. `body` introduce clauses, so use different variable names there (e.g. `request_headers`, `payload`). +#### Streaming a response incrementally + +`read content` / `read response` buffer the whole body before returning. For a +large download, or an upstream that emits output progressively (for example a +model endpoint sending newline-delimited JSON), use `stream response` instead. +It returns as soon as the status and headers arrive — **without buffering the +body** — and binds a streaming handle you pull from piece by piece: + +```wfl +open url at "https://api.example.com/events" + with method "POST" + and headers request_headers + and body payload + and stream response as upstream + +display "Status: " with upstream.status // available immediately +store content_type as upstream.headers["content-type"] + +// Pull the body one line at a time. Each read returns the next line, or +// `nothing` once the stream ends cleanly. +count from 1 to 1000000: + wait for next line from upstream as line + check if line is nothing: + break + otherwise: + display line + end check +end count + +close upstream +``` + +Two incremental reads are available on a streaming handle: + +- `wait for next line from as ` — binds the next + newline-delimited line (the trailing newline, and a paired carriage return, + are stripped). A final line with no trailing newline is still delivered. +- `wait for next chunk from as ` — binds the next raw byte chunk + (`binary`) exactly as it arrives off the network, for non-line-oriented + payloads. + +Both bind `nothing` at a clean end of stream, so `check if line is nothing` +ends the loop. `close ` releases the stream early and cancels the +in-flight upstream request; reading from a closed (or fully-drained) handle +raises a catchable error. + +The same limits as buffered requests apply: the running total of body bytes is +held under `web_server_max_response_size`, and each read is bounded by the +**smaller** of the request's idle timeout and the stream's remaining absolute +lifetime (`outbound_stream_max_seconds`, measured from when the stream is opened, +including connect/header time). So even a trickling or stalled upstream can never +outlive the absolute cap. Cooperative cancellation interrupts a read waiting on +the peer, and a mid-stream network error surfaces as a catchable error from the +`wait for next ...` statement. + +A stream is released — cancelling the in-flight upstream request — when it +reaches a clean end of stream, hits an error, you `close` it explicitly, **or the +handler/program that owns it ends on any path** (normal return, a caught error, a +panic contained by `main loop concurrently:`, timeout, or cancellation). In a web +server that proxies an upstream, a **downstream client disconnect** also cancels a +read that is *currently blocked* on the upstream — the handler wakes with a +catchable error instead of waiting out the absolute deadline — and its upstream is +closed as the handler unwinds. Still prefer an explicit `close upstream` when you +stop early or break out of the read loop before EOF, to free the connection the +moment you are done rather than at handler exit. + ### 4. **Web Standards** WFL web servers work with standard HTTP: diff --git a/Docs/04-advanced-features/web-servers.md b/Docs/04-advanced-features/web-servers.md index 41f33356..364e612a 100644 --- a/Docs/04-advanced-features/web-servers.md +++ b/Docs/04-advanced-features/web-servers.md @@ -68,6 +68,73 @@ wait for request [that] comes in on as This blocks until a request arrives, then stores the request in the variable. Both "comes in" and "that comes in" are supported. +### Concurrent request handling + +A typical server wraps `wait for request` … `respond to` in a `main loop`: + +```wfl +listen on port 8080 as web_server +main loop: + wait for request comes in on web_server as req + respond to req with "Hello!" +end loop +``` + +A plain `main loop` is **serial**: it fully handles one request — including any +`wait for`, outbound call, or stream — before accepting the next. That is simple +and predictable, but one slow handler (say, proxying a slow model stream) makes +every other request wait behind it. + +Write `main loop concurrently:` to handle requests **concurrently** instead: + +```wfl +listen on port 8080 as web_server +main loop concurrently: + wait for request comes in on web_server as req + // ... a slow handler here no longer blocks other requests ... + respond to req with "Hello!" +end loop +``` + +Each iteration runs in its own isolated scope, so concurrent requests never +clobber each other's variables. A login, a health check, or another chat is +served while a slow stream is still running. + +**Important — concurrent, not parallel:** this is *cooperative* concurrency on a +single thread. Handlers interleave at their `await` points (`wait for`, outbound +HTTP, stream reads/writes, `respond`). A handler doing tight CPU-bound work with +no such pause still holds the thread until it yields — concurrency helps I/O- +bound work (the common web case), not CPU-bound loops. + +**What you get with `concurrently`:** + +- A slow handler does not block its siblings. +- Each request handler is isolated (its own scope). +- A handler that errors or panics is contained: that request fails on its own and + the server keeps serving everyone else. How the client sees the failure depends + on how far the handler got: if it had **not** sent a response yet, the client + gets a `500` (or a `504` if the handler never answers in time); if it had already + called `start streaming response`, the status and headers are on the wire + (typically `200`), so the error cannot change them — the response body is just + ended early (the stream is closed), leaving the client a truncated body under the + already-sent status. +- In-flight work is bounded; the transport still sheds excess load with 503 and + times out a stalled handler with 504, exactly as for the serial loop. + +Plain `main loop` keeps its exact serial behavior — adding `concurrently` is the +only way to opt in; nothing changes silently. + +> **A `main loop concurrently:` body must begin with `wait for request`.** A +> concurrent loop starts its handler slots up front, each running the body from +> the top, so any statement placed *before* the first `wait for request` would +> run once per slot before a single request arrives. WFL rejects that at analysis +> time with a clear error. Put per-server setup **above** the loop; the loop body +> starts by waiting for the next request. (Serial `main loop` has no such +> requirement — it runs one iteration at a time.) + +> `concurrently` is only special right after `main loop`; it is not a reserved +> word, so existing programs that use `concurrently` as a name keep working. + ### Responding to Requests Use `respond to` to send HTTP responses: @@ -412,6 +479,105 @@ respond to req with "Created!" and status 201 and content_type "application/json All optional clauses (`status`, `content_type`, `headers`) can appear in any order after the content. +### Streaming a response + +`respond to` sends the whole body at once. When the body is large, or produced +progressively (for example newline-delimited JSON streamed to a browser's +`fetch()` reader), start a **streaming response** instead. It sends the status +and headers immediately and binds a stream handle you write to piece by piece: + +```wfl +start streaming response to req with status 200 and content type "application/x-ndjson" as out + +write line "{\"event\": \"start\"}" to out +write line "{\"event\": \"tick\", \"n\": 1}" to out +flush out +write line "{\"event\": \"done\"}" to out + +close out +``` + +- `start streaming response to [with status ] [and content type ] + [and headers ] as ` — begin the response. The status defaults to 200 + and the content type to `application/octet-stream`. The body has no declared + length (no `Content-Length`); it is streamed incrementally as you write (over + HTTP/1.1 that is chunked transfer-encoding; HTTP/2+ frames it without a + `Content-Length` instead). +- `write line to ` — write `value` followed by a newline (ideal for + NDJSON). `value` may be text, a number, or a boolean. +- `write chunk to ` — write raw bytes verbatim, no newline added. + `value` may be text or `binary` (a number or boolean is also accepted and + written as its text form). + + > **Note (backward compatibility).** `write line to ` / + > `write chunk to ` shares its surface with the classic file write + > `write to `, and WFL allows space-separated variable names, so + > the form `write line to ` is ambiguous. Only that ambiguous + > **bare-identifier** form carries a fallback: if `` turns out to be a file + > handle or path rather than a streaming handle, it runs the classic file write + > (so a variable literally named `line …`/`chunk …` keeps working). The + > unambiguous forms — a literal, number, or boolean value (e.g. + > `write line "x" to out`) — are **stream-only** and error if `` is not a + > streaming-response handle. + > + > **Value operators.** The value accepts `with`-concatenation and the usual + > arithmetic/comparison operators directly in the statement — e.g. + > `write line prefix with json to out`. (An identifier-led value is a variable + > or `field of object` followed by such operators; postfix accessors on the + > leading identifier — indexing `payload[1]` or property access `payload.field` + > — are composed onto it, so `write line chunks[0] to out` and `write line + > upstream.status to out` work.) For the ambiguous bare-identifier form, + > the continuation applies to both readings: `write line note with "!" to out` + > streams `note` + `"!"`, while the same statement targeting a file writes the + > variable `line note` + `"!"` — so pre-existing classic file writes that + > concatenate keep working. +- `flush ` — advisory: yield so queued bytes are handed to the socket. + (Chunks are already forwarded as you write them; hyper writes as it receives.) +- `close ` — end the response body. Writing after `close` is an error. + +**Lifecycle & backpressure:** the body channel is bounded, so a slow client +slows your `write` calls (backpressure) instead of buffering without bound. If +the client disconnects, hyper drops the response body and your next `write` to +that stream fails with a catchable error — use `try`/`catch` to detect it and +stop producing (and `close` any upstream you are proxying). + +**Prefer an explicit `close out`** to finalize the response promptly — that is +what signals the end of the body to the client, and doing it as soon as you are +done frees the connection without waiting. As a safety net, the stream is also +**closed automatically when the handler ends on any path** (normal return, a +caught error, or a panic contained by `main loop concurrently:`), so a handler +that forgets `close out` still finalizes the client's body rather than leaving +it hanging. For long-lived handlers, still `close` as soon as you are finished — +and put it in a `finally:` block if the handler can error partway through — so +the client is not left waiting until the handler happens to return. + +**Proxying an upstream to the browser** — combine with the outbound streaming +client ([Interoperability → Streaming a response +incrementally](interoperability.md#streaming-a-response-incrementally)): + +```wfl +open url at "https://model.example.com/generate" with method "POST" and body prompt and stream response as upstream +start streaming response to req with status 200 and content type "application/x-ndjson" as out + +count from 1 to 1000000: + wait for next line from upstream as line + check if line is nothing: + break + otherwise: + write line line to out + end check +end count + +close upstream +close out +``` + +> **Concurrency note:** a plain `main loop` handles one request at a time, so a +> long-running stream occupies the handler until it finishes. To let a slow +> stream run without blocking other requests (login, history, health checks, +> other chats), opt into [concurrent handling](#concurrent-request-handling) +> with `main loop concurrently:`. + ## The QUERY Method (RFC 10008) WFL supports [RFC 10008](https://www.rfc-editor.org/info/rfc10008/), the HTTP diff --git a/Docs/development/concurrency-phase-plan.md b/Docs/development/concurrency-phase-plan.md index ee0d8c02..1c9871a3 100644 --- a/Docs/development/concurrency-phase-plan.md +++ b/Docs/development/concurrency-phase-plan.md @@ -46,9 +46,60 @@ HARD RULES: | 0 | 0a | Docs honesty + `panic=unwind` CI | ✅ Done | | 0 | 0b | `spawn_blocking` for blocking crypto | ✅ Done | | 0 | 0c | Bound accept/queue (OOM shed) | ✅ Done | -| 1 | 1a | Runtime spike (bridge, no surface) | ⬜ Not started | -| 1 | 1b | `main loop concurrently:` surface + ops defaults | ⬜ Not started | -| 1 | 1c | Honesty docs for real concurrent model | ⬜ Not started | +| 1 | 1a | Runtime spike (bridge, no surface) | ✅ Done (folded into 1b) | +| 1 | 1b | `main loop concurrently:` surface + ops defaults | ✅ Done — awaiting maintainer review | +| 1 | 1c | Honesty docs for real concurrent model | ✅ Docs landed — some 1b/1c checklist items still open (request-ID logging; RefCell shutdown/signal-path audit), see notes below | + +> **Phase 1 landed in one change** (`Dev diary/2026-07-22-concurrent-request-handlers.md`), +> not the staged 1a→1b→1c sequence. What is covered: `main loop concurrently:` +> surface (locked marker); plain `main loop` byte-compatible serial (tested); +> `FuturesUnordered` of `!Send`, `&self`-borrowing handler futures on the +> existing runtime; per-request isolation of both the **environment** scope +> *and* interpreter run-state (see the run-state note below) — note that a +> handler's environment is a fresh child of the shared parent, so top-level +> (global) bindings and any shallow-shared collections reachable through them +> remain shared, by design; a slow handler not blocking a fast sibling (tested); +> in-flight cap (`CONCURRENT_HANDLER_LIMIT`), with 503/504/500 provided by the +> transport plus the interpreter's per-handler exit sweep (bounded queue → 503, +> response deadline → 504, `ResponseCompletion` drop → 500 mid-`respond`, and a +> handler that dequeues a request and ends **without** responding → an immediate +> 500 rather than waiting out the request timeout — *this immediate-500 case is +> covered by a dedicated test*; the 503/504 cases are transport-provided and are +> **not** exercised by a dedicated Phase-1 test); the empty-set +> busy-spin trap is avoided (cap ≥ 1 keeps the set non-empty). Cooperative, not +> parallel: handlers interleave only at await points, so a CPU-bound handler with +> no await still holds the interpreter thread (documented in `web-servers.md`). +> +> **Containment:** *runtime-error* containment is tested (an erroring handler +> does not kill the server). *Panic* containment is by construction via +> `catch_unwind` on each handler future but is **not** covered by a dedicated +> test — a deterministic WFL-level Rust panic is not readily expressible from +> the language, so the guarantee rests on the wrapper + the `panic = "unwind"` +> gate, not a regression test. +> +> **Run-state isolation (review gap — FIXED):** the concurrent loop already +> isolates the **environment** per handler; it now also isolates interpreter +> run-state. Each handler carries its own `RunState` (`current_count`/ +> `in_count_loop`, `call_depth`, `call_stack`, and the block overload-dup set), +> and an `IsolatedHandler` poll wrapper swaps that state into the interpreter +> only for the duration of each `poll`, swapping it back out the instant the poll +> returns (ready *or* pending). While a handler is suspended at an `await`, its +> count-loop/recursion/call-stack bookkeeping is parked in its own `RunState`, so +> a sibling polled next neither sees nor clobbers it. Serial execution is +> untouched (the wrapper is used only by `execute_concurrent_main_loop`). +> Regression: `tests/concurrent_main_loop_test.rs::test_concurrent_handlers_do_not_share_count_loop_state` +> (Red without the swap: `/a` observed `/b`'s entire count range). +> +> **Also lighter than the full 1b checklist (open items):** request-ID +> *structured* logging is not yet added; the eval-core `RefCell`-across-await +> audit is **only partially covered** — the crate-wide +> `#![deny(clippy::await_holding_refcell_ref)]` backstop catches borrows held +> across `.await`, but the **shutdown/signal lifecycle paths** (dropped borrows on +> teardown, e.g. `close server`'s `web_servers.borrow_mut()` held across a short +> await) are **not yet separately audited or tested**. Treat the RefCell audit as +> **open**, with the clippy lint as supplementary — not complete — coverage. +> +> **This is the maintainer STOP/review point** — please review before Phase 2. | 2 | 2a | Structured nursery + join engine | ⬜ Not started | | 2 | 2b | `change shared` critical region | ⬜ Not started | | 3 | 3a | Multi-process workers (if profiling forces) | ⬜ Deferred | @@ -203,31 +254,44 @@ HARD RULES: **Goal:** User-visible opt-in concurrent loop; serial path untouched. +> **Status:** shipped in the single Phase 1 change (see the tracker note above). +> The checklist below is updated to the as-shipped state; `[~]` marks items +> provided by the existing transport layer rather than net-new here, and the two +> remaining known gaps are called out explicitly. + #### TODOs — language / runtime -- [ ] Parser: `main loop concurrently:` (and matching `end`) -- [ ] Analyzer / typechecker / keyword docs if needed -- [ ] Runtime: concurrent path uses proven 1a bridge -- [ ] **G1:** plain `main loop` remains serial and byte-compatible -- [ ] Isolated-per-request scopes (default) -- [ ] Semaphore / in-flight cap (default e.g. 256) → shed **503** -- [ ] Per-request timeout (default e.g. 30s) → **504** (only at await points — document cliff) +- [x] Parser: `main loop concurrently:` (and matching `end`) +- [x] Analyzer / typechecker / keyword docs if needed +- [x] Runtime: concurrent path uses proven 1a bridge +- [x] **G1:** plain `main loop` remains serial and byte-compatible +- [x] Isolated-per-request scopes (default) — plus per-handler run-state isolation +- [~] Semaphore / in-flight cap (default e.g. 256) → shed **503** (cap in the + concurrent loop; 503 shedding from the transport's bounded queue) +- [~] Per-request timeout (default e.g. 30s) → **504** (transport response + deadline; await-point cliff documented) - [ ] Request-ID structured logging on accept / complete / fail / shed / timeout -- [ ] catch_unwind boundary → **500**, siblings survive -- [ ] Eval-core audit: every `RefCell` borrow/borrow_mut on await paths drops before `.await` - - [ ] PR description lists each site and drop-before-await story -- [ ] clippy `await_holding_refcell_ref` enabled/enforced where applicable (backstop only) + — **known gap** (not yet added) +- [x] catch_unwind boundary → **500**, siblings survive +- [~] Eval-core audit: every `RefCell` borrow/borrow_mut on await paths drops + before `.await` — **partial**: the clippy backstop catches await-holding + borrows, but the shutdown/signal lifecycle paths are not yet separately audited + (see the "open items" note above); treat as open + - [~] PR description lists each site and drop-before-await story (mechanical + lint in lieu of a written per-site walkthrough) +- [x] clippy `await_holding_refcell_ref` enabled/enforced where applicable (backstop only) #### TODOs — tests (write first where possible) -- [ ] Serial `main loop` still processes one request at a time (no silent upgrade) -- [ ] Concurrent loop: slow handler does not block fast sibling -- [ ] Cap exceeded → 503 -- [ ] Timeout → 504 -- [ ] Panic in A → 500; B still completes -- [ ] No empty-set busy-spin -- [ ] Request IDs present in logs (if testable) -- [ ] Existing web TestPrograms still pass on serial path +- [x] Serial `main loop` still processes one request at a time (no silent upgrade) +- [x] Concurrent loop: slow handler does not block fast sibling +- [~] Cap exceeded → 503 (transport-provided; not a dedicated Phase 1 test) +- [~] Timeout → 504 (transport-provided; not a dedicated Phase 1 test) +- [ ] Panic in A → 500; B still completes — **known gap**: only *runtime-error* + containment is tested; panic containment is by construction (see tracker note) +- [x] No empty-set busy-spin (cap ≥ 1 keeps the set non-empty) +- [ ] Request IDs present in logs — **known gap** (tied to the logging item above) +- [x] Existing web TestPrograms still pass on serial path #### TODOs — docs (minimum for ship; full honesty pass may be 1c) diff --git a/Docs/development/response-streaming-design.md b/Docs/development/response-streaming-design.md new file mode 100644 index 00000000..d0c5bfa3 --- /dev/null +++ b/Docs/development/response-streaming-design.md @@ -0,0 +1,224 @@ +# WFL Response Streaming — Design & Status + +**Audience:** WFL maintainer + AI implementers +**Origin:** downstream request (2026-07-22) for a browser chat UI proxying a slow +upstream model endpoint to the browser without buffering. + +This tracks the five requested runtime capabilities, what has shipped, and the +locked design for what remains. It complements — and defers to — +`concurrency-phase-plan.md`, which governs item 4. + +--- + +## The five capabilities + +| # | Capability | Status | +|---|------------|--------| +| 1 | Outbound response streaming (`stream response as`) | ✅ Shipped | +| 2 | Incremental reads (`wait for next chunk\|line`) | ✅ Shipped | +| 3 | Streamed server responses (start / write / flush / close) | ✅ Shipped | +| 4 | Concurrent request handlers | ✅ Shipped (`main loop concurrently:`; Phase 1 of `concurrency-phase-plan.md`, awaiting maintainer review) | +| 5 | Lifecycle (timeouts, backpressure, cancellation, catchable errors, close-on-exit) | ✅ Client + server streaming + per-handler isolation/containment shipped | + +--- + +## Shipped (items 1, 2, client-side 5) + +See `Dev diary/2026-07-22-outbound-response-streaming.md`. Surface: + +```wfl +open url at "" [with method .. and headers .. and body ..] and stream response as upstream +wait for next line from upstream as line // Text, nothing at clean EOF +wait for next chunk from upstream as chunk // Binary, nothing at clean EOF +close upstream +``` + +Handle model: an opaque id into `IoClient.stream_handles`, wrapped in an object +exposing `status`/`ok`/`headers`/`_stream`. Reads go through the existing +`run_http_with_budget` select (timeouts + cancellation), enforce the +response-byte ceiling on the running total, are individually catchable, and drop +the handle (cancelling the upstream) on EOF/error/close/teardown. + +--- + +## Item 3 — Streamed server responses (✅ shipped) + +Shipped as designed below. Surface: `start streaming response to [with +status ] [and content type ] [and headers ] as `, `write +line|chunk to `, `flush `, `close `. See +`tests/http_server_streaming_test.rs` and the web-servers guide's "Streaming a +response" section. The original design (kept for reference): + +## Item 3 — Streamed server responses (design) + +### Surface (chosen for consistency with the client side + existing `respond`) + +```wfl +// Send status + headers immediately; body stays open. +start streaming response to req with status 200 and content type "application/x-ndjson" as out + +// Write body pieces. `write line` appends a newline (NDJSON-friendly); +// `write chunk` writes raw bytes/text verbatim. +write line json_text to out +write chunk raw_bytes to out + +// Advisory: hand queued bytes to the transport (yields to the runtime). +flush out + +// End the response body. +close out +``` + +- `start streaming response` leads with the merged identifier `start streaming` + followed by the `response` keyword — dispatched like `send websocket message` + in `parser/mod.rs`. (Distinguishable from any future Phase-2 `start + as `.) +- `write line|chunk to ` — branch inside the `write` dispatch on a + following `line`/`chunk` identifier; `to` (not `into`) distinguishes it from + file writes. +- `flush ` — leading identifier `flush`. +- `close out` — reuse `CloseFileStatement`, extended for a `_server_stream` + object (as it already was for `_stream`). + +### Mechanism + +- **Reply payload becomes an enum.** Replace `oneshot::Sender` + with `oneshot::Sender` where + + ```rust + enum HandlerReply { + Buffered(WflHttpResponse), + Streaming { status: u16, content_type: String, + headers: HashMap, + body: mpsc::Receiver> }, + } + ``` + + Update `PendingResponseSender`, `WflHttpRequest.response_sender`, + `ResponseCompletion` (its `Drop` sends `Buffered` 500), and the `respond` path + (builds `Buffered`). + +- **Transport reply type becomes `warp::hyper::Body`.** Convert the five reply + helpers (`overloaded_response`, `plain_status_response`, + `payload_too_large_response`, `gateway_timeout_response`, + `request_timeout_response`), `handle_overloaded`, and the main route's + buffered arm from `Response>` to `Response` (`.body(Body::from( + bytes))`). The streaming arm builds + `Body::wrap_stream(futures_util::stream::unfold(rx, ...))` (no new dep). The + separate redirect `warp::serve` route is untouched. + +- **`start streaming response`** creates a bounded `mpsc::channel::>` + (capacity = backpressure knob, align with existing web limits), sends + `HandlerReply::Streaming { head.., body: rx }` over the request's oneshot, and + stores the `tx` in a new interpreter-side map + `server_response_streams: RefCell>`. Binds + `out` = object `{ _server_stream: }`. + +- **`write line|chunk`** resolves `_server_stream`, `tx.send(bytes).await` + (bounded → backpressure). A closed receiver (browser disconnected / hyper + dropped the body) makes `send` fail → surfaced as a catchable error. In + addition, a handler blocked in an upstream `wait for next line|chunk` no longer + has to wait for its next `write` to notice the disconnect: the read is + `select!`ed against `Sender::closed()` for the handler's open response streams, + so a downstream disconnect cancels the blocked upstream read promptly (dropping + the upstream), and the handler's owned outbound handles are then closed as it + unwinds. This is how browser-disconnect cancellation propagates to the upstream + (item 5, cooperatively). + +- **`flush`** = advisory; `tokio::task::yield_now().await` so the transport task + is scheduled. Documented as advisory (hyper already writes as it receives). + +- **`close out`** drops the `tx` → ends the body stream → hyper finalizes the + response. Idempotent; writes after close fail predictably. + +### Lifecycle (item 5, server side) + +- Timeouts: the existing per-request `overall_deadline` bounds head delivery; a + stalled *body producer* is bounded by the handler timeout at await points (the + yield-cliff caveat from the concurrency plan). Outbound reads are additionally + bounded by `min(idle, remaining absolute stream deadline)`: `run_http_with_budget` + composes the operation deadline as the minimum of the run/budget remaining time + and the caller's configured timeout (which already carries the stream's idle + + absolute bound), and the absolute clock starts at request initiation. +- Backpressure: bounded `mpsc` — a slow browser slows the handler's `write`. +- Disconnect → upstream cancel, in BOTH phases: a blocked upstream operation is + `select!`ed against `any_client_disconnected`, which fires on EITHER an open + downstream response stream's `Sender::closed()` (post-`start streaming + response`, event-driven) OR the request's oneshot receiver being dropped + (pre-`start streaming response` — the head phase — polled via + `is_closed()`). So a browser disconnect cancels the handler whether it is + blocked opening the upstream head or reading its body, dropping the upstream. +- Whole response evaluation is also cancellation-aware. The runtime snapshots + handler state and owned resources before evaluating the `respond` or + `start streaming response` request operand, races that operand and every + fallible response field against the pending request's disconnect signal, and + carries the same snapshot through the transport commit. Cancellation drops + the active evaluation future first, then restores action/loop state and closes + only outbound streams, response streams, and pending requests created by that + response attempt. This covers actions that sleep or perform buffered work, not + only outbound streaming operations that already observe disconnects. +- A disconnect discovered by the ownership precheck, sender take, or final + oneshot send follows the same cleanup path and returns + `ErrorKind::Cancelled`. Ordinary expression failures and duplicate/forged + response errors retain their existing classifications. +- Disconnect is a normal cancellation, not a handler failure: it unwinds with + `ErrorKind::Cancelled`, which the concurrent `main loop` treats as an expected + outcome (it does NOT feed the structural consecutive-failure breaker), so a + burst of disconnects cannot tear the loop down. +- Absolute lifetime (`outbound_stream_max_seconds`) is enforced before EVERY + read return — including reads served from locally-buffered bytes — not only on + a network read, so a buffered drain cannot outlive the stream's absolute cap. +- Expiry removes the heavy live stream slot, aborts the body, and immediately + removes handler ownership. To preserve one follow-up typed terminal result + without an unbounded tombstone table, the registry keeps at most 64 recent + lightweight terminal records (clean EOF and timeout records combined), each + for at most 60 seconds; reading a record consumes it. Active readers share a + first-wins terminal signal, and the post-`select!` recheck makes expiry win + over a simultaneously ready body chunk. +- Clean EOF terminalizes the stream as soon as the runtime observes upstream + EOF: it removes the live slot and handler ownership, aborts the reaper, and + retains only a one-shot `CleanEof` record in that bounded recent-terminal + queue. A final unterminated line is returned once. The next read returns + `nothing` only if its `CleanEof` record is still retained; if the record has + expired after 60 seconds, was evicted by the 64-record bound, or was already + consumed, the read reports `Unknown or already-closed stream handle`. +- Outbound close-on-exit (shipped): outbound `httpstream*` handles are also + handler-owned — tracked in `RunState.open_http_streams` (swapped per poll) and + dropped from `IoClient.stream_handles` when the handler ends on any path, + cancelling the in-flight upstream request so an abandoned proxy read never leaks. + A cancelled/dropped `interpret()` future (an embedder cancels the run) also + closes them via an RAII guard, rather than leaking until interpreter teardown. +- Close-on-exit (shipped): each handler tracks the `respstream*` ids it opened in + its per-handler run-state (`open_response_streams`, part of the `RunState` + swapped in/out per poll under `main loop concurrently:`). When the handler ends + on **any** path — normal return, caught error, or a panic contained by + `catch_unwind` — those ids are removed from `server_response_streams`, dropping + their `tx` and ending the response body. Concurrent handlers close via + `IsolatedHandler`'s `Drop`; the serial `main loop` drains per iteration; a + top-level stream closes at program exit. So a handler that forgets `close out` + never leaves the client hanging, and the table cannot leak dead senders. + Explicit `close out` is still preferred to finalize promptly. + +### Tests (write first) + +Parser tests for all four statements; runtime tests via a client that reads the +streamed body: status/headers arrive before the body, `write line` frames NDJSON, +`close` ends the stream, a dropped client makes `write` fail catchably. + +--- + +## Item 4 — Concurrent handlers + +Governed by `concurrency-phase-plan.md` (Phase 1, locked marker `main loop +concurrently:`, no Rc→Arc, TDD-first). It is the keystone that makes a slow +streamed response (items 1–3) not stall login/history/health. Server streaming +(item 3) is deliberately usable on the serial loop first; true isolation between +a slow stream and other requests arrives with Phase 1. + +--- + +## Related + +- `Docs/04-advanced-features/interoperability.md` — user docs (client streaming shipped) +- `Docs/development/concurrency-phase-plan.md` — item 4 governance +- `Dev diary/2026-07-22-outbound-response-streaming.md` diff --git a/Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md b/Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md new file mode 100644 index 00000000..e4ab6e9a --- /dev/null +++ b/Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md @@ -0,0 +1,275 @@ +# WFL testing-policy exception draft: PR #641 Red chronology + +> **STATUS: DRAFT — PENDING MAINTAINER APPROVAL** +> +> This exception is not active. It does not turn missing Red evidence into a +> pass. PR #641's merge and release gates remain unresolved until every approval +> condition and signature below is complete. Silence, a green test run, or merge +> authority alone is not approval. + +## Exception record + +| Field | Value | +|---|---| +| Exception ID | `WFL-TEST-EXC-2026-07-24-PR641` | +| Repository | `WebFirstLanguage/wfl` | +| Change record | [PR #641](https://github.com/WebFirstLanguage/wfl/pull/641) | +| Repair ticket | [Issue #642](https://github.com/WebFirstLanguage/wfl/issues/642), item 11 | +| Risk class | **R3** — concurrency, cancellation, lifecycle, streaming, backward compatibility, untrusted archive input, and release-test controls | +| Requested start | `2026-07-24T00:00:00-05:00` (`America/Chicago`) | +| Expiration | `2026-07-31T00:00:00-05:00` (`America/Chicago`), exactly seven days after the requested start | +| Maximum affected releases | **One** WFL release: the first release containing the approved candidate, and no later release | +| Affected base | `b25aed57ea50697c596796446d1f47466668773d` | +| Commits containing the earlier mixed Green work | `5e01e446ab9250d72a0f255bc81a27a79c5b5d63`, `fce5d86fe923666885e40ec484d902cfd18c4c85`, and `8e8be0fcde944d0d7b357b94d5951497af5ff0b7` | +| Exact executable candidate for this draft | `de34e32e513d7d73634b0d7308c681930953a4db` | +| Latest executable-test evidence descendant | `81b25745e757671538a833ddf7bc837e19ad83c7` (test-only; no production code) | +| Requested project/reliability owner approval | Brad, Maintainer, Logbie LLC — **PENDING** | +| Requested security-owner approval | Brad, Maintainer, Logbie LLC — **PENDING**, required for the archive-path item | + +The exact executable candidate above is the latest production-code commit +covered by this draft. Evidence-only test or documentation descendants do not +expand the affected production scope. Any later executable change, force-push, +or different candidate invalidates this draft until the SHA, scope, evidence, +and approvals are updated and reviewed again. + +If approval occurs after the requested start, the exception becomes active only +at the recorded approval time and still expires at the fixed expiration above. +It never applies retroactively to authorize an earlier merge or release. + +## Exact rule and affected scope + +This draft requests a temporary exception only from the retained chronology +requirements in root `testing.md`: + +- Section 6.1, which requires the Red step before the Green implementation; +- Section 6.2, which requires a test-only Red ancestor or independently + timestamped pre-Green artifact tied to the affected base and requires the + base, Red, and Green identifiers; and +- Section 15, only the requirement to attach retained Red evidence for each + behavior or defect fix. + +It does **not** waive the R3 classification, any required test layer, Section +11.3 concurrency/lifecycle coverage, independent review, a required CI job, a +known product failure, or any non-waivable condition in Section 14. A required +test may not be skipped, retried to manufacture Green, quarantined, muted, +weakened, or relabeled under this record. + +The missing chronology is limited to the following behavior that remains in the +candidate from the earlier mixed implementation commits: + +| Area | Exact behavior lacking retained pre-Green Red chronology | Present regression/verification surface | +|---|---|---| +| Concurrent request containment | Once a handler has accepted a request, request-local `Cancelled`, finite request-wait timeout, response-send failure, and other post-accept failures do not feed the global structural-failure breaker; an owned pending request removed by sibling pruning is cancellation, while an unowned duplicate response remains an ordinary error; unrelated `/ping` work stays serviceable after disconnect bursts. | `concurrent_disconnect_paths_burst_test`, concurrent-handler classifier units, and finite request-timeout units | +| Pending-response and dropped-run cleanup | Dropping a handler/interpreter future closes its owned pending request and server response streams, emits the documented dropped-request response where applicable, and does not leave work attached to a reused interpreter. | `dropped_interpret_server_cleanup_test` and interpreter cleanup units | +| Response-stream backpressure | A connected client that stops reading cannot park a response-stream write indefinitely; timeout and disconnect remain distinct typed outcomes, and unrelated handlers continue. | `response_stream_backpressure_test` and response-disconnect classifier units | +| Outbound-stream deadline and close lifecycle | The configured absolute lifetime includes response-head time, bounds active and unread streams, closes the upstream socket, wakes an active reader with the typed terminal result, gives expiry priority on a ready/expiry tie, prevents reinsertion after terminal state, aborts reapers on terminal paths, and safely clamps extreme positive durations. | `outbound_stream_deadline_test`, `outbound_stream_open_expiry_test`, `outbound_stream_reaper_race_test`, and deterministic active-read close/expiry units | +| Ambiguous classic/streaming writes | Type checking selects the runtime-viable classic file or response-stream branch for concrete targets and checks both branches for gradual targets; container/property context and the analyzer's shared continuations do not hide undefined names or reject the inactive reading. | `ambiguous_write_branch_typecheck_test`, `ambiguous_write_analyzer_test`, `stream_handle_type_test`, and focused analyzer/typechecker units | +| Merged write and response operands | Existing write, response content-type, and response-header operands retain ordinary expression composition for property/index/`at` access, concatenation, nested calls, builtins, unary forms, explicit call arguments, and clause boundaries. This row excludes the later status-operand and post-`of` fixes listed below. | `write_web_postfix_test` and parser AST units | +| Legacy merged `flush` compatibility | A previously valid action, overload, non-callable binding, split/find/replace binding, postfix target, or full fallback expression beginning with merged `flush` retains expression-statement behavior; analyzer, unused-variable analysis, typechecker, and runtime use the same preserved legacy binding metadata. This row excludes the later unmerged-target dispatch and post-`of` fixes listed below. | `flush_action_backcompat_test`, `write_web_postfix_test`, and static-analyzer units | +| Fractional request waits | A positive request timeout below one millisecond is rejected deterministically instead of being rounded into the distinct zero/unlimited behavior. | request-wait timeout units | +| Portable archive containment | A rooted archive entry such as `/etc/shadow` is rejected as rooted on Windows as well as Unix before extraction and cannot escape the destination. | `wflpkg` security tests, including archive traversal/rooted-path cases | +| Gate and fixture correctness | The official Windows integration runner handles equivalent `Path`/`PATH` entries without hiding conflicting values and retains the child process exit status; the Windows web runner fails on cleanup failures; subprocess, free-port, and directory-performance fixtures exercise repository-owned bounded resources instead of shell-only commands, fixed ports, or the repository tree. | Official Linux/Windows integration and web scripts, `execute_file_test`, `file_io_performance_test`, and `subprocess_comprehensive.wfl` | + +The following repairs are explicitly **outside** this exception because the +repair branch contains genuine test-only Red ancestors followed by Green +implementation commits: + +- complete status-clause operands; +- postfix continuation after `of`; +- removal of the nonexistent bare `type` response boundary; +- unmerged streaming `flush` dispatch; +- ResponseStream child scopes and conservative branch/loop joins; +- local opened-File symbol recreation; +- clean EOF after a final unterminated line; +- bounded expired-stream terminal metadata; +- response-expression disconnect cancellation, including request operands, + early prechecks, and commit-time cleanup. + +The final candidate adds these three genuine Red-to-Green chains after the +previous draft candidate: + +| Repair | Test-only Red | Green implementation | Evidence broadening | +|---|---|---|---| +| Immediate bounded clean-EOF terminalization | `96d53052388f75bd809c2af42f12445944e8fc69` | `b32ff55fa76fd03b07e2ade7159d3719f2ac0642` | `81b25745e757671538a833ddf7bc837e19ad83c7` deterministically covers close/reaper missing-slot races | +| Loop-header fixed points, joined `try` endpoints, and deferred handler type isolation | `68569b31b9fd969cb5adc3b8c0832ec604bb98e2` | `527b8fb184245e7df35fe5229b23e2a969c74520` | Focused integration suites retained in the Green ancestry | +| First-wins EOF observation and analyzer `try`-scope parity | `03966f06e78aec7c3bcdbd40feabc2bdff37a16d` | `de34e32e513d7d73634b0d7308c681930953a4db` | `81b25745e757671538a833ddf7bc837e19ad83c7` broadens terminal-race coverage without executable changes | + +These chains do not repair the older mixed-commit chronology rows in this +exception. They do establish ordinary policy-compliant chronology for every +behavior changed after `c73260ff61a32694c5ecfe72ab8749810033de0d`. + +The final-unterminated-line defect also has retained pre-Green CI evidence in +[Actions run 30106107011](https://github.com/WebFirstLanguage/wfl/actions/runs/30106107011). +Neither that behavior nor any later genuine Red-to-Green repair depends on this +exception. + +## Why normal compliance cannot now be supplied + +The earlier implementation combined regression tests and production changes in +the same commits. The focused failures described in the completion diary were +observed locally, but no test-only ancestor commit and no independently +timestamped pre-Green artifact was retained for the affected behaviors above. +A local `.git/objects/maintenance.lock` blocked the intended Git object writes +during that pass. + +The missing historical ordering cannot be created after the implementation +date. Reverting or disabling completed code now would only demonstrate test +sensitivity; under Section 6.2 it would not prove the original TDD chronology. +Rewriting timestamps or presenting later characterization as earlier Red +evidence would manufacture evidence and is prohibited. This request is +therefore for temporarily unavailable historical evidence, not for schedule +pressure, test duration, inconvenience, a small-change claim, or permission to +ignore a current failure. + +## Current Green evidence + +- The reviewed Green head + `8e8be0fcde944d0d7b357b94d5951497af5ff0b7` completed + [Actions run 30142079511](https://github.com/WebFirstLanguage/wfl/actions/runs/30142079511), + including Linux and Windows integration, TestPrograms, documentation + validation, web/TLS, PostgreSQL, MariaDB, and fuzz-target compilation. +- The completion diary maps the affected contracts to focused Rust, + real-socket, parser/typechecker/analyzer, WFL end-to-end, and security tests. +- Later issue #642 repairs use retained test-only Red ancestors and Green + commits; those repairs strengthen the candidate but do not retroactively + supply the chronology missing from the earlier mixed commits. +- The latest executable candidate is + `de34e32e513d7d73634b0d7308c681930953a4db`; the test-only descendant + `81b25745e757671538a833ddf7bc837e19ad83c7` adds deterministic coverage for + the clean-EOF close/reaper missing-slot paths without changing production + behavior. + +Actions run 30142079511 is evidence for the reviewed Green head, not automatic +evidence for the exact candidate in this draft. Before approval, the approval +record must link one complete, successful, unretried Actions run for the final +integrated PR head: a documentation descendant of +`81b25745e757671538a833ddf7bc837e19ad83c7` that contains the deterministic +evidence tests and this completed record. A run on executable identity +`de34e32e513d7d73634b0d7308c681930953a4db` alone is insufficient because it +omits those later tests and documents. Until that run and the final local gate +record are complete, current Green evidence is incomplete for merge. + +## Compensating verification and containment + +Approval is conditional on all of the following: + +1. Run, once and without changing product-test selection, the host-appropriate + complete local gate. The recorded Windows gate is: + `cargo fmt --all -- --check`, `git diff --check`, + `cargo clippy --all-targets --all-features --jobs 1 -- -D warnings`, + `cargo build --release --jobs 1`, `cargo test --all --jobs 1`, + `run_integration_tests.ps1 -TestOnly`, `run_web_tests.ps1`, and + `python scripts/validate_docs_examples.py --ci --force`. The single Cargo + job bounds compiler memory after an unbounded rustc invocation ended before + any test result with Windows pagefile error 1455; it does not alter test + selection. Exact Windows PowerShell invocation details are retained in the + Dev Diary. The required final Actions matrix separately runs the repository's + supported Linux and Windows commands. +2. Preserve the exact commands, exit conclusions, candidate SHA, and complete + logs in the PR evidence record. +3. Require one complete GitHub Actions matrix on the final integrated PR head + described above, covering Linux and Windows integration, TestPrograms, + documentation validation, web tests, TLS, PostgreSQL, MariaDB, and + fuzz-target compilation. Every required job must pass. +4. Obtain an independent R3 review of the implementation, regression + assertions, real-boundary coverage, cleanup paths, and this exception's + exact scope. +5. Confirm in the approval record that no required test was skipped, retried, + quarantined, muted, weakened, or converted into a timing-only success + assertion. +6. Freeze the executable candidate after approval. Any executable change + requires a new full gate, scope review, exact SHA, and approval decision. +7. Do not release more than the single affected release, and do not merge or + release after expiration. Expiration fails closed. + +These controls establish present behavior and contain the exposure. They do not +replace or reconstruct the missing historical chronology. + +## Residual risk + +- Because the tests and earlier implementation were committed together, the + record cannot prove that each test was specified independently of the chosen + implementation. A test could encode the implementation while missing a + different contract-preserving failure mode. +- Concurrency and socket lifecycle tests cover deterministic checkpoints and + supported CI platforms, but do not exhaust every OS scheduler, socket-buffer + size, cancellation ordering, or long-duration accumulation pattern. +- Parser/typechecker compatibility matrices cover the reported operand and + scope shapes but cannot prove compatibility for every existing WFL program. +- Windows and Unix archive containment tests cover known rooted and traversal + forms but do not constitute a proof over every filesystem namespace or future + archive format. +- The older local Red observations are narrative only. They must not be cited + as policy-compliant Red evidence. + +There is no accepted known product-test failure in this draft. Discovery of a +reproducible product failure, authorization bypass, data loss/corruption, +exposed secret, unresolved Critical vulnerability, or another Section 14 +non-waivable condition immediately invalidates this exception and blocks merge +or release. + +## Rollback and recovery + +No deployment, schema, persistent data, or external service state is changed by +PR #641. + +- **Before merge:** stop the PR and rebuild the candidate from affected base + `b25aed57ea50697c596796446d1f47466668773d`, preserving genuine test-only Red + commits before each production repair. Do not force-push or rewrite evidence + without an explicit maintainer decision and a retained mapping from the old + candidate to the replacement. +- **After merge, before release:** revert the PR's merge/squash commit (or the + exact affected commits if merged unsquashed), then run the complete gate on + the revert candidate. Prefer forward repair when a broad revert would remove + compatibility or lifecycle fixes that other changes now depend on. +- **After the one permitted release:** publish a normal tested forward repair + or a revert release under the ordinary release gate. This exception cannot be + reused for that release. + +If a concurrency or cleanup regression appears, first disable release of the +candidate, preserve the failing boundary evidence, and repair it with a genuine +Red commit. If archive containment regresses, stop distribution of the affected +package artifacts and route the finding through `SECURITY.md`; do not disclose +new vulnerability details in a public issue. + +## Repair ticket, owner, and deadline + +- **Ticket:** [WebFirstLanguage/wfl issue #642](https://github.com/WebFirstLanguage/wfl/issues/642), + testing-policy evidence gap. +- **Owner:** Brad, Maintainer and WFL test/reliability owner, Logbie LLC. +- **Deadline:** `2026-07-31T00:00:00-05:00`, before this exception expires and + before merge or release. +- **Required resolution:** either (a) locate and retain admissible pre-Green + artifacts for every row above, reducing or eliminating this scope; (b) + replace the mixed implementation stack from the recorded affected base with + genuine Red-to-Green ancestry and rerun the full gate; or (c) complete and + approve this narrowly scoped record for the exact candidate and archive it + with the release evidence. A later characterization run alone does not + satisfy options (a) or (b). + +Issue closure does not itself approve this exception. If the deadline passes +without one of these resolutions, the exception expires and the affected merge +or release remains blocked. + +## Approval record — must be completed before activation + +| Approval field | Required entry | +|---|---| +| Exact final executable candidate SHA | `de34e32e513d7d73634b0d7308c681930953a4db` | +| Final evidence-only descendant SHA, if any | `81b25745e757671538a833ddf7bc837e19ad83c7` (latest code/test descendant; the commit containing this documentation record is documentation-only) | +| Final local gate record | **PASSED 2026-07-25** — Windows NT `10.0.26200.0`, rustc/cargo `1.97.0`, PowerShell `7.6.4`; the official `.ps1` runners executed under Windows PowerShell `5.1.26100.8875`. `cargo fmt --all -- --check`, `cargo clippy --all-targets --all-features --jobs 1 -- -D warnings`, `cargo test --all --jobs 1`, `cargo build --release --jobs 1`, and `git diff --check` passed. The official Windows integration runner passed all Rust targets plus 110 WFL programs (0 failed, 24 documented skips); web passed 2/2 runnable journeys with its OpenSSL-dependent certificate journey visibly skipped; forced docs validation passed 18/18. The earlier unbounded `cargo test --all` stopped in rustc before a test result with Windows pagefile error 1455; the unchanged suite passed with one compiler job. Full command context is in `Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md`. | +| Final GitHub Actions run | **PENDING** — URL, final integrated PR-head SHA (a documentation descendant of `81b25745`), and every required job conclusion | +| Independent R3 reviewer | Independent Codex review task `/root/final_independent_review`, `2026-07-25`; reviewed `c9c748ce..de34e32e` for lifecycle/concurrency correctness, type-flow soundness, compatibility, and test integrity. Its three Important evidence/documentation findings are addressed by `81b25745` and the documentation-only descendant containing this record; this is review evidence, not approval authority. | +| Requester | **PENDING** — identity and date | +| Project/reliability owner decision | **PENDING** — Brad must record `APPROVE` or `REJECT`, rationale, date, and signature | +| Security-owner decision for archive-path scope | **PENDING** — Brad must record `APPROVE` or `REJECT`, rationale, date, and signature | +| No skip/retry/quarantine/muting/weakening/timing-only conversion attestation | **RECORDED 2026-07-25** — no changed-behavior test was skipped, retried, quarantined, muted, weakened, or converted to timing-only proof. The error-1455 compiler interruption produced no product-test result and was rerun only with bounded compiler parallelism. Existing Rust ignores, 24 documented WFL program skips, and the host's OpenSSL-dependent web skip remained visible; exact-candidate CI coverage is still required. | +| Maximum-release and expiration acknowledgment | **PENDING** | + +The requester must not be the sole approver. If Brad is also the requester, a +separate authorized project/domain approver must approve; an independent review +that has no approval authority is not a substitute. The completed record must +remain attached to the PR and archived with release evidence for the retention +period required by Section 15. + +**PENDING MAINTAINER APPROVAL — PR #641 MERGE GATE UNRESOLVED.** diff --git a/Docs/reference/configuration-reference.md b/Docs/reference/configuration-reference.md index baac7050..8c782d72 100644 --- a/Docs/reference/configuration-reference.md +++ b/Docs/reference/configuration-reference.md @@ -214,6 +214,7 @@ All keys currently loaded from config files, with defaults. | `web_server_max_response_size` | integer ≥ 1 | `67108864` (64 MiB) | Max handler or outbound HTTP response body size (bytes) | | `web_server_request_queue_bound` | integer ≥ 1 | `256` | Max queued HTTP requests before shedding with 503 | | `web_server_response_timeout_seconds` | integer ≥ 0 | `300` | Seconds to await a handler before shedding with 504; `0` disables | +| `outbound_stream_max_seconds` | integer ≥ 0 | `300` | Absolute total lifetime (seconds) of one outbound streaming response, distinct from the per-read idle timeout; `0` disables; values above one year use the one-year safety cap | | `web_socket_queue_bound` | integer ≥ 1 | `1024` | Max queued frames/events per WebSocket channel before shedding | | `web_socket_max_connections` | integer ≥ 1 | `1024` | Max simultaneous live WebSocket connections | | `web_socket_max_message_size` | integer ≥ 1 | `1048576` (1 MiB) | Max size of a single WebSocket text message (bytes); larger frames are dropped | @@ -558,13 +559,27 @@ or omits `Content-Length`. #### `web_server_response_timeout_seconds` -Maximum time, in seconds, the transport waits for a handler to answer an accepted request before shedding it with a `504 Gateway Timeout` and freeing its in-flight slot. This bounds a dequeued-but-never-answered request so it cannot pin an in-flight slot indefinitely. +Maximum time, in seconds, the transport waits for a handler to answer an accepted request before shedding it with a `504 Gateway Timeout` and freeing its in-flight slot. This bounds a dequeued-but-never-answered request so it cannot pin an in-flight slot indefinitely. It also bounds a single **streaming-response** `write` (`write line|chunk ... to `): when a connected client stops reading, the bounded body channel fills and the write applies backpressure, so this timeout caps how long that write parks before failing — a stalled client can slow a handler but not pin it forever. - **Type:** Integer (0 or more) - **Default:** `300` - **Example:** `web_server_response_timeout_seconds = 30` -A value of `0` disables the timeout. The in-flight request cap (`web_server_request_queue_bound`) is enforced globally across every `listen` server via one shared budget, and a request's slot is held from the moment its body starts streaming until the handler responds, this timeout fires, or the client disconnects. +A value of `0` disables the timeout (including the streaming-write bound above). The in-flight request cap (`web_server_request_queue_bound`) is enforced globally across every `listen` server via one shared budget, and a request's slot is held from the moment its body starts streaming until the handler responds, this timeout fires, or the client disconnects. + +#### `outbound_stream_max_seconds` + +Absolute total lifetime, in seconds, of a single **outbound** streaming response opened with `open url ... and stream response as `, measured from when the stream is opened. This is distinct from `timeout_seconds`, which is the per-read **idle** timeout: an upstream that trickles one byte just before every idle timeout would otherwise run forever, but it can never live past this hard cap. The cap is enforced in **real time**, not only on the next read: each incremental read (`wait for next line/chunk`) is bounded by the time remaining to this deadline, AND a background reaper closes the stream (cancelling the upstream request) when the deadline elapses — so even a stream that is opened and then never read cannot outlive the cap. + +- **Type:** Integer (0 or more) +- **Default:** `300` +- **Example:** `outbound_stream_max_seconds = 60` + +A value of `0` disables the absolute cap (the idle timeout still applies per +read). Positive values above 31,536,000 seconds (one year) are safely clamped to +one year when the runtime creates the deadline. This keeps extreme configuration +values finite and prevents platform `Instant` overflow; timeout diagnostics +report the effective clamped duration. #### `web_socket_queue_bound` diff --git a/Docs/superpowers/plans/2026-07-25-parser-compat-regressions.md b/Docs/superpowers/plans/2026-07-25-parser-compat-regressions.md new file mode 100644 index 00000000..9476c815 --- /dev/null +++ b/Docs/superpowers/plans/2026-07-25-parser-compat-regressions.md @@ -0,0 +1,106 @@ +# Parser Compatibility Regressions Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Restore legacy file-write, zero-argument `flush` action, display-fold, and JavaScript transpilation behavior without weakening the new HTTP-stream syntax. + +**Architecture:** Keep ambiguous syntax represented explicitly in the AST and defer branch selection to the existing analyzer/typechecker/runtime or transpiler boundary. Restrict trailing postfix parsing to postfix forms that were historically owned by the preceding expression, using source adjacency where whitespace distinguishes a display fold from an index. + +**Tech Stack:** Rust 2024, WFL lexer/parser/analyzer/interpreter, JavaScript transpiler, Cargo integration tests, WFL end-to-end programs. + +## Global Constraints + +- Risk class is R3 because this changes language backward compatibility and streaming syntax. +- Preserve every previously valid WFL program; do not broaden streaming dispatch over legacy expression syntax. +- Follow Red → Green → Refactor → Broaden → Record with observed failing tests before production edits. +- Add coverage at the parser/transpiler layer and through the real `wfl` binary. + +--- + +### Task 1: Record compatibility regressions as failing tests + +**Files:** +- Modify: `tests/write_web_postfix_test.rs` +- Modify: `tests/transpiler_test.rs` +- Create: `TestPrograms/parser_streaming_compat_regression.wfl` + +**Interfaces:** +- Consumes: public lexer/parser, `wfl::transpiler::JavaScriptTranspiler`, and the built `wfl` binary. +- Produces: regression tests for classic `write line|chunk` continuations, exact `flush` action fallback, display folding, and transpiler fallback. + +- [ ] Add parser/runtime tests for `write line with "!"`, `write line at 0`, and `write line[0]` targeting a file. +- [ ] Add runtime tests proving zero-argument action `flush` still runs for `flush (expr)` and same-line `flush call ...`. +- [ ] Add parser/runtime tests proving `display alice.name [1, 2]` and `display alice.name 5` remain display folds. +- [ ] Add a transpiler test proving `write line note to "f.txt"` uses its classic file-write fallback. +- [ ] Add a real WFL end-to-end program that asserts the compatible runtime results. +- [ ] Run the focused tests and retain their expected failures as Red evidence. + +### Task 2: Restore classic write ownership for bare marker continuations + +**Files:** +- Modify: `src/parser/stmt/io.rs` +- Test: `tests/write_web_postfix_test.rs` + +**Interfaces:** +- Consumes: the token immediately following an exact `line`/`chunk` contextual marker. +- Produces: `WriteToStatement` for legacy continuations and `StreamWriteStatement` for genuine stream operands. + +- [ ] Extend the bare-marker guard to recognize legacy continuation starters (`with`, `at`, `[`, and `.`) rather than only `to`. +- [ ] Run the focused write parser/runtime tests and confirm Green. + +### Task 3: Preserve exact `flush` action fallback + +**Files:** +- Modify: `src/parser/stmt/web.rs` +- Test: `tests/write_web_postfix_test.rs` + +**Interfaces:** +- Consumes: exact `flush` dispatch followed by a parenthesized or explicit-call stream target. +- Produces: `FlushStreamStatement.action_fallback` containing the legacy zero-argument `flush` action expression. + +- [ ] Build the exact-token legacy fallback from the `flush` binding while independently parsing the stream target. +- [ ] Run the focused flush parser/runtime tests and confirm Green. + +### Task 4: Stop postfix parsing from stealing display folds + +**Files:** +- Modify: `src/parser/expr/primary.rs` +- Test: `tests/write_web_postfix_test.rs` + +**Interfaces:** +- Consumes: a property/method expression followed by possible postfix tokens. +- Produces: adjacent `property[index]` chaining while leaving whitespace-separated list and scalar expressions to the display fold. + +- [ ] Require bracket adjacency after a property/method expression before treating `[` as its postfix. +- [ ] Do not reinterpret a trailing bare integer as direct indexing after property/method access. +- [ ] Run the focused display and existing property-index tests and confirm Green. + +### Task 5: Transpile ambiguous classic writes through their fallback + +**Files:** +- Modify: `src/transpiler/javascript.rs` +- Test: `tests/transpiler_test.rs` + +**Interfaces:** +- Consumes: `StreamWriteStatement` with `fallback_content: Some`. +- Produces: the same `WFL.file.write(...)` JavaScript emitted for the legacy file-write reading; unambiguous streaming statements remain unsupported. + +- [ ] Split the transpiler match arm so ambiguous writes use `fallback_content` and `target`. +- [ ] Keep unambiguous HTTP stream writes as clear transpilation errors. +- [ ] Run transpiler tests and confirm Green. + +### Task 6: Broaden verification and record evidence + +**Files:** +- Modify: `Dev diary/2026-07-25-parser-streaming-compat-regressions.md` + +**Interfaces:** +- Consumes: final implementation and test output. +- Produces: durable R3 acceptance criteria, Red/Green commands, and residual-risk record. + +- [ ] Run `cargo fmt --all -- --check`. +- [ ] Run focused parser, transpiler, analyzer/typechecker, and property-index suites. +- [ ] Run `cargo test --all`. +- [ ] Run `cargo build --release` followed by `scripts/run_integration_tests.ps1`. +- [ ] Run `cargo clippy --all-targets --all-features -- -D warnings`. +- [ ] Record exact evidence and any residual risk in the Dev Diary. diff --git a/Docs/superpowers/plans/2026-07-25-stream-terminal-typeflow-fixes.md b/Docs/superpowers/plans/2026-07-25-stream-terminal-typeflow-fixes.md new file mode 100644 index 00000000..6faabe22 --- /dev/null +++ b/Docs/superpowers/plans/2026-07-25-stream-terminal-typeflow-fixes.md @@ -0,0 +1,479 @@ +# Stream Terminal and Type-Flow Fixes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Eliminate completed outbound-stream retention and make typechecking match runtime state across `try`, loop backedges, and deferred handler bodies. + +**Architecture:** Clean EOF becomes a lightweight `StreamTerminal::CleanEof` entry in the existing bounded recent-terminal queue; the live slot, owner ID, body, cancel channel, and reaper are removed as soon as the final unterminated line is returned. The typechecker will use symbol-type snapshots as a small control-flow lattice: branch endpoints are joined before `finally`, loop headers are widened to a fixed point and checked once at that fixed point, and deferred event/WebSocket bodies are checked in isolated child scopes whose refinements are restored afterward. + +**Tech Stack:** Rust 2024, Tokio, reqwest streaming, WFL analyzer/typechecker, Rust unit and integration tests. + +## Global Constraints + +- Risk class is **R3** because this changes streaming lifecycle and language backward-compatibility behavior. +- Every behavioral fix requires a test-only Red commit that is an ancestor of its Green commit. +- Existing WFL programs remain compatible; gradual `Unknown`/`Any` joins must remain permissive while concrete invalid branches remain diagnostics. +- Error aliases introduced by `when` clauses remain clause-local. +- Clean EOF retention uses the existing hard limits: at most **64** lightweight records for at most **60 seconds**, and the record is consumed by one follow-up read. +- No required test may be retried, skipped, quarantined, muted, or weakened. +- Required final gates are `cargo fmt --all -- --check`, `cargo clippy --all-targets --all-features --jobs 1 -- -D warnings`, `cargo test --all --jobs 1`, a release build, Windows integration/web scripts, and forced docs example validation. Cargo jobs are bounded to one on this Windows host because the unbounded compiler process hit pagefile error 1455 before any test binary ran. + +--- + +### Task 1: Test-only Red for clean EOF terminalization + +**Files:** +- Modify: `src/interpreter/mod.rs` (unit-test module only) + +**Interfaces:** +- Consumes: `IoClient::open_http_stream`, `IoClient::claim_stream_owner`, `IoClient::next_line`, `StreamRegistry::{live,recent}`, and `MAX_RECENT_STREAM_TERMINALS`. +- Produces: a regression proving live state is gone before the one-shot EOF read. + +- [x] **Step 1: Strengthen the final-unterminated-line test** + +After the first `Some("abc")`, inspect the registry and owner before any follow-up read: + +```rust +let (live_slots, clean_eof_records) = { + let registry = interpreter + .io_client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + ( + registry.live.len(), + registry + .recent + .iter() + .filter(|entry| entry.reason == StreamTerminal::CleanEof) + .count(), + ) +}; +assert_eq!(live_slots, 0); +assert_eq!(clean_eof_records, 1); +assert_eq!( + interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len(), + 0 +); +``` + +Then retain the existing assertions that one delayed read returns `None` and a later read reports an already-closed handle. + +- [x] **Step 2: Add a bounded no-follow-up-read wave** + +Open and claim more than `MAX_RECENT_STREAM_TERMINALS` `/unterminated` streams under one `Interpreter`, read only each final line, and assert: + +```rust +assert_eq!(registry.live.len(), 0); +assert!(registry.recent.len() <= MAX_RECENT_STREAM_TERMINALS); +assert!(registry + .recent + .iter() + .all(|entry| entry.reason == StreamTerminal::CleanEof)); +assert!(interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty()); +``` + +- [x] **Step 3: Run the focused tests and verify Red** + +Run: + +```powershell +cargo test --lib interpreter::outbound_stream_deadline_tests::final_unterminated_line_survives_deadline_after_clean_eof -- --nocapture --test-threads=1 +cargo test --lib interpreter::outbound_stream_deadline_tests::unconsumed_clean_eof_records_are_bounded -- --nocapture --test-threads=1 +``` + +Expected and observed: the lifecycle assertions fail because the completed +handle remains in `registry.live`; Red is a behavioral failure, not a +compile-failure placeholder. + +- [x] **Step 4: Commit Red evidence** + +```powershell +git add src/interpreter/mod.rs +git commit -m "test: expose retained clean eof stream state" +``` + +### Task 2: Bounded one-shot `CleanEof` + +**Files:** +- Modify: `src/interpreter/mod.rs` + +**Interfaces:** +- Consumes: `StreamRegistry::remember_recent` and `take_recent`. +- Produces: `StreamTerminal::CleanEof`; `take_stream` returns `Ok(None)` for that one-shot result. + +- [x] **Step 1: Add the terminal reason and optional take result** + +```rust +enum StreamTerminal { + CleanEof, + Timeout, + Closed, +} + +fn take_stream( + &self, + handle_id: &str, +) -> Result, HttpClientError> +``` + +When `take_recent` yields `CleanEof`, return `Ok(None)`; typed failures continue through `stream_terminal_error`. + +- [x] **Step 2: Terminalize in `put_stream`** + +Make the upstream `None` observation the linearization point: +`stream_pull` first-wins latches `CleanEof` on the shared cancel state before +returning control to `next_line`/`next_chunk`. In `put_stream`, handle the +latched `handle.done` case before generic cancellation or wall-deadline +rejection: + +```rust +if handle.done { + let terminal = cancel.terminate(StreamTerminal::CleanEof); + if terminal == StreamTerminal::CleanEof { + drop(handle); + let now = Instant::now(); + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + registry.prune_recent(now); + if let Some(mut slot) = registry.live.remove(handle_id) { + slot.cancel.terminate(StreamTerminal::CleanEof); + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + } + registry.remember_recent( + handle_id.to_string(), + StreamTerminal::CleanEof, + now, + ); + return Ok(()); + } +} +``` + +`remember_recent` replaces any record for the same ID, so this restores exactly +one one-shot result even when close or the reaper already removed the live +slot. Do not clear either deadline and do not park a completed handle. If +`Timeout` or `Closed` won before upstream EOF was observed, preserve that +earlier typed terminal instead. + +- [x] **Step 3: Consume clean EOF in both read APIs** + +```rust +let Some(TakenStream { mut handle, cancel }) = self.take_stream(handle_id)? else { + return Ok(None); +}; +``` + +Use the same form in `next_chunk` and `next_line`. Lifecycle-only call sites that cannot legitimately observe clean EOF map it to closed without retaining heavy state. + +- [x] **Step 4: Verify Green and adjacent lifecycle behavior** + +Run: + +```powershell +cargo test --lib interpreter::outbound_stream_deadline_tests -- --nocapture --test-threads=1 +cargo test --test http_stream_test -- --nocapture --test-threads=1 +``` + +Expected: all focused lifecycle and real-boundary streaming tests pass. + +- [x] **Step 5: Commit Green** + +```powershell +git add src/interpreter/mod.rs +git commit -m "fix: terminalize clean eof streams immediately" +``` + +### Task 3: Test-only Red for checker control-flow state + +**Files:** +- Modify: `tests/typechecker_response_stream_join_test.rs` +- Modify: `tests/typechecker_response_stream_scope_test.rs` +- Create: `tests/typechecker_try_finally_join_test.rs` + +**Interfaces:** +- Consumes: direct `Program`/`Statement` AST construction and the public `TypeChecker::check_types`. +- Produces: later-iteration, `finally`, alias-isolation, event-handler, and WebSocket-handler regressions. + +- [x] **Step 1: Add later-iteration loop tests** + +Construct `WhileLoop` and `RepeatWhileLoop` bodies in this order: + +```rust +Statement::StreamWriteStatement { + value: Expression::BinaryOperation { + left: Box::new(Expression::Literal(Literal::Integer(10), 2, 1)), + operator: Operator::Minus, + right: Box::new(text_literal("not a number")), + line: 2, + column: 1, + }, + target: Expression::Variable("out".to_string(), 2, 1), + is_line: true, + fallback_content: Some(Box::new(text_literal("valid file text"))), + line: 2, + column: 1, +}, +stream_binding(), +``` + +Precede the loop with an `OpenFileStatement` binding `out`. Each test must expect `Cannot perform Minus operation`: the first iteration takes the valid File fallback, while the backedge can make the next iteration take the invalid ResponseStream reading. + +- [x] **Step 2: Add `try` endpoint/finally tests** + +Build a `TryStatement` whose handler binds `out` as a response stream and whose `finally` flushes `out`, with an outer concrete File binding. Assert the checker accepts the joined gradual state rather than resolving only the outer File. + +Add a control where outer Number bindings reuse the handler error name and `error_message`; subtraction in `finally` must remain valid, proving both aliases stay clause-local. + +- [x] **Step 3: Add deferred-handler isolation tests** + +For both `EventHandler` and `WebSocketHandlerStatement`, start with outer `out: Number`, put `stream_binding()` in the registered body, and subtract one from outer `out` after registration. Assert typechecking succeeds. + +- [x] **Step 4: Run focused tests and verify Red** + +Run: + +```powershell +cargo test --test typechecker_response_stream_join_test -- --nocapture +cargo test --test typechecker_try_finally_join_test -- --nocapture +cargo test --test typechecker_response_stream_scope_test -- --nocapture +``` + +Expected: the new later-iteration tests miss the invalid stream branch, the `finally` test rejects a File flush, and the event/WebSocket tests leak `ResponseStream` into outer `out`. + +- [x] **Step 5: Commit Red evidence** + +```powershell +git add tests/typechecker_response_stream_join_test.rs tests/typechecker_response_stream_scope_test.rs tests/typechecker_try_finally_join_test.rs +git commit -m "test: expose checker backedge and handler state gaps" +``` + +### Task 4: Checker joins, fixed points, and deferred scopes + +**Files:** +- Modify: `src/analyzer/mod.rs` +- Modify: `src/typechecker/mod.rs` + +**Interfaces:** +- Consumes: `Analyzer::{push_scope,pop_scope,snapshot_symbol_types,restore_symbol_types}` and `TypeChecker::join_type_snapshots`. +- Produces: `Analyzer::pop_scope_promoting_except` and `TypeChecker::check_loop_body_fixed_point`. + +- [x] **Step 1: Add selective clause-scope promotion** + +```rust +pub fn pop_scope_promoting_except(&mut self, excluded: &[String]) { + if let Some(mut parent) = self.current_scope.parent.take() { + for (name, symbol) in std::mem::take(&mut self.current_scope.symbols) { + if !excluded.iter().any(|excluded_name| excluded_name == &name) { + parent.define_or_replace(symbol); + } + } + self.current_scope = *parent; + } +} +``` + +This models the runtime’s shared try child while dropping only the temporary error aliases. + +- [x] **Step 2: Join `try` endpoints before `finally`** + +Within the shared try checker scope: + +1. Snapshot entry. +2. Check the body and capture the success endpoint. +3. Form a conservative handler entry from entry plus body endpoint. +4. Restore that entry before every handler. +5. Check a handler in an alias child scope, promote all non-alias bindings, and capture its endpoint. +6. Restore handler entry before `otherwise` and capture its endpoint. +7. Join success, handler, otherwise, and possible unmatched-error endpoints. +8. Restore the join, then check `finally` once. + +- [x] **Step 3: Compute a widening loop-header fixed point** + +```rust +fn check_loop_body_fixed_point(&mut self, body: &[Statement]) { + let entry = self.analyzer.snapshot_symbol_types(); + let mut header = entry.clone(); + loop { + self.analyzer.restore_symbol_types(header.clone()); + let error_count = self.errors.len(); + for statement in body { + self.check_statement_types(statement); + } + if self.budget_error.is_some() { + return; + } + self.errors.truncate(error_count); + let backedge = self.analyzer.snapshot_symbol_types(); + let next = + Self::join_type_snapshots(&[entry.clone(), header.clone(), backedge]); + if next == header { + break; + } + header = next; + } + self.analyzer.restore_symbol_types(header.clone()); + for statement in body { + self.check_statement_types(statement); + } + self.analyzer.restore_symbol_types(header); +} +``` + +Use it inside the persistent child scope for `RepeatWhileLoop` and in the current scope for `WhileLoop`. Validate each condition once under the stable header. + +- [x] **Step 4: Isolate deferred callback bodies** + +For `EventHandler` and `WebSocketHandlerStatement`, push a checker child scope, snapshot types, check the body, restore the snapshot, and pop. The WebSocket server operand remains checked outside the child. + +- [x] **Step 5: Verify Green and affected checker suites** + +Run: + +```powershell +cargo test --test typechecker_response_stream_join_test -- --nocapture +cargo test --test typechecker_try_finally_join_test -- --nocapture +cargo test --test typechecker_response_stream_scope_test -- --nocapture +cargo test --test nothing_reassign_widen_test -- --nocapture +cargo test --test open_file_local_type_test -- --nocapture +``` + +Expected: all pass with no duplicated diagnostics. + +- [x] **Step 6: Commit Green** + +```powershell +git add src/analyzer/mod.rs src/typechecker/mod.rs +git commit -m "fix: stabilize checker control-flow state" +``` + +### Task 4.25: Linearize EOF and align analyzer `try` scopes + +The first Green pass exposed two adjacent gaps during independent review. +They were repaired with another genuine Red-to-Green pair: + +- [x] **Step 1: Commit deterministic Red regressions** + +Commit `03966f06e78aec7c3bcdbd40feabc2bdff37a16d` adds a deterministic +EOF-observation-versus-later-timeout regression, analyzer-created +handler/`otherwise` binding regressions, and full-pipeline alias-shadowing +coverage. + +- [x] **Step 2: Commit the Green implementation** + +Commit `de34e32e513d7d73634b0d7308c681930953a4db` makes terminal signals +first-wins at upstream EOF and gives analyzer clauses/finally the same shared +runtime child environment while retaining clause-local error aliases. + +### Task 4.5: Resolve independent-review evidence gaps + +**Files:** +- Modify: `src/interpreter/mod.rs` (unit-test module only) +- Modify: `Docs/development/response-streaming-design.md` +- Modify: `Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md` + +- [x] **Step 1: Cover close/reaper missing-slot paths after observed EOF** + +Add deterministic tests that latch `CleanEof`, remove the live slot through +the real close helper and a reaper-equivalent critical section, then prove +`put_stream` leaves exactly one one-shot `CleanEof`, no owner, no live slot, +one `nothing` read, and then the closed-handle error. + +- [x] **Step 2: Document bounded clean-EOF retention** + +Document that the one-shot result shares the 64-record recent-terminal queue, +expires after 60 seconds, and can be evicted or consumed. + +- [x] **Step 3: Refresh the candidate chronology** + +Record the exact executable candidate and all three new Red-to-Green chains +without treating independent review as maintainer approval. + +- [x] **Step 4: Obtain follow-up review** + +Have the independent reviewer verify the new deterministic tests and both +documentation repairs, and resolve any remaining Critical or Important issue. + +### Task 5: Evidence, documentation, and full verification + +**Files:** +- Create: `Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md` +- Modify: `Docs/superpowers/plans/2026-07-25-stream-terminal-typeflow-fixes.md` +- Modify: `Docs/development/response-streaming-design.md` +- Modify: `Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md` + +**Interfaces:** +- Consumes: Red/Green commit IDs and exact command output. +- Produces: durable R3 acceptance-criteria mapping and residual-risk record. + +- [x] **Step 1: Record change evidence** + +The Dev Diary entry must include: + +```markdown +- Risk class: R3 +- Acceptance criteria -> exact test names +- Base, Red, and Green commit IDs for all three Red/Green pairs +- Focused, unit, integration, web, docs, format, and clippy commands +- Windows platform result and any explicitly non-applicable layers +- Rollback: revert the complete post-base repair range, including test-only + commits, so intentionally failing Red tests are not left active; preserve + the original Red/Green commits in history as evidence +- Residual risk: recent CleanEof records are intentionally capped at 64/60s +``` + +- [x] **Step 2: Run static and complete Rust gates** + +```powershell +cargo fmt --all -- --check +cargo clippy --all-targets --all-features --jobs 1 -- -D warnings +cargo test --all --jobs 1 +cargo build --release --jobs 1 +``` + +Expected: every command exits zero without warnings from changed code. + +- [x] **Step 3: Run real-boundary and documentation gates** + +```powershell +$env:CARGO_BUILD_JOBS='1' +& 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' -NoProfile -ExecutionPolicy Bypass -File '.\scripts\run_integration_tests.ps1' -TestOnly +& 'C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe' -NoProfile -ExecutionPolicy Bypass -File '.\scripts\run_web_tests.ps1' +python scripts/validate_docs_examples.py --ci --force +``` + +Expected: all required programs, web journeys, and docs examples pass without retry. + +- [x] **Step 4: Obtain independent review** + +Review the complete diff from `c9c748ce` through the final code commit for specification compliance, concurrency/lifecycle correctness, type-lattice soundness, compatibility, and test integrity. Resolve every Critical or Important finding and rerun its covering tests. + +- [x] **Step 5: Commit evidence** + +```powershell +git add "Dev diary/2026-07-25-stream-terminal-typeflow-review-fixes.md" Docs/development/response-streaming-design.md Docs/development/testing-policy-exceptions/2026-07-24-pr-641-red-chronology.md Docs/superpowers/plans/2026-07-25-stream-terminal-typeflow-fixes.md +git commit -m "docs: record stream state review fixes" +``` + +## Self-Review + +- Spec coverage: all four review findings map to Tasks 1–4; R3 evidence and full gates map to Task 5. +- Placeholder scan: no TBD/TODO/later placeholders remain. +- Type consistency: `CleanEof`, `take_stream -> Result, _>`, `pop_scope_promoting_except`, and `check_loop_body_fixed_point` are named consistently in every task. +- Execution choice: this request already asks for the fixes in the current branch, so execute inline in this session without pausing between tasks. diff --git a/TestPrograms/docs_examples/_meta/manifest.json b/TestPrograms/docs_examples/_meta/manifest.json index a07be1ad..1749cd4b 100644 --- a/TestPrograms/docs_examples/_meta/manifest.json +++ b/TestPrograms/docs_examples/_meta/manifest.json @@ -401,5 +401,56 @@ "containers" ], "doc_purpose": "Docs README tour: containers/objects" + }, + "docs_examples/interoperability/streaming_response.wfl": { + "doc_section": "Docs/04-advanced-features/interoperability.md#streaming-a-response-incrementally", + "type": "snippet", + "validate_layers": [ + 1, + 2, + 3, + 4 + ], + "skip_execution": true, + "tags": [ + "http-client", + "streaming", + "interoperability" + ], + "description": "Streaming an outbound HTTP response incrementally with stream response / wait for next line." + }, + "docs_examples/web_servers/streaming_response.wfl": { + "doc_section": "Docs/04-advanced-features/web-servers.md#streaming-a-response", + "type": "snippet", + "validate_layers": [ + 1, + 2, + 3, + 4 + ], + "skip_execution": true, + "tags": [ + "web-server", + "streaming", + "response" + ], + "description": "Streaming a server response with start streaming response / write line / flush / close." + }, + "docs_examples/web_servers/concurrent_main_loop.wfl": { + "doc_section": "Docs/04-advanced-features/web-servers.md#concurrent-request-handling", + "type": "snippet", + "validate_layers": [ + 1, + 2, + 3, + 4 + ], + "skip_execution": true, + "tags": [ + "web-server", + "concurrency", + "main-loop" + ], + "description": "Concurrent request handling with main loop concurrently." } } diff --git a/TestPrograms/docs_examples/interoperability/streaming_response.wfl b/TestPrograms/docs_examples/interoperability/streaming_response.wfl new file mode 100644 index 00000000..02c1c2df --- /dev/null +++ b/TestPrograms/docs_examples/interoperability/streaming_response.wfl @@ -0,0 +1,26 @@ +// CI-SKIP: needs a live upstream; validated via docs-examples layers 1-4 +// Streaming an outbound response incrementally. +// +// `stream response as` returns as soon as the status and headers arrive, +// without buffering the body. Pull the body one line (or chunk) at a time; +// each read binds `nothing` at a clean end of stream. +// +// Validated for syntax/analysis/lint only (layers 1-4): running it needs a +// live upstream, so execution is skipped. + +open url at "https://api.example.com/events" and stream response as upstream + +display "Status: " with upstream["status"] +store content_type as upstream["headers"]["content-type"] +display "Content type: " with content_type + +count from 1 to 1000000: + wait for next line from upstream as line + check if line is nothing: + break + otherwise: + display line + end check +end count + +close upstream \ No newline at end of file diff --git a/TestPrograms/docs_examples/web_servers/concurrent_main_loop.wfl b/TestPrograms/docs_examples/web_servers/concurrent_main_loop.wfl new file mode 100644 index 00000000..07e45ce1 --- /dev/null +++ b/TestPrograms/docs_examples/web_servers/concurrent_main_loop.wfl @@ -0,0 +1,26 @@ +// CI-SKIP: starts a server + concurrent loop; needs HTTP clients (layers 1-4) +// Concurrent request handling with `main loop concurrently:`. +// +// Iterations run in their own child scope, and their count-loop/recursion +// run-state is isolated per handler — so per-request `store` variables never +// clobber another request's. Global (top-level) bindings and any collections +// shared through them remain shared, by design. Concurrency is cooperative on +// one thread: a slow handler yields to its siblings at await points (wait for, +// outbound HTTP, stream reads/writes, respond), so it does not block them there +// — but CPU-bound work with no await still holds the thread. Plain `main loop` +// stays serial; adding `concurrently` is the only way to opt in. +// +// Validated for syntax/analysis/lint only (layers 1-4): running it needs live +// clients, so execution is skipped. + +listen on port 8080 as site + +main loop concurrently: + wait for request comes in on site as req + store p as req["path"] + check if p is equal to "/health": + respond to req with "ok" + otherwise: + respond to req with "Hello!" + end check +end loop \ No newline at end of file diff --git a/TestPrograms/docs_examples/web_servers/streaming_response.wfl b/TestPrograms/docs_examples/web_servers/streaming_response.wfl new file mode 100644 index 00000000..03e12dfc --- /dev/null +++ b/TestPrograms/docs_examples/web_servers/streaming_response.wfl @@ -0,0 +1,23 @@ +// CI-SKIP: starts a web server and needs an HTTP client; validated via docs-examples layers 1-4 +// Streaming a server response. +// +// `start streaming response` sends status/headers immediately and binds a +// stream handle; `write line`/`write chunk` append body pieces incrementally, +// and `close` ends the response. +// +// Validated for syntax/analysis/lint only (layers 1-4): running it needs a +// live client, so execution is skipped. + +listen on port 8080 as site + +wait for request comes in on site as req with timeout 10000 + +start streaming response to req with status 200 and content type "application/x-ndjson" as out + +write line "first line" to out +write line "second line" to out +flush out +write chunk "no newline here" to out + +close out +close server site \ No newline at end of file diff --git a/TestPrograms/parser_streaming_compat_regression.test.wfl b/TestPrograms/parser_streaming_compat_regression.test.wfl new file mode 100644 index 00000000..4b18947d --- /dev/null +++ b/TestPrograms/parser_streaming_compat_regression.test.wfl @@ -0,0 +1,50 @@ +// End-to-end compatibility coverage for syntax made ambiguous by HTTP streaming. + +define action called flush: + display "FLUSH_ACTION_CALLED" +end action + +define action called acquire stream: + return 1 +end action + +describe "Parser streaming compatibility": + + test "classic write keeps a with continuation on a variable named line": + store line as ["hello"] + write line with "!" to "parser_streaming_compat_with.txt" + open file at "parser_streaming_compat_with.txt" for reading as with_reader + wait for store with_result as read content from with_reader + close with_reader + expect with_result to contain "!" + delete file at "parser_streaming_compat_with.txt" + end test + + test "classic write keeps an at continuation on a variable named line": + store line as ["first"] + write line at 0 to "parser_streaming_compat_at.txt" + open file at "parser_streaming_compat_at.txt" for reading as at_reader + wait for store at_result as read content from at_reader + close at_reader + expect at_result to equal "first" + delete file at "parser_streaming_compat_at.txt" + end test + + test "classic write keeps a bracket continuation on a variable named line": + store line as ["first"] + write line[0] to "parser_streaming_compat_bracket.txt" + open file at "parser_streaming_compat_bracket.txt" for reading as bracket_reader + wait for store bracket_result as read content from bracket_reader + close bracket_reader + expect bracket_result to equal "first" + delete file at "parser_streaming_compat_bracket.txt" + end test + + test "exact flush forms keep the zero argument action": + store ignored as 1 + flush (ignored) + flush call acquire stream + expect ignored to equal 1 + end test + +end describe diff --git a/TestPrograms/simple_web_test.wfl b/TestPrograms/simple_web_test.wfl index e72f9561..14e0682d 100644 --- a/TestPrograms/simple_web_test.wfl +++ b/TestPrograms/simple_web_test.wfl @@ -14,4 +14,11 @@ display "Got a request!" respond to incoming_request with "Hello from WFL!" display "Response sent!" + +// `respond` hands the reply to the async transport; reaching EOF immediately +// can tear the runtime down before the socket flushes. `close server` has a +// short grace that lets the pending response drain first, so the single-shot +// client reliably receives the body. +close server test_server + display "=== Test Complete ===" diff --git a/TestPrograms/subprocess_blocking_helper.wfl b/TestPrograms/subprocess_blocking_helper.wfl new file mode 100644 index 00000000..e83f9297 --- /dev/null +++ b/TestPrograms/subprocess_blocking_helper.wfl @@ -0,0 +1,4 @@ +// CI-SKIP: helper process for subprocess_comprehensive.wfl +// Helper for subprocess_comprehensive.wfl. The integration runner skips this +// standalone file; its parent starts it and proves live-process termination. +wait for 10 seconds diff --git a/TestPrograms/subprocess_comprehensive.wfl b/TestPrograms/subprocess_comprehensive.wfl index 4d6fbd77..1d0ed6d1 100644 --- a/TestPrograms/subprocess_comprehensive.wfl +++ b/TestPrograms/subprocess_comprehensive.wfl @@ -6,19 +6,19 @@ display "" // Test 1: Simple Command Execution display "Test 1: Execute Command" -wait for execute command "echo Hello from subprocess" as cmd_result +wait for execute command "cargo --version" as cmd_result display " Command executed successfully" display "" // Test 2: Execute Without Storing Result display "Test 2: Execute Without Variable" -wait for execute command "echo No variable needed" +wait for execute command "cargo --version" display " Execution completed" display "" // Test 3: Background Process Spawn and Wait display "Test 3: Spawn and Wait for Process" -wait for spawn command "echo Background process" as bg_proc +wait for spawn command "cargo --version" as bg_proc display " Process spawned" wait for process bg_proc to complete as exit_status display " Process completed" @@ -26,7 +26,7 @@ display "" // Test 4: Process Status Check display "Test 4: Check Process Status" -wait for spawn command "echo Quick task" as status_proc +wait for spawn command "cargo --version" as status_proc store proc_status as process status_proc is running check if proc_status: display " Process was running (or completed too fast)" @@ -37,17 +37,48 @@ display "" // Test 5: Process Termination display "Test 5: Kill Process" -wait for spawn command "echo Terminated" as term_proc -wait for 100 milliseconds -kill process term_proc +// Use the repo-owned WFL binary and a blocking fixture, then prove the child is +// live before termination and absent afterward. +check if file exists at "target/release/wfl.exe": + wait for spawn command "target/release/wfl.exe" with arguments ["TestPrograms/subprocess_blocking_helper.wfl"] as windows_term_proc + wait for 100 milliseconds + store windows_term_running as process windows_term_proc is running + check if windows_term_running: + kill process windows_term_proc + otherwise: + store windows_live_kill_failure as 1 divided by 0 + display windows_live_kill_failure + end check + store windows_term_after_kill as process windows_term_proc is running + check if windows_term_after_kill: + store windows_post_kill_failure as 1 divided by 0 + display windows_post_kill_failure + end check +otherwise: + wait for spawn command "target/release/wfl" with arguments ["TestPrograms/subprocess_blocking_helper.wfl"] as unix_term_proc + wait for 100 milliseconds + store unix_term_running as process unix_term_proc is running + check if unix_term_running: + kill process unix_term_proc + otherwise: + store unix_live_kill_failure as 1 divided by 0 + display unix_live_kill_failure + end check + store unix_term_after_kill as process unix_term_proc is running + check if unix_term_after_kill: + store unix_post_kill_failure as 1 divided by 0 + display unix_post_kill_failure + end check +end check display " Process terminated" display "" // Test 6: Read Process Output display "Test 6: Capture Process Output" -wait for spawn command "echo Output captured" as out_proc +wait for spawn command "cargo --version" as out_proc wait for 200 milliseconds wait for read output from process out_proc as captured_data +wait for process out_proc to complete display " Output captured successfully" display "" @@ -63,8 +94,8 @@ display "" // Test 8: Multiple Processes display "Test 8: Multiple Concurrent Processes" -wait for spawn command "echo Process 1" as p1 -wait for spawn command "echo Process 2" as p2 +wait for spawn command "cargo --version" as p1 +wait for spawn command "cargo --version" as p2 display " Two processes spawned" wait for process p1 to complete wait for process p2 to complete diff --git a/crates/wflpkg/src/archive.rs b/crates/wflpkg/src/archive.rs index b244ab31..77fcf5fe 100644 --- a/crates/wflpkg/src/archive.rs +++ b/crates/wflpkg/src/archive.rs @@ -267,7 +267,9 @@ pub fn extract_archive(archive_path: &Path, dest_dir: &Path) -> Result<(), Packa .into_owned(); // Reject absolute paths - if entry_path.is_absolute() { + // `Path::is_absolute` requires a drive prefix on Windows, but archive + // paths are portable and a leading slash is still rooted there. + if entry_path.has_root() { return Err(PackageError::General(format!( "Archive contains absolute path: {}", entry_path.display() diff --git a/scripts/run_integration_tests.ps1 b/scripts/run_integration_tests.ps1 index a630f4f7..0bc49a27 100644 --- a/scripts/run_integration_tests.ps1 +++ b/scripts/run_integration_tests.ps1 @@ -22,6 +22,52 @@ if ($Help) { exit 0 } +# Windows treats environment variable names case-insensitively, but a process +# launched from a cross-platform host can still inherit both Path and PATH. +# Windows PowerShell 5.1's Start-Process rejects that environment block. Keep a +# single canonical key only when the duplicate values are identical; never +# merge conflicting executable search paths. +if ([System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT) { + $processEnvironment = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + $pathKeys = @( + $processEnvironment.Keys | Where-Object { + [string]::Equals( + [string]$_, + "Path", + [System.StringComparison]::OrdinalIgnoreCase + ) + } + ) + + if ($pathKeys.Count -gt 1) { + $pathValue = [string]$processEnvironment[$pathKeys[0]] + foreach ($pathKey in $pathKeys) { + if (-not [string]::Equals( + $pathValue, + [string]$processEnvironment[$pathKey], + [System.StringComparison]::Ordinal + )) { + throw "Conflicting case-variant PATH values in the process environment." + } + } + + foreach ($pathKey in $pathKeys) { + [System.Environment]::SetEnvironmentVariable( + [string]$pathKey, + $null, + [System.EnvironmentVariableTarget]::Process + ) + } + [System.Environment]::SetEnvironmentVariable( + "Path", + $pathValue, + [System.EnvironmentVariableTarget]::Process + ) + } +} + Write-Host "[INFO] WFL Integration Test Runner" -ForegroundColor Blue Write-Host "[INFO] ==========================" -ForegroundColor Blue @@ -112,7 +158,8 @@ $SkipTests = @( "websocket_test.wfl", # WebSocket - needs WS client "web_route_params_test.wfl", # Web server - tested via run_web_tests.ps1 "module_helper.wfl", # Helper module, not a standalone program - "module_bare_zero_arg_helper.wfl" # Helper module for #592 fixture, not standalone + "module_bare_zero_arg_helper.wfl", # Helper module for #592 fixture, not standalone + "subprocess_blocking_helper.wfl" # Helper process for subprocess_comprehensive.wfl ) # Tests that intentionally end with an error; they pass when wfl exits nonzero @@ -163,8 +210,19 @@ if (-not (Test-Path "TestPrograms")) { Write-Host "[INFO] Testing: $($wflFile.Name)" -ForegroundColor Blue - # Run with timeout to prevent hangs - $process = Start-Process -FilePath ".\$BinaryPath" -ArgumentList $wflArgs -NoNewWindow -PassThru -RedirectStandardOutput "NUL" -RedirectStandardError "NUL" + # Run with timeout to prevent hangs. Start-Process requires DISTINCT + # file targets for stdout and stderr — PowerShell 7 errors when the same + # path is reused for both, and "NUL" is not a valid redirect target + # there — so redirect to two temp files and discard them. (Redirecting + # both to a single "NUL" left the whole Windows integration command + # unrunnable, so its assertions never actually ran.) + $outFile = New-TemporaryFile + $errFile = New-TemporaryFile + $process = Start-Process -FilePath ".\$BinaryPath" -ArgumentList $wflArgs -NoNewWindow -PassThru -RedirectStandardOutput $outFile.FullName -RedirectStandardError $errFile.FullName + # Windows PowerShell 5.1 can discard the process handle before a + # timed WaitForExit, leaving ExitCode null. Materialize it while the + # child is live so timeout and exit-code assertions remain valid. + $null = $process.Handle $completed = $process.WaitForExit($TestTimeout * 1000) $isExpectedFail = $ExpectedFailTests -contains $wflFile.Name @@ -187,6 +245,7 @@ if (-not (Test-Path "TestPrograms")) { Write-Host "[ERROR] FAIL $($wflFile.Name) (exit code: $($process.ExitCode))" -ForegroundColor Red $failedPrograms++ } + Remove-Item $outFile.FullName, $errFile.FullName -ErrorAction SilentlyContinue } Write-Host "" diff --git a/scripts/run_integration_tests.sh b/scripts/run_integration_tests.sh old mode 100644 new mode 100755 diff --git a/scripts/run_web_tests.ps1 b/scripts/run_web_tests.ps1 index 9ab1125e..c4b549a7 100644 --- a/scripts/run_web_tests.ps1 +++ b/scripts/run_web_tests.ps1 @@ -22,6 +22,52 @@ if ($Help) { exit 0 } +# Windows treats environment variable names case-insensitively, but a process +# launched from a cross-platform host can still inherit both Path and PATH. +# Windows PowerShell 5.1's Start-Process rejects that environment block. Keep a +# single canonical key only when the duplicate values are identical; never +# merge conflicting executable search paths. +if ([System.Environment]::OSVersion.Platform -eq [System.PlatformID]::Win32NT) { + $processEnvironment = [System.Environment]::GetEnvironmentVariables( + [System.EnvironmentVariableTarget]::Process + ) + $pathKeys = @( + $processEnvironment.Keys | Where-Object { + [string]::Equals( + [string]$_, + "Path", + [System.StringComparison]::OrdinalIgnoreCase + ) + } + ) + + if ($pathKeys.Count -gt 1) { + $pathValue = [string]$processEnvironment[$pathKeys[0]] + foreach ($pathKey in $pathKeys) { + if (-not [string]::Equals( + $pathValue, + [string]$processEnvironment[$pathKey], + [System.StringComparison]::Ordinal + )) { + throw "Conflicting case-variant PATH values in the process environment." + } + } + + foreach ($pathKey in $pathKeys) { + [System.Environment]::SetEnvironmentVariable( + [string]$pathKey, + $null, + [System.EnvironmentVariableTarget]::Process + ) + } + [System.Environment]::SetEnvironmentVariable( + "Path", + $pathValue, + [System.EnvironmentVariableTarget]::Process + ) + } +} + Write-Host "[INFO] WFL Web Server Test Runner" -ForegroundColor Blue Write-Host "[INFO] ============================" -ForegroundColor Blue @@ -40,6 +86,97 @@ if (-not (Test-Path $BinaryPath)) { } Write-Host "[SUCCESS] Binary found: $BinaryPath" -ForegroundColor Green +# Dump a server's captured stdout/stderr on failure. Without this a genuine +# server error (a panic, a bind failure, a bad response) is invisible behind the +# runner's generic TIMEOUT/assertion message, since the process output is +# redirected to files. Call on every failure path before returning. +function Show-ServerLogs { + param( + [string]$OutLog, + [string]$ErrLog, + $Process + ) + if ($Process) { + if ($Process.HasExited) { + Write-Host "[LOG] server process exited with code $($Process.ExitCode)" -ForegroundColor Gray + } else { + Write-Host "[LOG] server process was still running at failure time" -ForegroundColor Gray + } + } + foreach ($pair in @(@("stdout", $OutLog), @("stderr", $ErrLog))) { + $label = $pair[0] + $path = $pair[1] + if ($path -and (Test-Path $path)) { + $content = (Get-Content -Raw -ErrorAction SilentlyContinue $path) + if ([string]::IsNullOrWhiteSpace($content)) { + Write-Host "[LOG] server $label ($path): " -ForegroundColor Gray + } else { + Write-Host "[LOG] server $label ($path):" -ForegroundColor Gray + Write-Host $content -ForegroundColor Gray + } + } + } +} + +# Kill a background server process and wait for it to actually exit, so any temp +# files/handles it holds are released before the caller removes them (avoids +# Windows cleanup races on the TLS temp dir). +function Stop-ServerProcess { + param($Process) + if ($Process -and -not $Process.HasExited) { + # Kill() itself can throw (e.g. access denied, or the process exiting + # concurrently), so guard it too rather than leaving it outside the try. + # A failure to kill/wait is a REAL failure — a leaked server holds its + # cert/log handles and can race the temp-dir cleanup — so record it so the + # run fails rather than reporting a false pass. (The pass count is + # incremented before the `finally` cleanup, so a leak here must be able to + # turn the overall result red.) + try { + $Process.Kill() + } catch { + Write-Host "[ERROR] Kill() on server process failed: $_" -ForegroundColor Red + $script:cleanupFailed = $true + } + # WaitForExit(ms) returns $true only if the process actually exited in + # time; report honestly rather than always claiming success (a process + # still alive can race temp-file/cert cleanup that follows). Callers that + # then delete temp files re-check HasExited before doing so. + $exited = $false + try { $exited = $Process.WaitForExit(5000) } catch { } + if ($exited) { + Write-Host "[INFO] Server process terminated" -ForegroundColor Gray + } else { + Write-Host "[ERROR] Server process did not exit within 5s of Kill()" -ForegroundColor Red + $script:cleanupFailed = $true + } + } +} + +# Extract the Location header from a redirect response across PowerShell/response +# shapes. With -MaximumRedirection 0, pwsh 7 throws and $_.Exception.Response is +# an HttpResponseMessage whose Headers has NO string indexer, so ["Location"] +# silently returns $null; its Location is the strongly-typed .Headers.Location +# (a Uri). Invoke-WebRequest's own response object (and Windows PowerShell 5.1's +# HttpWebResponse) use a string indexer instead, whose value may be a string[]. +function Get-LocationHeader { + param($Response) + if ($null -eq $Response) { return $null } + # Match the HttpResponseMessage shape by type NAME, not the type literal + # [System.Net.Http.HttpResponseMessage]: a clean Windows PowerShell 5.1 + # process may not have System.Net.Http loaded, so resolving the literal would + # throw before the HttpWebResponse string-indexer fallback below. + if ($Response.GetType().FullName -eq 'System.Net.Http.HttpResponseMessage') { + if ($Response.Headers.Location) { return $Response.Headers.Location.AbsoluteUri } + return $null + } + try { + $loc = $Response.Headers["Location"] + if ($loc -is [array]) { $loc = $loc[0] } + if ($loc) { return [string]$loc } + } catch { } + return $null +} + # Function to test a web server function Test-WflWebServer { param( @@ -53,30 +190,34 @@ function Test-WflWebServer { Write-Host "" Write-Host "[INFO] Testing: $testName on port $Port" -ForegroundColor Blue - # Start the WFL server in background - $serverProcess = Start-Process -FilePath ".\$BinaryPath" -ArgumentList $TestFile -NoNewWindow -PassThru -RedirectStandardOutput "NUL" -RedirectStandardError "NUL" + # Start the WFL server in background. Start-Process rejects the same target + # for both redirects (the "NUL"/"NUL" collision errored on PowerShell 7), so + # discard stdout and stderr to two distinct, port-keyed temp files. + $outLog = Join-Path ([System.IO.Path]::GetTempPath()) "wfl_web_$Port.out.log" + $errLog = Join-Path ([System.IO.Path]::GetTempPath()) "wfl_web_$Port.err.log" + $serverProcess = Start-Process -FilePath ".\$BinaryPath" -ArgumentList $TestFile -NoNewWindow -PassThru -RedirectStandardOutput $outLog -RedirectStandardError $errLog + $null = $serverProcess.Handle try { - # Wait for server to start (with retries) + # Wait for the server to start, bounded by a real wall-clock deadline. + # A fixed retry count misleads: each failed 2s request stacks on top of + # the 500ms sleep, so "20 tries" of a 10s wait could actually run ~50s. + # A Stopwatch caps total wait at TimeoutSeconds (plus one in-flight probe). $serverReady = $false - $retries = 0 - $maxRetries = $TimeoutSeconds * 2 # Check every 500ms - - while (-not $serverReady -and $retries -lt $maxRetries) { - Start-Sleep -Milliseconds 500 - $retries++ - - # Try to connect + $deadline = [System.Diagnostics.Stopwatch]::StartNew() + while (-not $serverReady -and $deadline.Elapsed.TotalSeconds -lt $TimeoutSeconds) { try { - $response = Invoke-WebRequest -Uri "http://localhost:$Port/" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop + $response = Invoke-WebRequest -Uri "http://127.0.0.1:$Port/" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop $serverReady = $true } catch { - # Server not ready yet, continue waiting + # Server not ready yet, wait briefly and retry until the deadline. + Start-Sleep -Milliseconds 500 } } if (-not $serverReady) { Write-Host "[ERROR] TIMEOUT: Server did not start within ${TimeoutSeconds}s" -ForegroundColor Red + Show-ServerLogs -OutLog $outLog -ErrLog $errLog -Process $serverProcess return $false } @@ -88,13 +229,13 @@ function Test-WflWebServer { Write-Host "[ERROR] FAIL: Unexpected response" -ForegroundColor Red Write-Host " Expected: $ExpectedResponse" -ForegroundColor Gray Write-Host " Got: $($response.Content)" -ForegroundColor Gray + Show-ServerLogs -OutLog $outLog -ErrLog $errLog -Process $serverProcess return $false } } finally { - # Clean up - kill the server + # Clean up - kill the server and wait for exit if (-not $serverProcess.HasExited) { - $serverProcess.Kill() - Write-Host "[INFO] Server process terminated" -ForegroundColor Gray + Stop-ServerProcess -Process $serverProcess } } } @@ -102,6 +243,10 @@ function Test-WflWebServer { # Run web server tests $totalTests = 0 $passedTests = 0 +# Set true by cleanup that leaks (a server that will not die, or a temp dir that +# will not delete). Because a test's pass is counted before its `finally` cleanup +# runs, a leak here must be able to fail the overall run — see the summary below. +$script:cleanupFailed = $false # Test 1: simple_web_test.wfl if (Test-Path "TestPrograms\simple_web_test.wfl") { @@ -131,24 +276,27 @@ if (Test-Path "TestPrograms\web_route_params_test.wfl") { Write-Host "" Write-Host "[INFO] Testing: web_route_params_test.wfl on port 8096" -ForegroundColor Blue - $routeProcess = Start-Process -FilePath ".\$BinaryPath" -ArgumentList "TestPrograms\web_route_params_test.wfl" -NoNewWindow -PassThru -RedirectStandardOutput "NUL" -RedirectStandardError "NUL" + # Distinct redirect targets (see the note in Test-WflWebServer): the same + # path for both streams errors on PowerShell 7. + $routeOutLog = Join-Path ([System.IO.Path]::GetTempPath()) "wfl_web_route.out.log" + $routeErrLog = Join-Path ([System.IO.Path]::GetTempPath()) "wfl_web_route.err.log" + $routeProcess = Start-Process -FilePath ".\$BinaryPath" -ArgumentList "TestPrograms\web_route_params_test.wfl" -NoNewWindow -PassThru -RedirectStandardOutput $routeOutLog -RedirectStandardError $routeErrLog + $null = $routeProcess.Handle try { + # Wall-clock deadline (see Test-WflWebServer) instead of a retry count. $serverReady = $false - $retries = 0 - $maxRetries = $Timeout * 2 - - while (-not $serverReady -and $retries -lt $maxRetries) { - Start-Sleep -Milliseconds 500 - $retries++ + $deadline = [System.Diagnostics.Stopwatch]::StartNew() + while (-not $serverReady -and $deadline.Elapsed.TotalSeconds -lt $Timeout) { try { - $rootResponse = Invoke-WebRequest -Uri "http://localhost:8096/" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop + $rootResponse = Invoke-WebRequest -Uri "http://127.0.0.1:8096/" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop if ($rootResponse.Content -like "*Route server ready*") { $serverReady = $true } } catch { # Intentionally empty - server not ready yet, continue polling } + if (-not $serverReady) { Start-Sleep -Milliseconds 500 } } $routeOk = $true @@ -156,18 +304,25 @@ if (Test-Path "TestPrograms\web_route_params_test.wfl") { Write-Host "[ERROR] TIMEOUT: Route params server did not start within ${Timeout}s" -ForegroundColor Red $routeOk = $false } else { - # Route parameter extraction: /users/:id - $userResponse = Invoke-WebRequest -Uri "http://localhost:8096/users/42" -TimeoutSec 2 -UseBasicParsing - if ($userResponse.Content -like "*User 42*") { - Write-Host "[SUCCESS] PASS: /users/42 -> '$($userResponse.Content)'" -ForegroundColor Green - } else { - Write-Host "[ERROR] FAIL: /users/42 returned '$($userResponse.Content)'" -ForegroundColor Red + # Route parameter extraction: /users/:id. Wrapped so a request + # failure marks the test failed (and dumps server logs below) instead + # of throwing out of the script and skipping the summary/other tests. + try { + $userResponse = Invoke-WebRequest -Uri "http://127.0.0.1:8096/users/42" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop + if ($userResponse.Content -like "*User 42*") { + Write-Host "[SUCCESS] PASS: /users/42 -> '$($userResponse.Content)'" -ForegroundColor Green + } else { + Write-Host "[ERROR] FAIL: /users/42 returned '$($userResponse.Content)'" -ForegroundColor Red + $routeOk = $false + } + } catch { + Write-Host "[ERROR] FAIL: /users/42 request failed: $_" -ForegroundColor Red $routeOk = $false } # Non-matching route returns 404 try { - Invoke-WebRequest -Uri "http://localhost:8096/missing" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop | Out-Null + Invoke-WebRequest -Uri "http://127.0.0.1:8096/missing" -TimeoutSec 2 -UseBasicParsing -ErrorAction Stop | Out-Null Write-Host "[ERROR] FAIL: unknown route did not return 404" -ForegroundColor Red $routeOk = $false } catch { @@ -180,21 +335,27 @@ if (Test-Path "TestPrograms\web_route_params_test.wfl") { } # Header access regression - $agentResponse = Invoke-WebRequest -Uri "http://localhost:8096/agent" -TimeoutSec 2 -UseBasicParsing -UserAgent "wfl-route-test" - if ($agentResponse.Content -like "*wfl-route-test*") { - Write-Host "[SUCCESS] PASS: header access echoes User-Agent" -ForegroundColor Green - } else { - Write-Host "[ERROR] FAIL: /agent returned '$($agentResponse.Content)'" -ForegroundColor Red + try { + $agentResponse = Invoke-WebRequest -Uri "http://127.0.0.1:8096/agent" -TimeoutSec 2 -UseBasicParsing -UserAgent "wfl-route-test" -ErrorAction Stop + if ($agentResponse.Content -like "*wfl-route-test*") { + Write-Host "[SUCCESS] PASS: header access echoes User-Agent" -ForegroundColor Green + } else { + Write-Host "[ERROR] FAIL: /agent returned '$($agentResponse.Content)'" -ForegroundColor Red + $routeOk = $false + } + } catch { + Write-Host "[ERROR] FAIL: /agent request failed: $_" -ForegroundColor Red $routeOk = $false } } - if ($routeOk) { $passedTests++ } - } finally { - if (-not $routeProcess.HasExited) { - $routeProcess.Kill() - Write-Host "[INFO] Server process terminated" -ForegroundColor Gray + if ($routeOk) { + $passedTests++ + } else { + Show-ServerLogs -OutLog $routeOutLog -ErrLog $routeErrLog -Process $routeProcess } + } finally { + Stop-ServerProcess -Process $routeProcess } } @@ -215,25 +376,27 @@ if (Test-Path "TestPrograms\web_server_tls.wfl") { # The test program uses relative cert paths, so run it from the temp dir $absBinary = Join-Path (Get-Location) $BinaryPath $absTest = Join-Path (Get-Location) "TestPrograms\web_server_tls.wfl" - $tlsProcess = Start-Process -FilePath $absBinary -ArgumentList $absTest -WorkingDirectory $tlsDir -NoNewWindow -PassThru -RedirectStandardOutput "NUL" -RedirectStandardError "NUL" + # Distinct redirect targets under the (auto-cleaned) temp dir; the same + # path for both streams errors on PowerShell 7. + $tlsProcess = Start-Process -FilePath $absBinary -ArgumentList $absTest -WorkingDirectory $tlsDir -NoNewWindow -PassThru -RedirectStandardOutput (Join-Path $tlsDir "server.out.log") -RedirectStandardError (Join-Path $tlsDir "server.err.log") + $null = $tlsProcess.Handle try { # Probe readiness via the redirect port: it answers natively and does # not consume the program's single `wait for request` + # Wall-clock deadline (see Test-WflWebServer) instead of a retry count. $serverReady = $false - $retries = 0 - $maxRetries = $Timeout * 2 - while (-not $serverReady -and $retries -lt $maxRetries) { - Start-Sleep -Milliseconds 500 - $retries++ + $deadline = [System.Diagnostics.Stopwatch]::StartNew() + while (-not $serverReady -and $deadline.Elapsed.TotalSeconds -lt $Timeout) { try { - Invoke-WebRequest -Uri "http://localhost:8090/" -TimeoutSec 2 -UseBasicParsing -MaximumRedirection 0 -ErrorAction Stop | Out-Null + Invoke-WebRequest -Uri "http://127.0.0.1:8090/" -TimeoutSec 2 -UseBasicParsing -MaximumRedirection 0 -ErrorAction Stop | Out-Null $serverReady = $true } catch { if ($_.Exception.Response -and [int]$_.Exception.Response.StatusCode -eq 301) { $serverReady = $true } } + if (-not $serverReady) { Start-Sleep -Milliseconds 500 } } $tlsOk = $true @@ -244,14 +407,14 @@ if (Test-Path "TestPrograms\web_server_tls.wfl") { # Redirect server: 301 with Location preserving path/query on the HTTPS port $location = $null try { - $redirectResponse = Invoke-WebRequest -Uri "http://localhost:8090/some/path?x=1" -TimeoutSec 2 -UseBasicParsing -MaximumRedirection 0 -ErrorAction Stop - $location = $redirectResponse.Headers["Location"] + $redirectResponse = Invoke-WebRequest -Uri "http://127.0.0.1:8090/some/path?x=1" -TimeoutSec 2 -UseBasicParsing -MaximumRedirection 0 -ErrorAction Stop + $location = Get-LocationHeader -Response $redirectResponse } catch { if ($_.Exception.Response) { - $location = $_.Exception.Response.Headers["Location"] + $location = Get-LocationHeader -Response $_.Exception.Response } } - if ($location -eq "https://localhost:8443/some/path?x=1") { + if ($location -eq "https://127.0.0.1:8443/some/path?x=1") { Write-Host "[SUCCESS] PASS: redirect returns 301 to $location" -ForegroundColor Green } else { Write-Host "[ERROR] FAIL: redirect Location was '$location'" -ForegroundColor Red @@ -263,7 +426,7 @@ if (Test-Path "TestPrograms\web_server_tls.wfl") { # PowerShell 5.1 skip this check gracefully instead of failing. if ($PSVersionTable.PSVersion.Major -ge 6) { try { - $httpsResponse = Invoke-WebRequest -Uri "https://localhost:8443/" -TimeoutSec 3 -UseBasicParsing -SkipCertificateCheck -ErrorAction Stop + $httpsResponse = Invoke-WebRequest -Uri "https://127.0.0.1:8443/" -TimeoutSec 3 -UseBasicParsing -SkipCertificateCheck -ErrorAction Stop if ($httpsResponse.Content -like "*Hello over HTTPS!*") { Write-Host "[SUCCESS] PASS: HTTPS response '$($httpsResponse.Content)'" -ForegroundColor Green } else { @@ -279,13 +442,33 @@ if (Test-Path "TestPrograms\web_server_tls.wfl") { } } - if ($tlsOk) { $passedTests++ } + if ($tlsOk) { + $passedTests++ + } else { + Show-ServerLogs -OutLog (Join-Path $tlsDir "server.out.log") -ErrLog (Join-Path $tlsDir "server.err.log") -Process $tlsProcess + } } finally { - if (-not $tlsProcess.HasExited) { - $tlsProcess.Kill() - Write-Host "[INFO] Server process terminated" -ForegroundColor Gray + # Kill AND wait for exit before removing the temp dir, so the server + # has released its cert/log file handles (avoids a Windows cleanup + # race that would leave the dir or fail the Remove-Item). + Stop-ServerProcess -Process $tlsProcess + if ($tlsProcess -and -not $tlsProcess.HasExited) { + # Still alive after Kill()+WaitForExit — deleting now races the + # process's open cert/log handles. Give a brief extra grace before + # attempting cleanup so we don't fight live handles. + Write-Host "[WARN] TLS server still running after Kill(); waiting briefly before temp cleanup" -ForegroundColor Yellow + try { $null = $tlsProcess.WaitForExit(2000) } catch { } + } + # Attempt cleanup and FAIL the run on a leaked temp dir / still-open + # handle instead of hiding it behind a warning — the test's pass was + # already counted above, so a cleanup leak must be able to turn the + # result red. + try { + Remove-Item -Recurse -Force $tlsDir -ErrorAction Stop + } catch { + Write-Host "[ERROR] Failed to remove TLS temp dir ${tlsDir}: $_" -ForegroundColor Red + $script:cleanupFailed = $true } - Remove-Item -Recurse -Force $tlsDir -ErrorAction SilentlyContinue } } } @@ -295,7 +478,10 @@ Write-Host "" Write-Host "[INFO] ============================" -ForegroundColor Blue Write-Host "[INFO] Results: $passedTests/$totalTests tests passed" -ForegroundColor Blue -if ($passedTests -eq $totalTests) { +if ($script:cleanupFailed) { + Write-Host "[ERROR] A server/temp-dir cleanup leaked; failing the run even though ${passedTests}/${totalTests} test assertions passed" -ForegroundColor Red + exit 1 +} elseif ($passedTests -eq $totalTests) { Write-Host "[SUCCESS] All web server tests passed!" -ForegroundColor Green exit 0 } else { diff --git a/scripts/run_web_tests.sh b/scripts/run_web_tests.sh old mode 100644 new mode 100755 diff --git a/scripts/validate_docs_examples.py b/scripts/validate_docs_examples.py old mode 100644 new mode 100755 diff --git a/src/analyzer/mod.rs b/src/analyzer/mod.rs index 57a40136..c2519f4b 100644 --- a/src/analyzer/mod.rs +++ b/src/analyzer/mod.rs @@ -713,6 +713,38 @@ impl Analyzer { self.budget_error.take() } + /// Analyze the body of an unbounded loop (`forever` / `main loop`). Shared by + /// both so the scope handling, flow tracking, and the handler-body error + /// demotion stay identical. Web-server main-loop bodies reference + /// handler-provided names the analyzer cannot model, so errors raised *inside* + /// the body are demoted to warnings (a latched budget breach stays fatal); + /// errors the caller raises about the loop itself are pushed before this runs + /// and are not swept up. + fn analyze_loop_body(&mut self, body: &[Statement]) { + let outer_scope = std::mem::take(&mut self.current_scope); + self.current_scope = Scope::with_parent(outer_scope); + + let flow_entry = self.flow_entry(); + self.push_mutation_frame(); + let errors_before = self.errors.len(); + for stmt in body { + self.analyze_statement(stmt); + } + if self.budget_error.is_none() { + let demoted: Vec<_> = self.errors.drain(errors_before..).collect(); + self.warnings.extend(demoted); + } + let flow_body = self.take_flow_branch(&flow_entry); + let mutated = self.pop_mutation_frame(); + self.join_flow_branches(&[flow_body, flow_entry]); + self.degrade_mutated_aliases(&mutated); + + let loop_scope = std::mem::take(&mut self.current_scope); + if let Some(parent) = loop_scope.parent { + self.current_scope = *parent; + } + } + /// Report an undefined-name reference. Inside a `try` body this is a /// warning rather than a fatal error: the reference raises a catchable /// runtime error, which is documented behavior that programs rely on. @@ -1197,33 +1229,41 @@ impl Analyzer { self.current_scope = *parent; } } - Statement::ForeverLoop { body, .. } | Statement::MainLoop { body, .. } => { - let outer_scope = std::mem::take(&mut self.current_scope); - self.current_scope = Scope::with_parent(outer_scope); - - let flow_entry = self.flow_entry(); - self.push_mutation_frame(); - let errors_before = self.errors.len(); - for stmt in body { - self.analyze_statement(stmt); - } - // Same demotion as the repeat forms: web-server main loops - // reference handler-provided names this analyzer cannot - // model, and these bodies were previously unanalyzed. A - // latched budget breach stays fatal (see the repeat forms). - if self.budget_error.is_none() { - let demoted: Vec<_> = self.errors.drain(errors_before..).collect(); - self.warnings.extend(demoted); - } - let flow_body = self.take_flow_branch(&flow_entry); - let mutated = self.pop_mutation_frame(); - self.join_flow_branches(&[flow_body, flow_entry]); - self.degrade_mutated_aliases(&mutated); - - let loop_scope = std::mem::take(&mut self.current_scope); - if let Some(parent) = loop_scope.parent { - self.current_scope = *parent; + Statement::ForeverLoop { body, .. } => { + self.analyze_loop_body(body); + } + Statement::MainLoop { + body, + concurrent, + line, + column, + } => { + // `main loop concurrently:` starts up to the concurrency cap of + // handler futures at once, each running the body from the top. Any + // statement before the first `wait for request` therefore runs + // once *per handler slot* before a single request is dequeued — + // speculative side effects the author almost never intends. Require + // the body to begin with `wait for request` so nothing runs before + // a request is in hand. (Serial `main loop` has no such hazard.) + // Emitted before the body is analyzed so it is NOT swept into the + // handler-body error demotion below. + if *concurrent + && !matches!( + body.first(), + Some(Statement::WaitForRequestStatement { .. }) + ) + { + self.errors.push(SemanticError::new( + "A `main loop concurrently:` body must begin with `wait for request ...`. \ + Concurrent handlers start before any request arrives, so statements before \ + the first `wait for request` run once per handler slot. Move that setup above \ + the loop, or make `wait for request` the first statement in the loop." + .to_string(), + *line, + *column, + )); } + self.analyze_loop_body(body); } Statement::DisplayStatement { value, .. } => { self.analyze_expression(value); @@ -1321,15 +1361,19 @@ impl Analyzer { let flow_handler_entry = self.flow_entry(); let mut flow_paths: Vec = vec![flow_try]; - let try_scope = std::mem::take(&mut self.current_scope); - if let Some(parent) = try_scope.parent { - self.current_scope = *parent; - } + // Runtime keeps this try child environment alive through the + // selected handler/otherwise clause and finally. Snapshot the + // post-body structure so every statically possible clause is + // analyzed independently, then union its ordinary bindings + // back into the shared try scope for finally. + let clause_entry_scope = self.current_scope.clone(); + let mut joined_scope_symbols = clause_entry_scope.symbols.clone(); // Analyze each when clause for when_clause in when_clauses { - let outer_scope = std::mem::take(&mut self.current_scope); - self.current_scope = Scope::with_parent(outer_scope); + self.current_scope = clause_entry_scope.clone(); + self.restore_flow(&flow_handler_entry); + self.push_scope(); let error_symbol = Symbol { name: when_clause.error_name.clone(), @@ -1339,9 +1383,7 @@ impl Analyzer { column: 0, }; - if let Err(error) = self.current_scope.define(error_symbol) { - self.errors.push(error); - } + self.define_or_replace_symbol(error_symbol); // `error_message` is always available in error-handling // clauses as an alias for the caught error's message. @@ -1353,53 +1395,66 @@ impl Analyzer { line: 0, column: 0, }; - let _ = self.current_scope.define(error_message_symbol); + self.define_or_replace_symbol(error_message_symbol); } - self.restore_flow(&flow_handler_entry); for stmt in &when_clause.body { self.analyze_statement(stmt); } - flow_paths.push(self.take_flow_branch(&flow_handler_entry)); + let mut excluded_aliases = vec![when_clause.error_name.clone()]; + if when_clause.error_name != "error_message" { + excluded_aliases.push("error_message".to_string()); + } + self.pop_scope_promoting_except(&excluded_aliases); - let when_scope = std::mem::take(&mut self.current_scope); - if let Some(parent) = when_scope.parent { - self.current_scope = *parent; + flow_paths.push(self.take_flow_branch(&flow_handler_entry)); + for (name, symbol) in &self.current_scope.symbols { + joined_scope_symbols + .entry(name.clone()) + .or_insert_with(|| symbol.clone()); } } if let Some(otherwise_stmts) = otherwise_block { - let outer_scope = std::mem::take(&mut self.current_scope); - self.current_scope = Scope::with_parent(outer_scope); - + self.current_scope = clause_entry_scope.clone(); self.restore_flow(&flow_handler_entry); for stmt in otherwise_stmts { self.analyze_statement(stmt); } flow_paths.push(self.take_flow_branch(&flow_handler_entry)); - - let otherwise_scope = std::mem::take(&mut self.current_scope); - if let Some(parent) = otherwise_scope.parent { - self.current_scope = *parent; + for (name, symbol) in &self.current_scope.symbols { + joined_scope_symbols + .entry(name.clone()) + .or_insert_with(|| symbol.clone()); } + } else if !when_clauses.iter().any(|when_clause| { + matches!( + &when_clause.error_type, + crate::parser::ast::ErrorType::General + ) + }) { + // Without a catch-all or otherwise block, a non-matching + // error reaches finally directly from the handler entry. + flow_paths.push(flow_handler_entry.clone()); + } + + self.current_scope = clause_entry_scope; + for symbol in joined_scope_symbols.into_values() { + self.define_or_replace_symbol(symbol); } - // After the construct, any of the recorded paths may have run. + // Finally can be reached from success, any selected error + // clause, or an unmatched error. Join those flow endpoints + // before checking it in the shared runtime try scope. self.join_flow_branches(&flow_paths); if let Some(finally_stmts) = finally_block { - let outer_scope = std::mem::take(&mut self.current_scope); - self.current_scope = Scope::with_parent(outer_scope); - for stmt in finally_stmts { self.analyze_statement(stmt); } - - let finally_scope = std::mem::take(&mut self.current_scope); - if let Some(parent) = finally_scope.parent { - self.current_scope = *parent; - } } + + self.pop_scope(); } Statement::ReadFileStatement { variable_name, .. } => { let symbol = Symbol { @@ -1535,6 +1590,141 @@ impl Analyzer { } } + Statement::HttpStreamStatement { + url, + method, + headers, + body, + variable_name, + .. + } => { + self.analyze_expression(url); + if let Some(method) = method { + self.analyze_expression(method); + } + if let Some(headers) = headers { + self.analyze_expression(headers); + } + if let Some(body) = body { + self.analyze_expression(body); + } + + // Binds a streaming-response handle object (status/ok/headers). + let symbol = Symbol { + name: variable_name.clone(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: None, + line: 0, + column: 0, + }; + self.current_scope.define_or_replace(symbol); + } + + Statement::WaitForNextChunkStatement { + source, + variable_name, + .. + } + | Statement::WaitForNextLineStatement { + source, + variable_name, + .. + } => { + self.analyze_expression(source); + + // Binds the next chunk/line, or `nothing` at end of stream, so + // the type is left open. Refreshed on every wait (loop-friendly). + let symbol = Symbol { + name: variable_name.clone(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: None, + line: 0, + column: 0, + }; + self.current_scope.define_or_replace(symbol); + } + + Statement::StartStreamingResponseStatement { + request, + status, + content_type, + headers, + variable_name, + .. + } => { + self.analyze_expression(request); + if let Some(status) = status { + self.analyze_expression(status); + } + if let Some(content_type) = content_type { + self.analyze_expression(content_type); + } + if let Some(headers) = headers { + self.analyze_expression(headers); + } + // Binds a server response-stream handle object. + let symbol = Symbol { + name: variable_name.clone(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: None, + line: 0, + column: 0, + }; + self.current_scope.define_or_replace(symbol); + } + + Statement::StreamWriteStatement { + value, + target, + fallback_content, + line, + column, + .. + } => { + self.analyze_expression(target); + match fallback_content { + // Unambiguous form: check the stream value normally. + None => self.analyze_expression(value), + // Ambiguous merged form (`write line ... to `): + // the live reading — stream write of `` vs classic file + // write of the variable `line ` — depends on the runtime + // target type, and the two readings differ ONLY at the leftmost + // leaf (the merged lead). `analyze_ambiguous_write` walks both + // readings in parallel: it analyzes every sub-expression they + // share (operator right-hand sides, concatenation tails) so a + // genuinely undefined variable in the continuation is still + // caught, and at the lead reports undefined only when NEITHER + // reading resolves. Call-based desugarings (`starts/ends with`, + // `is between`, patterns) bury the lead inside a call where the + // shapes diverge; those still defer to runtime rather than risk + // rejecting a valid classic file write. + Some(fallback) => { + self.analyze_ambiguous_write(value, fallback, *line, *column); + } + } + } + + Statement::FlushStreamStatement { + target, + legacy_binding, + action_fallback, + .. + } => { + // When the legacy full-name expression's root is defined, analyze + // that expression (expression-statement path). Otherwise analyze + // the stream target. + let legacy_root_defined = legacy_binding + .as_deref() + .is_some_and(|name| self.name_is_defined(name)); + if legacy_root_defined { + if let Some(fb) = action_fallback { + self.analyze_expression(fb); + } + } else { + self.analyze_expression(target); + } + } + Statement::CreateDirectoryStatement { path, .. } => { self.analyze_expression(path); } @@ -2578,6 +2768,21 @@ impl Analyzer { layers } + /// Snapshot the symbols owned by the current scope. + /// + /// Type-state joins normally only need [`snapshot_symbol_types`], but + /// independently checked control-flow branches can also introduce new + /// bindings. Restoring this map before each branch prevents a binding from + /// one branch shadowing names while another branch is checked. + pub fn snapshot_current_scope_symbols(&self) -> HashMap { + self.current_scope.symbols.clone() + } + + /// Restore the exact set of symbols owned by the current scope. + pub fn restore_current_scope_symbols(&mut self, symbols: HashMap) { + self.current_scope.symbols = symbols; + } + /// Restore `symbol_type` values previously captured by /// [`snapshot_symbol_types`]. Only updates symbols that still exist; does /// not remove symbols defined after the snapshot. @@ -2700,6 +2905,19 @@ impl Analyzer { } } + /// Pop the current scope while promoting every binding except the listed + /// temporary aliases into its parent. + pub fn pop_scope_promoting_except(&mut self, excluded: &[String]) { + if let Some(mut parent) = self.current_scope.parent.take() { + for (name, symbol) in std::mem::take(&mut self.current_scope.symbols) { + if !excluded.iter().any(|excluded_name| excluded_name == &name) { + parent.define_or_replace(symbol); + } + } + self.current_scope = *parent; + } + } + /// Validates a call against every registered signature of `name`: /// filters candidates by argument count, then (when several share the /// count) by static argument types. A single surviving candidate gets the @@ -3379,6 +3597,281 @@ impl Analyzer { } } + /// Analyze an ambiguous `write line|chunk` value against its classic + /// file-write fallback. Both readings are parsed from the SAME tokens and + /// differ only at the leftmost leaf (the merged lead), so walk them in + /// parallel: analyze every shared sub-expression (an operator's right-hand + /// side, a concatenation's tail) so an undefined variable in the continuation + /// is caught, and at the lead report undefined only when NEITHER reading + /// resolves (so neither valid interpretation is rejected). Diverging, + /// call-based desugarings (`starts/ends with`, `is between`, patterns) bury the + /// lead where the shapes no longer line up; those hit the catch-all arm and are + /// left to runtime rather than risk rejecting a valid classic file write. + fn analyze_ambiguous_write( + &mut self, + value: &Expression, + fallback: &Expression, + line: usize, + column: usize, + ) { + // A subtree that is IDENTICAL under both readings carries no lead + // difference (the two readings are parsed from the same tokens, so a + // genuinely shared sub-expression has the same names AND positions) — it is + // pure continuation, so analyze it normally. This catches an undefined + // variable anywhere in the shared part, including inside a call or index. + if value == fallback { + self.analyze_expression(value); + return; + } + // Otherwise the lead lies somewhere below. Recurse in PARALLEL on both + // children so a lead that a desugaring DUPLICATED into the right operand + // (e.g. `is between`) is still matched against the fallback's copy — not + // mistaken for an undefined continuation variable. + match (value, fallback) { + ( + Expression::BinaryOperation { + left: vl, + operator: vo, + right: vr, + .. + }, + Expression::BinaryOperation { + left: fl, + operator: fo, + right: fr, + .. + }, + ) if vo == fo => { + self.analyze_ambiguous_write(vl, fl, line, column); + self.analyze_ambiguous_write(vr, fr, line, column); + } + ( + Expression::Concatenation { + left: vl, + right: vr, + .. + }, + Expression::Concatenation { + left: fl, + right: fr, + .. + }, + ) => { + self.analyze_ambiguous_write(vl, fl, line, column); + self.analyze_ambiguous_write(vr, fr, line, column); + } + // Call- and pattern-based desugarings (`starts/ends with`, `contains`, + // pattern ops, indexing, method/function/action calls) bury the lead in + // one child while the OTHER children are shared continuation. When both + // readings are the same shape (same operator/name and arity), walk the + // corresponding children in parallel so an undefined name in a shared + // argument is still caught; only genuinely non-aligning shapes fall to + // the catch-all and defer to runtime. + ( + Expression::PatternMatch { + text: vt, + pattern: vp, + .. + }, + Expression::PatternMatch { + text: ft, + pattern: fp, + .. + }, + ) + | ( + Expression::PatternFind { + text: vt, + pattern: vp, + .. + }, + Expression::PatternFind { + text: ft, + pattern: fp, + .. + }, + ) + | ( + Expression::PatternSplit { + text: vt, + pattern: vp, + .. + }, + Expression::PatternSplit { + text: ft, + pattern: fp, + .. + }, + ) + | ( + Expression::StringSplit { + text: vt, + delimiter: vp, + .. + }, + Expression::StringSplit { + text: ft, + delimiter: fp, + .. + }, + ) => { + self.analyze_ambiguous_write(vt, ft, line, column); + self.analyze_ambiguous_write(vp, fp, line, column); + } + ( + Expression::PatternReplace { + text: vt, + pattern: vp, + replacement: vrp, + .. + }, + Expression::PatternReplace { + text: ft, + pattern: fp, + replacement: frp, + .. + }, + ) => { + self.analyze_ambiguous_write(vt, ft, line, column); + self.analyze_ambiguous_write(vp, fp, line, column); + self.analyze_ambiguous_write(vrp, frp, line, column); + } + ( + Expression::IndexAccess { + collection: vc, + index: vi, + .. + }, + Expression::IndexAccess { + collection: fc, + index: fi, + .. + }, + ) => { + self.analyze_ambiguous_write(vc, fc, line, column); + self.analyze_ambiguous_write(vi, fi, line, column); + } + ( + Expression::FunctionCall { + function: vf, + arguments: va, + .. + }, + Expression::FunctionCall { + function: ff, + arguments: fa, + .. + }, + ) if va.len() == fa.len() => { + self.analyze_ambiguous_write(vf, ff, line, column); + for (v, f) in va.iter().zip(fa.iter()) { + self.analyze_ambiguous_write(&v.value, &f.value, line, column); + } + } + ( + Expression::ActionCall { + name: vn, + arguments: va, + .. + }, + Expression::ActionCall { + name: fnn, + arguments: fa, + .. + }, + ) if vn == fnn && va.len() == fa.len() => { + for (v, f) in va.iter().zip(fa.iter()) { + self.analyze_ambiguous_write(&v.value, &f.value, line, column); + } + } + ( + Expression::MethodCall { + object: vo, + method: vm, + arguments: va, + .. + }, + Expression::MethodCall { + object: fo, + method: fm, + arguments: fa, + .. + }, + ) if vm == fm && va.len() == fa.len() => { + self.analyze_ambiguous_write(vo, fo, line, column); + for (v, f) in va.iter().zip(fa.iter()) { + self.analyze_ambiguous_write(&v.value, &f.value, line, column); + } + } + ( + Expression::PropertyAccess { + object: vo, + property: vp, + .. + }, + Expression::PropertyAccess { + object: fo, + property: fp, + .. + }, + ) if vp == fp => { + // `write line missing.field to ...` — walk the object under both + // readings so a one-sided undefined lead is not skipped solely + // because PropertyAccess was absent from the parallel walker. + self.analyze_ambiguous_write(vo, fo, line, column); + } + // Reached a differing leaf — the lead. Report only when NEITHER + // reading resolves, so neither valid interpretation is rejected. + // Branch-specific one-sided undefined leads are enforced by the + // typechecker against the concrete target branch (issue #642). + (Expression::Variable(sn, ..), Expression::Variable(fal, ..)) + if !self.name_is_defined(sn) && !self.name_is_defined(fal) => + { + self.report_undefined_name( + format!("Variable '{fal}' is not defined"), + line, + column, + ); + } + // PropertyAccess leaf whose object is a Variable: treat the object + // name as the lead (e.g. `missing.field` vs `line missing.field`). + ( + Expression::PropertyAccess { object: vo, .. }, + Expression::PropertyAccess { object: fo, .. }, + ) => { + self.analyze_ambiguous_write(vo, fo, line, column); + } + // A diverging, non-decomposable shape (a call-based desugaring where the + // lead is buried): defer to runtime rather than risk a false positive. + _ => {} + } + } + + /// Whether a bare name resolves to something known (an action parameter, the + /// `count` loop variable, a builtin, an in-scope binding, or a container + /// property) — i.e. it would NOT be reported as an undefined variable. Used + /// to decide the ambiguous `write line|chunk` case without emitting. + fn name_is_defined(&self, name: &str) -> bool { + self.name_is_defined_for_write(name) + } + + /// Public for the typechecker so write-branch definedness matches analysis + /// (container properties, inherited bindings, etc.). + pub fn name_is_defined_for_write(&self, name: &str) -> bool { + if self.action_parameters.contains(name) + || name == "count" + || name == "loopcounter" + || Self::is_builtin_function(name) + || self.current_scope.resolve(name).is_some() + { + return true; + } + if let Some(container_name) = &self.current_container { + return self.is_container_property(container_name, name); + } + false + } + fn analyze_expression(&mut self, expression: &Expression) { // Recursive front-end checkpoint for expressions. `analyze_statement` // polls per statement, but one statement can hold an arbitrarily large @@ -3938,6 +4431,58 @@ mod tests { assert!(errors[0].message.contains("not defined")); } + #[test] + fn test_concurrent_main_loop_requires_wait_for_request_first() { + use crate::lexer::lex_wfl_with_positions; + use crate::parser::Parser; + + fn analyze_src(src: &str) -> Result<(), Vec> { + let program = Parser::new(&lex_wfl_with_positions(src)) + .parse() + .expect("parse"); + Analyzer::new().analyze(&program) + } + + // A concurrent loop whose body does NOT begin with `wait for request` + // would run its opening statements once per handler slot before any + // request arrives. Static analysis must reject it with an actionable error + // (and that error must survive the handler-body error demotion). + let bad = "listen on port 8080 as srv\n\ + main loop concurrently:\n \ + store tick as 1\n \ + wait for request comes in on srv as req\n \ + respond to req with \"ok\"\nend loop"; + let errs = + analyze_src(bad).expect_err("setup-before-wait concurrent loop must be rejected"); + assert!( + errs.iter() + .any(|e| e.message.contains("must begin with `wait for request")), + "expected the concurrent-loop ordering error, got: {errs:?}" + ); + + // The identical body under a plain serial `main loop` is fine — a serial + // loop runs one iteration at a time, so there is no speculative fan-out. + let serial = "listen on port 8080 as srv\n\ + main loop:\n \ + store tick as 1\n \ + wait for request comes in on srv as req\n \ + respond to req with \"ok\"\nend loop"; + assert!( + analyze_src(serial).is_ok(), + "serial main loop must not require wait-for-request first" + ); + + // A concurrent loop that DOES begin with `wait for request` is accepted. + let good = "listen on port 8080 as srv\n\ + main loop concurrently:\n \ + wait for request comes in on srv as req\n \ + respond to req with \"ok\"\nend loop"; + assert!( + analyze_src(good).is_ok(), + "concurrent loop starting with wait-for-request must analyze cleanly" + ); + } + // Issue #592: a bare unresolved name in a program that uses `include from` // may be a zero-argument action exposed by the included file at runtime, so // the `Expression::Variable` arm must relax to a non-fatal warning (like the diff --git a/src/analyzer/static_analyzer.rs b/src/analyzer/static_analyzer.rs index 119de7c2..747faf23 100644 --- a/src/analyzer/static_analyzer.rs +++ b/src/analyzer/static_analyzer.rs @@ -1339,6 +1339,71 @@ impl Analyzer { self.mark_used_in_expression(content, usages); self.mark_used_in_expression(file, usages); } + Statement::StreamWriteStatement { + value, + target, + fallback_content, + .. + } => { + // Count both interpretations of the ambiguous merged form as + // usages (stream value AND the classic file-write fallback), so a + // variable named `line ` written to a file is not falsely + // reported unused. + self.mark_used_in_expression(value, usages); + self.mark_used_in_expression(target, usages); + if let Some(fallback) = fallback_content { + self.mark_used_in_expression(fallback, usages); + } + } + Statement::FlushStreamStatement { + target, + legacy_binding, + action_fallback, + .. + } => { + // The runtime chooses between the stream target and the legacy + // expression interpretation. Mark both conservatively, matching + // ambiguous stream writes, and mark the preserved merged binding + // separately because split/find/replace rewrites may legitimately + // discard that seed from the fallback AST. + self.mark_used_in_expression(target, usages); + if let Some(name) = legacy_binding + && let Some(usage) = usages.get_mut(name) + { + usage.used = true; + } + if let Some(fallback) = action_fallback { + self.mark_used_in_expression(fallback, usages); + } + } + Statement::HttpStreamStatement { + url, + method, + headers, + body, + .. + } => { + self.mark_used_in_expression(url, usages); + for expr in [method, headers, body].into_iter().flatten() { + self.mark_used_in_expression(expr, usages); + } + } + Statement::WaitForNextChunkStatement { source, .. } + | Statement::WaitForNextLineStatement { source, .. } => { + self.mark_used_in_expression(source, usages); + } + Statement::StartStreamingResponseStatement { + request, + status, + content_type, + headers, + .. + } => { + self.mark_used_in_expression(request, usages); + for expr in [status, content_type, headers].into_iter().flatten() { + self.mark_used_in_expression(expr, usages); + } + } Statement::WriteContentStatement { content, target, .. } @@ -2156,6 +2221,82 @@ mod tests { assert_eq!(diagnostics[0].code, "ANALYZE-UNUSED"); } + #[test] + fn test_streaming_statement_variables_are_not_reported_unused() { + // Variables referenced only inside the streaming/incremental-read + // statements must count as used — otherwise a program that opens a + // stream from a URL/body variable gets a false unused warning. Exercise + // every new arm and every metadata operand: + // - HttpStreamStatement: url, method, body + // - WaitForNextChunkStatement / WaitForNextLineStatement: source + // - StartStreamingResponseStatement: status, content type + // and a genuinely-unused variable to prove the pass still flags real + // dead code (negative assertion). + let input = "store my_url as \"http://example.com/s\"\n\ +store my_method as \"POST\"\n\ +store my_body as \"payload\"\n\ +open url at my_url with method my_method and body my_body and stream response as upstream\n\ +wait for next chunk from upstream as ch\n\ +wait for next line from upstream as ln\n\ +store st as 200\n\ +store ctype as \"application/x-ndjson\"\n\ +start streaming response to req with status st and content type ctype as out\n\ +write line \"x\" to out\n\ +store dead as \"never read\"\n\ +display ch\n\ +display ln"; + let tokens = crate::lexer::lex_wfl_with_positions(input); + let program = crate::parser::Parser::new(&tokens).parse().unwrap(); + + let analyzer = Analyzer::new(); + let diagnostics = analyzer.check_unused_variables(&program, 0); + + // The only unused variable is `dead`; every streaming operand counts as + // used (a missing arm would surface `my_url`/`my_method`/`my_body`/ + // `upstream`/`st`/`ctype` here too). + assert_eq!( + diagnostics.len(), + 1, + "expected only `dead` unused, got: {diagnostics:?}" + ); + assert!( + diagnostics[0].message.contains("dead"), + "expected the unused diagnostic to name `dead`, got: {:?}", + diagnostics[0].message + ); + } + + #[test] + fn test_legacy_flush_binding_and_fallback_operands_are_not_reported_unused() { + // `replace ... in ...` rewrites the expression AST and discards its + // seeded `flush cache` leaf. The explicit legacy-binding metadata must + // therefore count that declaration as used independently of the + // rewritten target/fallback expression. + let input = "create pattern letter_a:\n\ + \x20\x20\x20\x20\"a\"\n\ + end pattern\n\ + store flush cache as 1\n\ + store replacement_value as \"z\"\n\ + store text_value as \"abc\"\n\ + flush cache replace letter_a with replacement_value in text_value\n\ + store dead as \"never read\""; + let tokens = crate::lexer::lex_wfl_with_positions(input); + let program = crate::parser::Parser::new(&tokens).parse().unwrap(); + + let analyzer = Analyzer::new(); + let diagnostics = analyzer.check_unused_variables(&program, 0); + + assert_eq!( + diagnostics.len(), + 1, + "expected only `dead` unused, got: {diagnostics:?}" + ); + assert!( + diagnostics[0].message.contains("dead"), + "the legacy binding and fallback operands must count as used; got: {diagnostics:?}" + ); + } + #[test] fn test_respond_headers_expression_marks_variable_used() { // Regression: a variable referenced only in the diff --git a/src/config.rs b/src/config.rs index 71d6609d..6e105a99 100644 --- a/src/config.rs +++ b/src/config.rs @@ -62,6 +62,12 @@ pub struct WflConfig { /// HTTP request before shedding it with 504 and releasing its in-flight /// slot. `0` disables the timeout. Feeds `ExecutionBudget`. Default 300. pub web_server_response_timeout_seconds: u64, + /// Absolute total lifetime (seconds) of a single outbound streaming response + /// (`open url ... and stream response`), measured from when the stream is + /// opened. Distinct from `timeout_seconds` (the per-read idle timeout): an + /// upstream that trickles a byte before every idle timeout can never run past + /// this hard cap. `0` disables the total cap. Default 300. + pub outbound_stream_max_seconds: u64, // --- Shared ExecutionBudget limits (see src/exec/budget.rs) --- /// Hard ceiling on charged interpreter operations. `None`/`0` = unlimited /// (the default, matching historic behavior). Feeds `ExecutionBudget`. @@ -192,6 +198,8 @@ impl Default for WflConfig { // Free an accepted request's in-flight slot if its handler does not // answer within 5 minutes (far longer than any serial handler needs). web_server_response_timeout_seconds: 300, + // Absolute cap on a single outbound streaming response's lifetime. + outbound_stream_max_seconds: 300, // Shared ExecutionBudget limits (see src/exec/budget.rs). Defaults // are chosen so existing programs never trip them while runaway // behavior gets a clean error instead of a crash or OOM. @@ -819,6 +827,25 @@ fn parse_config_text(config: &mut WflConfig, text: &str, file: &Path) { file.display() ), }, + "outbound_stream_max_seconds" => match value.parse::() { + Ok(secs) => { + // 0 disables the total cap (the documented sentinel). + // Extreme values are accepted into config but clamped when + // converted to an Instant deadline (see interpreter + // `outbound_stream_deadline`) so Instant arithmetic cannot + // panic. + config.outbound_stream_max_seconds = secs; + log::debug!( + "Loaded outbound_stream_max_seconds: {secs} from {}", + file.display() + ); + } + Err(_) => log::warn!( + "Invalid outbound_stream_max_seconds '{}' in {}: expected a non-negative integer", + value, + file.display() + ), + }, "max_operations" => match value.parse::() { Ok(n) => { // 0 means "no operation ceiling" (the default). diff --git a/src/interpreter/error.rs b/src/interpreter/error.rs index 0803a32c..5956ebbd 100644 --- a/src/interpreter/error.rs +++ b/src/interpreter/error.rs @@ -8,6 +8,12 @@ pub enum ErrorKind { /// A shared `ExecutionBudget` ceiling other than the deadline was reached /// (operation count, recursion/import/execute-file depth, byte caps, etc.). ResourceLimit, + /// A cooperative cancellation of an in-flight operation triggered by an + /// expected external event rather than a fault — currently a downstream + /// (browser) disconnect cancelling a proxy handler's blocked upstream read. + /// Catchable like any other error, but the concurrent `main loop` treats it + /// as a normal handler outcome, not a structural failure. + Cancelled, FileNotFound, PermissionDenied, ProcessNotFound, @@ -51,6 +57,7 @@ impl fmt::Display for RuntimeError { ErrorKind::EnvDropped => "[Environment dropped] ", ErrorKind::Timeout => "[Timeout] ", ErrorKind::ResourceLimit => "[Resource limit] ", + ErrorKind::Cancelled => "[Cancelled] ", ErrorKind::FileNotFound => "[File not found] ", ErrorKind::PermissionDenied => "[Permission denied] ", ErrorKind::ProcessNotFound => "[Process not found] ", diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index 7d38c246..cdc24c68 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -1,4 +1,4 @@ -#![allow(clippy::await_holding_refcell_ref)] +#![deny(clippy::await_holding_refcell_ref)] mod assertion_helpers; pub mod bounded_buffer; pub mod command_sanitizer; @@ -53,7 +53,7 @@ use crate::parser::ast::{ use crate::pattern::CompiledPattern; use crate::stdlib; use std::cell::{Cell, RefCell}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet, VecDeque}; use std::io::{self, Write}; use std::net::IpAddr; use std::path::PathBuf; @@ -63,7 +63,11 @@ use std::time::{Duration, Instant}; use tokio::sync::{mpsc, oneshot}; // Type alias for complex pending response type -type PendingResponseSender = Arc>>>; +type PendingResponseSender = Arc>>>; + +/// An open server response stream: the bounded body-chunk sender plus the +/// running total of body bytes written (enforced against `max_response_bytes`). +type ServerResponseStream = (mpsc::Sender>, usize); /// A dequeued HTTP request parked in `pending_responses` awaiting a `respond`. /// @@ -89,6 +93,39 @@ use warp::Filter; /// `count & (STRIDE - 1)` is exact; large enough that the yield is negligible. const COOP_YIELD_STRIDE: u64 = 1024; +/// Bounded capacity of a server response stream's body-chunk channel. A slow +/// client fills this and then backpressures the handler's `write` (it awaits a +/// free slot) rather than letting queued chunks grow without bound. +const RESPONSE_STREAM_BUFFER: usize = 64; + +/// Maximum number of `main loop concurrently:` iterations in flight at once. The +/// transport already bounds the request *queue*; this bounds concurrent *handler* +/// execution so a burst cannot spawn unbounded cooperative tasks. Requests +/// beyond this run as handlers free up (and the transport sheds 503 if its queue +/// also fills). +const CONCURRENT_HANDLER_LIMIT: usize = 256; +const MAX_CONSECUTIVE_HANDLER_FAILURES: u32 = 256; +const REQUEST_WAIT_TIMEOUT_PREFIX: &str = "Timeout waiting for request"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ConcurrentHandlerDisposition { + RequestLocal, + Structural, +} + +fn classify_concurrent_handler_error( + error: &RuntimeError, + accepted_request: bool, +) -> ConcurrentHandlerDisposition { + let request_wait_timeout = + error.kind == ErrorKind::Timeout && error.message.starts_with(REQUEST_WAIT_TIMEOUT_PREFIX); + if accepted_request || error.kind == ErrorKind::Cancelled || request_wait_timeout { + ConcurrentHandlerDisposition::RequestLocal + } else { + ConcurrentHandlerDisposition::Structural + } +} + // Web server data structures #[derive(Debug)] pub struct WflHttpRequest { @@ -105,7 +142,7 @@ pub struct WflHttpRequest { /// binary value, so binary uploads survive intact. pub body: Vec, pub headers: HashMap, - pub response_sender: Arc>>>, + pub response_sender: Arc>>>, } #[derive(Debug, Clone)] @@ -119,19 +156,53 @@ pub struct WflHttpResponse { pub headers: HashMap, } -/// Ensures an HTTP `respond` always resolves its request. The response sender is -/// taken out of `pending_responses` (and out of its mutex) up front and held -/// here; if a fallible step in `respond` returns early before a response is -/// built, `Drop` answers 500 so the client is resolved deterministically instead -/// of hanging until the request timeout. A successful `respond` calls -/// [`ResponseCompletion::take_sender`] to disarm the fallback and deliver the -/// real response. +/// What a request handler delivers back to its warp transport task over the +/// per-request `oneshot`. +/// +/// `respond` sends a fully-buffered `Buffered` reply; `start streaming response` +/// sends a `Streaming` reply whose head (status/headers) is written immediately +/// and whose body is fed chunk-by-chunk over a bounded channel by `write +/// line|chunk`. A bounded channel gives backpressure: a slow client slows the +/// handler's writes. Dropping the sender (handler end, `close`, or a caught +/// error) closes the body stream and finalizes the response. +#[derive(Debug)] +pub enum HandlerReply { + Buffered(WflHttpResponse), + Streaming { + status: u16, + content_type: String, + headers: HashMap, + body: mpsc::Receiver>, + }, +} + +/// Ensures an HTTP `respond` always resolves its request after the commit point. +/// Fallible response expressions are evaluated while the pending sender remains +/// available as a disconnect signal. At commit, the sender is atomically removed +/// and held here; if delivery then exits early, `Drop` answers 500 rather than +/// leaving the client hanging. Successful delivery calls +/// [`ResponseCompletion::take_sender`] to disarm that fallback. struct ResponseCompletion { - sender: Option>, + sender: Option>, +} + +/// Interpreter state that must survive cancellation of fallible response +/// expressions. The evaluation future is explicitly dropped before this state +/// is restored, so partially-entered actions and loops cannot leak into a +/// handler that catches `Cancelled` and continues. +struct ResponsePrecommitSnapshot { + call_stack: Vec, + call_depth: usize, + current_count: Option, + in_count_loop: bool, + http_owner: StreamOwner, + http_streams: HashSet, + response_streams: HashSet, + pending_requests: HashSet, } impl ResponseCompletion { - fn take_sender(&mut self) -> Option> { + fn take_sender(&mut self) -> Option> { self.sender.take() } } @@ -139,12 +210,12 @@ impl ResponseCompletion { impl Drop for ResponseCompletion { fn drop(&mut self) { if let Some(sender) = self.sender.take() { - let _ = sender.send(WflHttpResponse { + let _ = sender.send(HandlerReply::Buffered(WflHttpResponse { content: b"Internal Server Error\n".to_vec(), status: 500, content_type: "text/plain; charset=utf-8".to_string(), headers: HashMap::new(), - }); + })); } } } @@ -751,6 +822,191 @@ impl Drop for CallDepthGuard<'_> { } } +/// A snapshot of the interpreter's per-execution "run state" — the mutable +/// bookkeeping that belongs to a single in-flight execution rather than to the +/// interpreter as a whole: the count-loop variable, recursion depth, the +/// diagnostic call stack, and the current block's overload-dup set. +/// +/// Under serial execution this state lives directly on `Interpreter` and is +/// never contended. Under `main loop concurrently:` several handler futures are +/// interleaved cooperatively on one thread, so at every `await` point one +/// handler's run state must not be visible to (or clobbered by) another. Each +/// handler owns a `RunState` that is swapped into the interpreter only while +/// that handler is actively being polled (see [`IsolatedHandler`]). +#[derive(Default)] +struct RunState { + current_count: Option, + in_count_loop: bool, + call_depth: usize, + call_stack: Vec, + block_overload_dups: Option>>, + /// Server response streams opened by this handler and not yet explicitly + /// closed. Closed automatically when the handler ends on any path (see + /// `IsolatedHandler`'s `Drop` and the serial main loop's per-iteration + /// drain), so a handler that forgets `close out` never hangs the client. + open_response_streams: Vec, + /// Request ids this handler dequeued (`wait for request comes in`) and has + /// not yet answered. If the handler ends on any path without responding, each + /// is answered 500 immediately instead of leaving the client to wait out the + /// request timeout (see `fail_unanswered_requests`). + open_pending_requests: Vec, + /// Outbound streaming-response handle ids (`... stream response as `) + /// this handler opened and has not yet closed/exhausted. Handler-OWNED: when + /// the handler ends on any path (normal, error, panic, cancellation, loop + /// exit) these are dropped from `IoClient.stream_handles`, which cancels the + /// in-flight upstream request — so an abandoned proxy read never leaks an + /// upstream connection or handle past the handler's lifetime. + open_http_streams: StreamOwner, + /// Sticky: this handler successfully dequeued at least one request via + /// `wait for request`. Used by the concurrent main loop to distinguish + /// structural pre-request failures (feed the consecutive-failure breaker) + /// from request-local outcomes (must never tear the server down because one + /// accepted request failed). Survives respond clearing `open_pending_requests`. + accepted_request: bool, +} + +impl RunState { + /// A fresh run state for a handler starting from `base_call_depth` (0 for a + /// top-level run; the parent's live depth for an `execute file` child). + fn fresh(base_call_depth: usize) -> Self { + RunState { + call_depth: base_call_depth, + ..RunState::default() + } + } +} + +/// Wraps a handler future so its [`RunState`] is swapped into the interpreter +/// for the duration of each `poll` and swapped back out again the instant the +/// poll returns (ready **or** pending). This makes the interpreter's run-state +/// fields effectively poll-local: while handler A is suspended at an `await`, +/// its count-loop variable, recursion depth, and call stack are parked in A's +/// own `RunState`, so handler B — polled next — neither sees nor corrupts them. +/// +/// `inner` is a boxed handler future (already wrapped in `catch_unwind`); a +/// panic therefore surfaces as `Poll::Ready` and the swap-back still runs, +/// leaving the interpreter's scratch fields restored for the next sibling. +/// +/// The wrapper's output is `(inner_output, accepted_request)` so the concurrent +/// loop can classify request-local vs structural failures after the handler's +/// run state has been swapped out (and its pending list drained by `Drop`). +struct IsolatedHandler<'a, T> { + interp: &'a Interpreter, + state: RunState, + inner: std::pin::Pin + 'a>>, +} + +impl<'a, T> std::future::Future for IsolatedHandler<'a, T> { + type Output = (T, bool); + + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<(T, bool)> { + // Every field is `Unpin` (`&`, `RunState`, and `Pin>`), so the + // wrapper itself is `Unpin` and `get_mut` is sound. + let this = self.get_mut(); + this.interp.swap_run_state(&mut this.state); + let result = this.inner.as_mut().poll(cx); + this.interp.swap_run_state(&mut this.state); + match result { + std::task::Poll::Ready(value) => { + std::task::Poll::Ready((value, this.state.accepted_request)) + } + std::task::Poll::Pending => std::task::Poll::Pending, + } + } +} + +impl<'a, T> Drop for IsolatedHandler<'a, T> { + fn drop(&mut self) { + // The handler is finished (normal return, error, panic contained by + // `catch_unwind`, or cancellation as the loop tears down). After the + // final poll's swap-out, `state` holds any streams it opened but never + // closed and any requests it dequeued but never answered. Close the + // streams (finalizing the client's body) and 500 the unanswered requests, + // so every exit path resolves the client instead of leaving it hanging. + self.interp + .close_response_streams(&self.state.open_response_streams); + self.interp + .fail_unanswered_requests(&self.state.open_pending_requests); + // Drop any outbound streams this handler still owns, cancelling their + // in-flight upstream requests so an abandoned proxy read never leaks. + self.interp + .close_http_streams(&self.state.open_http_streams); + } +} + +/// RAII guard that finalizes the interpreter's still-open server-request state +/// when the running `interpret()` future ends — crucially including when that +/// future is *dropped* (an embedder cancels the run) before the normal +/// handler-exit / program-cleanup sites execute. It shares the interpreter's +/// tracking lists and maps via `Rc`, so its `Drop` runs even as the interpreter +/// itself stays alive (e.g. a reused REPL). It covers, for the top-level/serial +/// context whose state lives directly on the interpreter (concurrent handlers +/// finalize their own via `IsolatedHandler`'s `Drop`): +/// +/// - **outbound streams** — dropped from `IoClient.stream_handles`, cancelling +/// the in-flight upstream request; +/// - **server response streams** — dropped from `server_response_streams`, ending +/// the client's body so a `start streaming response` that never `close`d does +/// not leave the connection hanging; +/// - **pending requests** — answered 500 so a dequeued-but-unanswered request is +/// resolved instead of waiting out the request timeout. +/// +/// On a normal run the cleanup sites have already drained the lists, so this is a +/// no-op. All work is synchronous and best-effort (`try_lock` never blocks in a +/// `Drop`). +struct OutboundStreamCleanup { + io_client: Rc, + open_http_streams: StreamOwner, + open_response_streams: Rc>>, + server_response_streams: Rc>>, + open_pending_requests: Rc>>, + pending_responses: Rc>>, +} + +impl Drop for OutboundStreamCleanup { + fn drop(&mut self) { + // Outbound upstream streams: removing a handle drops its reqwest stream, + // cancelling the in-flight upstream request. + self.io_client.close_stream_owner(&self.open_http_streams); + + // Server response streams: dropping the sender ends the client's body. + let stream_ids = std::mem::take(&mut *self.open_response_streams.borrow_mut()); + if !stream_ids.is_empty() { + let mut map = self.server_response_streams.borrow_mut(); + for id in &stream_ids { + map.remove(id); + } + } + + // Pending requests dequeued but never answered: resolve each with 500 so + // the client is not left waiting out its request timeout. Mirrors + // `fail_unanswered_requests`, but operates on the shared `Rc` maps so it + // runs even when the interpreter's own methods are unreachable (future + // dropped). The sender mutex is only held during `respond`, which a + // dropped run is no longer inside, so `try_lock` succeeds. + let request_ids = std::mem::take(&mut *self.open_pending_requests.borrow_mut()); + if !request_ids.is_empty() { + let mut pending = self.pending_responses.borrow_mut(); + for id in &request_ids { + if let Some(entry) = pending.remove(id) + && let Ok(mut guard) = entry.sender.try_lock() + && let Some(sender) = guard.take() + { + let _ = sender.send(HandlerReply::Buffered(WflHttpResponse { + content: b"Internal Server Error\n".to_vec(), + status: 500, + content_type: "text/plain; charset=utf-8".to_string(), + headers: HashMap::new(), + })); + } + } + } + } +} + /// RAII guard that ensures module loading context is restored on scope exit. /// Automatically pops loading_stack and restores current_source_file when dropped. struct ModuleLoadGuard<'a> { @@ -893,6 +1149,22 @@ fn stmt_type(stmt: &Statement) -> String { Statement::HttpRequestStatement { variable_name, .. } => { format!("HttpRequestStatement '{variable_name}'") } + Statement::HttpStreamStatement { variable_name, .. } => { + format!("HttpStreamStatement '{variable_name}'") + } + Statement::WaitForNextChunkStatement { variable_name, .. } => { + format!("WaitForNextChunkStatement '{variable_name}'") + } + Statement::WaitForNextLineStatement { variable_name, .. } => { + format!("WaitForNextLineStatement '{variable_name}'") + } + Statement::StartStreamingResponseStatement { variable_name, .. } => { + format!("StartStreamingResponseStatement '{variable_name}'") + } + Statement::StreamWriteStatement { is_line, .. } => { + format!("StreamWriteStatement (line={is_line})") + } + Statement::FlushStreamStatement { .. } => "FlushStreamStatement".to_string(), Statement::PushStatement { .. } => "PushStatement to list".to_string(), Statement::CreateListStatement { name, .. } => format!("CreateListStatement '{name}'"), Statement::MapCreation { name, .. } => format!("MapCreation '{name}'"), @@ -1099,7 +1371,47 @@ pub struct Interpreter { web_servers: RefCell>, // Web servers by name web_socket_servers: RefCell>, // WebSocket servers keyed by address ws_connections: WsConnectionRegistry, // Outbound senders for all live WebSocket connections - pending_responses: RefCell>, // Pending responses (channel + admission slot) by request ID + pending_responses: Rc>>, // Pending responses (channel + admission slot) by request ID + /// Open server response streams (`start streaming response`), keyed by + /// handle id ("respstream1", ...). Each holds the bounded body-chunk sender + /// plus the running total of body bytes written, enforced against + /// `max_response_bytes` so a stream cannot bypass the buffered response + /// ceiling. `write line|chunk`/`flush` push to it, `close` drops it. + server_response_streams: Rc>>, + next_response_stream_id: std::cell::Cell, + /// Handle ids of server response streams opened by the currently executing + /// handler and not yet explicitly closed. Part of the per-handler `RunState` + /// (swapped in/out per poll under `main loop concurrently:`) so each handler + /// tracks only its own streams; drained and closed when the handler ends on + /// any path so the client's body is always finalized (see + /// `close_response_streams`). + /// `Rc>` (not a bare `RefCell`) so the `interpret()`-scoped + /// cleanup guard can share it and finalize any still-open server response + /// bodies if the future is dropped before its normal exit sites run (see + /// `OutboundStreamCleanup`). + open_response_streams: Rc>>, + /// Request ids the currently executing handler dequeued but has not yet + /// answered. Part of the per-handler `RunState` (swapped per poll) so each + /// handler tracks only its own requests; any still unanswered when the handler + /// ends are answered 500 immediately (see `fail_unanswered_requests`). + /// `Rc>` (not a bare `RefCell`) so the `interpret()`-scoped + /// cleanup guard can share it and 500 any still-unanswered requests if the + /// future is dropped before its normal exit sites run (see + /// `OutboundStreamCleanup`). + open_pending_requests: Rc>>, + /// Outbound stream handle ids (`... stream response as `) the currently + /// executing handler opened and has not yet closed/exhausted. Part of the + /// per-handler `RunState` (swapped per poll); any still open when the handler + /// ends are dropped, cancelling their upstream requests (see + /// `close_http_streams`). + /// `Rc>` (not a bare `RefCell`) so an RAII cleanup guard tied to + /// the `interpret()` future can share the list and close these handles if the + /// future is dropped/cancelled before its normal exit sites run (see + /// `OutboundStreamCleanup`). + open_http_streams: Rc>, + /// Sticky per-handler flag: at least one request was dequeued and parked. + /// Part of `RunState` (swapped per poll); see `RunState::accepted_request`. + accepted_request: Cell, #[allow(dead_code)] // Used for future security features config: Arc, // Configuration for security and other settings current_source_file: RefCell>, // Currently executing source file (for path resolution) @@ -1398,9 +1710,233 @@ pub struct IoClient { next_process_id: Mutex, db_handles: Mutex>, next_db_id: Mutex, + /// Live outbound streaming response bodies, keyed by handle id + /// ("httpstream1", ...). See [`StreamSlot`] / [`HttpStreamHandle`]. + /// + /// Uses a **std** mutex so Drop/cleanup paths can lock reliably (tokio's + /// async mutex only offers `try_lock` from sync Drop, which previously + /// abandoned handles when the map was briefly held). Critical sections are + /// short (no `.await` while held). + stream_handles: Arc>, + next_stream_id: Mutex, + /// Test-only live-task accounting. The production build carries no + /// instrumentation; unit tests retain a runtime and assert that closing a + /// stream actually drops its sleeping reaper instead of relying on runtime + /// shutdown to hide leaked timers. + #[cfg(test)] + active_stream_reapers: Arc, config: Arc, } +#[cfg(test)] +struct ActiveStreamReaperGuard { + active: Arc, +} + +#[cfg(test)] +impl ActiveStreamReaperGuard { + fn new(active: Arc) -> Self { + active.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Self { active } + } +} + +#[cfg(test)] +impl Drop for ActiveStreamReaperGuard { + fn drop(&mut self) { + self.active + .fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } +} + +/// Hard ceiling on `outbound_stream_max_seconds` when converting to an +/// `Instant` deadline. Extreme `u64` values must not panic +/// `Instant::now() + Duration::from_secs(secs)` (which can overflow). +const MAX_OUTBOUND_STREAM_DEADLINE_SECS: u64 = 365 * 24 * 60 * 60; // 1 year + +/// Compute the absolute stream deadline from a configured second cap. +/// `0` is the documented sentinel for "no absolute total cap". Values above +/// [`MAX_OUTBOUND_STREAM_DEADLINE_SECS`] are clamped before `Instant` +/// arithmetic so even an extreme configuration remains finite and cannot +/// panic. +fn outbound_stream_deadline(secs: u64) -> Option { + let effective = outbound_stream_effective_seconds(secs)?; + Instant::now().checked_add(Duration::from_secs(effective)) +} + +fn outbound_stream_effective_seconds(secs: u64) -> Option { + (secs != 0).then(|| secs.min(MAX_OUTBOUND_STREAM_DEADLINE_SECS)) +} + +/// Shared per-stream cancellation: close, expire, and EOF all trip this so an +/// active body read can select against it and drop the upstream promptly +/// (rather than only noticing when `put_stream` finds a missing slot). +struct StreamCancel { + terminal: tokio::sync::watch::Sender>, +} + +impl StreamCancel { + fn new() -> Arc { + let (terminal, _receiver) = tokio::sync::watch::channel(None); + Arc::new(Self { terminal }) + } + + fn terminal(&self) -> Option { + *self.terminal.borrow() + } + + fn terminate(&self, reason: StreamTerminal) -> StreamTerminal { + self.terminal.send_if_modified(|terminal| { + if terminal.is_none() { + *terminal = Some(reason); + true + } else { + false + } + }); + self.terminal() + .expect("stream terminal reason must be set after terminate") + } + + fn subscribe(&self) -> tokio::sync::watch::Receiver> { + self.terminal.subscribe() + } +} + +/// Why a stream slot was terminated. Active readers observe the shared +/// first-wins signal; clean EOF and unread expiry are retained briefly in the +/// bounded recent queue so the next read can consume the terminal outcome. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StreamTerminal { + CleanEof, + Timeout, + Closed, +} + +type StreamOwner = Arc>>; + +/// Keep a short, bounded window of terminal outcomes after removing the live +/// body. This preserves one clean-EOF read or a typed timeout without retaining +/// the request body, cancel channel, owner, or sleeping task. +const MAX_RECENT_STREAM_TERMINALS: usize = 64; +const RECENT_STREAM_TERMINAL_TTL: Duration = Duration::from_secs(60); + +#[derive(Default)] +struct StreamRegistry { + live: HashMap, + recent: VecDeque, +} + +struct RecentStreamTerminal { + id: String, + reason: StreamTerminal, + expires_at: Instant, +} + +impl StreamRegistry { + fn prune_recent(&mut self, now: Instant) { + while self + .recent + .front() + .is_some_and(|entry| entry.expires_at <= now) + { + self.recent.pop_front(); + } + } + + fn remember_recent(&mut self, id: String, reason: StreamTerminal, now: Instant) { + self.prune_recent(now); + self.recent.retain(|entry| entry.id != id); + while self.recent.len() >= MAX_RECENT_STREAM_TERMINALS { + self.recent.pop_front(); + } + self.recent.push_back(RecentStreamTerminal { + id, + reason, + expires_at: now + RECENT_STREAM_TERMINAL_TTL, + }); + } + + fn take_recent(&mut self, id: &str, now: Instant) -> Option { + self.prune_recent(now); + let index = self.recent.iter().position(|entry| entry.id == id)?; + self.recent.remove(index).map(|entry| entry.reason) + } + + fn forget_recent(&mut self, id: &str) -> bool { + let before = self.recent.len(); + self.recent.retain(|entry| entry.id != id); + self.recent.len() != before + } +} + +/// Per-handle shared lifecycle for an outbound stream. +/// +/// Reads take the inner [`HttpStreamHandle`] out for the duration of the await +/// (so the global map lock is not held across the network). Explicit +/// finish/close removes the slot after signalling [`StreamCancel`]; expiry +/// drops the parked body, removes the live slot, and records a bounded recent +/// terminal so mid-read work aborts without losing the reason. Clean EOF uses +/// the same queue for its one follow-up read. +struct StreamSlot { + /// The live body handle. `None` while a body read owns it. + handle: Option, + /// Absolute deadline for the whole stream (`None` = no absolute total cap). + deadline: Option, + /// Shared cancel signal for active-read races. + cancel: Arc, + /// Abort handle for the reaper timer. Cancelled on EOF, error, or explicit + /// close so rapid open/close cycles do not accumulate sleeping tasks. + reaper_abort: Option, + /// Handler ownership is stored with the live slot so the reaper can remove + /// the id immediately. Recent terminal records never retain an owner. + owner: Option, +} + +fn remove_stream_owner(slot: &mut StreamSlot, handle_id: &str) { + if let Some(owner) = slot.owner.take() { + owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(handle_id); + } +} + +/// A live, parked outbound streaming response body. +/// +/// The status and headers were already handed to the WFL program by +/// `open url ... and stream response as `; this holds the still-open body +/// stream so `wait for next chunk|line` can pull it incrementally without ever +/// buffering the whole body. Dropping the handle — on clean EOF, a mid-stream +/// error, an explicit `close`, or interpreter teardown — drops the underlying +/// reqwest body future and thereby cancels the in-flight upstream request. +struct HttpStreamHandle { + /// The response body as a stream of raw byte chunks. `Vec` (not + /// `bytes::Bytes`) so the boxed trait object stays nameable here. + stream: std::pin::Pin>> + Send>>, + /// Bytes read from the network but not yet handed to the program: the + /// remainder after a line split, plus bytes accumulated while scanning for + /// the next newline. Bounded by the run's `max_response_bytes` because + /// every byte placed here was counted against that ceiling as it was read. + buffer: Vec, + /// True once the underlying stream has yielded a clean end of stream. + done: bool, + /// Total body bytes pulled from the network so far, enforced against + /// `max_response_bytes`. + bytes_read: usize, + /// Absolute deadline for the whole stream, set from + /// `outbound_stream_max_seconds` at open time. Distinct from the per-read + /// idle timeout: an upstream that trickles a byte before every idle timeout + /// still cannot run past this. `None` disables the total cap. + total_deadline: Option, +} + +/// A body read in progress: the handle plus a cancel watch shared with close/reaper. +struct TakenStream { + handle: HttpStreamHandle, + cancel: Arc, +} + /// Errors raised while an outbound HTTP request is in flight. /// /// Budget failures stay structured until the interpreter can attach source @@ -1411,7 +1947,18 @@ pub struct IoClient { enum HttpClientError { Request(String), Budget(BudgetExceeded), - Timeout { seconds: u64 }, + Timeout { + seconds: u64, + }, + /// The stream was explicitly closed (or finished) while a body read was in + /// flight. Distinct from absolute-lifetime [`Self::Timeout`]. + Closed, + /// The downstream (browser) client disconnected while a proxy handler was + /// blocked on this upstream read, so the read was cancelled cooperatively. + /// A normal, expected event — distinct from a fault — surfaced with + /// `ErrorKind::Cancelled` so the concurrent loop does not count it as a + /// handler failure. + Disconnected, } impl From for HttpClientError { @@ -1442,6 +1989,13 @@ enum OutboundHttpDeadline { /// quickly an in-flight socket operation observes `ExecutionBudget::cancel()`. const HTTP_CANCELLATION_POLL_INTERVAL: Duration = Duration::from_millis(10); +/// How often a blocked upstream operation polls its handler's pending requests +/// for a client disconnect (the transport drops the oneshot receiver). Polled +/// because the sender lives behind an `Arc>>` shared with the +/// transport rather than an awaitable primitive; small so a disconnect is +/// observed promptly. +const REQUEST_DISCONNECT_POLL_INTERVAL: Duration = Duration::from_millis(20); + #[derive(Debug)] enum FileReadError { Io(String), @@ -1495,6 +2049,10 @@ impl IoClient { next_process_id: Mutex::new(1), db_handles: Mutex::new(HashMap::new()), next_db_id: Mutex::new(1), + stream_handles: Arc::new(std::sync::Mutex::new(StreamRegistry::default())), + next_stream_id: Mutex::new(1), + #[cfg(test)] + active_stream_reapers: Arc::new(std::sync::atomic::AtomicUsize::new(0)), config, } } @@ -1590,106 +2148,731 @@ impl IoClient { self.send_http_request(request, method, budget).await } - /// Send a request and consume its body without ever buffering more than the - /// configured response ceiling. The budget passed here is deliberately the - /// interpreter's *live* budget, not construction-time IoClient state: the - /// REPL replaces its budget for every command. - async fn send_http_request( + /// Open a streaming outbound request: send it and return + /// `(status, response headers, stream handle id)` as soon as the response + /// head arrives, WITHOUT buffering the body. The body stays open behind the + /// returned handle id for incremental `wait for next chunk|line` reads. + /// + /// The head phase (connect + headers) is bounded by the same finite + /// deadline and cooperative-cancellation machinery as a buffered request; + /// each later body read is bounded per-chunk in [`Self::stream_pull`]. + async fn open_http_stream( &self, - request: reqwest::RequestBuilder, method: &str, + url: &str, + headers: &[(String, String)], + body: Option, budget: Arc, ) -> Result<(u16, Vec<(String, String)>, String), HttpClientError> { - let method = method.to_string(); - let operation_budget = Arc::clone(&budget); - let operation = async move { - use futures_util::StreamExt; - - let response = request.send().await.map_err(|e| { - HttpClientError::Request(format!("Failed to send HTTP {method} request: {e}")) - })?; + use futures_util::StreamExt; - let status = response.status().as_u16(); - // Header names are normalized to lowercase for consistent access - // from WFL (e.g. resp.headers["content-type"]), and non-UTF8 - // values are converted lossily instead of dropped. - let response_headers = response - .headers() - .iter() - .map(|(name, value)| { - ( - name.as_str().to_ascii_lowercase(), - String::from_utf8_lossy(value.as_bytes()).into_owned(), - ) - }) - .collect(); - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|value| value.to_str().ok()) - .map(str::to_owned); + let parsed_method = reqwest::Method::from_bytes(method.as_bytes()) + .map_err(|_| HttpClientError::Request(format!("Invalid HTTP method: {method}")))?; + let mut request = self.http_client.request(parsed_method, url); + for (name, value) in headers { + request = request.header(name.as_str(), value.as_str()); + } + if let Some(body) = body { + request = request.body(body); + } - let max_response_bytes = operation_budget.limits().max_response_bytes; - if let Some(content_length) = response.content_length() { - let max_as_u64 = u64::try_from(max_response_bytes).unwrap_or(u64::MAX); - if content_length > max_as_u64 { - return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { - limit: max_response_bytes, - actual: usize::try_from(content_length).unwrap_or(usize::MAX), - })); + let method_owned = method.to_string(); + // Start the absolute total-lifetime clock at request initiation — BEFORE + // `send()` — so connect + header time counts toward + // `outbound_stream_max_seconds`, matching the documented "total lifetime + // measured from when the stream is opened". The head phase below is then + // bounded by the remaining time to that deadline as well as the idle + // timeout, so a stalled connect/header handshake cannot outlive the total. + // `outbound_stream_deadline` clamps extreme config values so Instant + // arithmetic cannot panic. + let total_deadline = outbound_stream_deadline(self.config.outbound_stream_max_seconds); + let idle_timeout = Duration::from_secs(self.config.timeout_seconds.max(1)); + let configured_timeout = match total_deadline { + Some(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(self.outbound_stream_timeout_error()); } + idle_timeout.min(remaining) } + None => idle_timeout, + }; + let op = async move { + request.send().await.map_err(|e| { + HttpClientError::Request(format!("Failed to send HTTP {method_owned} request: {e}")) + }) + }; + // Only the head is awaited here; dropping this future on + // timeout/cancel aborts the connection cleanly. + let response = + Self::run_http_with_budget(Arc::clone(&budget), configured_timeout, op).await?; + + let status = response.status().as_u16(); + let response_headers = response + .headers() + .iter() + .map(|(name, value)| { + ( + name.as_str().to_ascii_lowercase(), + String::from_utf8_lossy(value.as_bytes()).into_owned(), + ) + }) + .collect(); - // Do not retain a full raw byte buffer and then allocate a second, - // potentially larger UTF-8 string. Decode each network chunk into - // bounded scratch space, and enforce the same ceiling on both wire - // bytes and decoded UTF-8 bytes. Invalid UTF-8 alone can expand 3x - // when replaced with U+FFFD. - let initial_capacity = response - .content_length() - .and_then(|len| usize::try_from(len).ok()) - .unwrap_or(0) - .min(max_response_bytes) - .min(64 * 1024); - let encoding = Self::http_text_encoding(content_type.as_deref()); - let mut decoder = encoding.new_decoder(); - let mut body = String::with_capacity(initial_capacity); - let mut wire_bytes = 0_usize; - let mut stream = response.bytes_stream(); - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|e| { - HttpClientError::Request(format!("Failed to read response body: {e}")) - })?; - let actual = wire_bytes.saturating_add(chunk.len()); - if chunk.len() > max_response_bytes.saturating_sub(wire_bytes) { - return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { - limit: max_response_bytes, - actual, - })); - } - wire_bytes = actual; - Self::decode_http_chunk( - &mut decoder, - &chunk, - false, - &mut body, - max_response_bytes, - )?; + // Reject an over-ceiling body up front when the length is advertised; + // per-chunk reads enforce the same ceiling for chunked/unknown lengths. + let max_response_bytes = budget.limits().max_response_bytes; + if let Some(content_length) = response.content_length() { + let max_as_u64 = u64::try_from(max_response_bytes).unwrap_or(u64::MAX); + if content_length > max_as_u64 { + return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { + limit: max_response_bytes, + actual: usize::try_from(content_length).unwrap_or(usize::MAX), + })); } - Self::decode_http_chunk(&mut decoder, &[], true, &mut body, max_response_bytes)?; + } - Ok((status, response_headers, body)) + let stream = response + .bytes_stream() + .map(|chunk| chunk.map(|b| b.to_vec())); + // `total_deadline` was started at request initiation above (before the + // head was sent) so it covers connect/header time too. + let handle = HttpStreamHandle { + stream: Box::pin(stream), + buffer: Vec::new(), + done: false, + bytes_read: 0, + total_deadline, + }; + let handle_id = { + let mut next_id = self.next_stream_id.lock().await; + let id = format!("httpstream{}", *next_id); + *next_id += 1; + id }; - // A custom live budget may deliberately have no run-wide deadline. In - // a lifetime-exempt main loop, fall back to the interpreter's - // configured `timeout_seconds` (minimum one second) so the individual - // network operation is still finite and user-configurable. - let configured_timeout = Duration::from_secs(self.config.timeout_seconds.max(1)); - Self::run_http_with_budget(budget, configured_timeout, operation).await + // Insert the slot FIRST, then arm the reaper under the same lock so the + // reaper can never fire before the slot exists (and so close/finish that + // races with open still sees a consistent cancel handle). + let cancel = StreamCancel::new(); + { + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(Instant::now()); + registry.live.insert( + handle_id.clone(), + StreamSlot { + handle: Some(handle), + deadline: total_deadline, + cancel: Arc::clone(&cancel), + reaper_abort: None, + owner: None, + }, + ); + if let Some(deadline) = total_deadline { + let handles = Arc::clone(&self.stream_handles); + let reap_id = handle_id.clone(); + let cancel_reap = Arc::clone(&cancel); + #[cfg(test)] + let reaper_guard = + ActiveStreamReaperGuard::new(Arc::clone(&self.active_stream_reapers)); + let join = tokio::spawn(async move { + #[cfg(test)] + let _reaper_guard = reaper_guard; + let remaining = deadline.saturating_duration_since(Instant::now()); + tokio::time::sleep(remaining).await; + // Remove the heavy live slot and preserve only a bounded, + // lightweight typed terminal record for one later read. + let now = Instant::now(); + let mut registry = handles.lock().unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(now); + if let Some(mut slot) = registry.live.remove(&reap_id) { + let terminal = cancel_reap.terminate(StreamTerminal::Timeout); + slot.reaper_abort = None; // we are the reaper + drop(slot.handle.take()); + remove_stream_owner(&mut slot, &reap_id); + registry.remember_recent(reap_id, terminal, now); + } + }); + if let Some(slot) = registry.live.get_mut(&handle_id) { + slot.reaper_abort = Some(join.abort_handle()); + } else { + // Already finished before we armed — cancel the timer. + join.abort(); + } + } + } + + Ok((status, response_headers, handle_id)) } - /// Select the response encoding while preserving reqwest's text behavior: + /// Atomically attach a handler owner to a freshly opened stream. If expiry + /// won the race, return its typed outcome without creating stale ownership. + fn claim_stream_owner( + &self, + handle_id: &str, + owner: &StreamOwner, + ) -> Result<(), HttpClientError> { + let now = Instant::now(); + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + registry.prune_recent(now); + if let Some(slot) = registry.live.get_mut(handle_id) { + owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(handle_id.to_string()); + slot.owner = Some(Arc::clone(owner)); + return Ok(()); + } + if let Some(terminal) = registry.take_recent(handle_id, now) { + return Err(self.stream_terminal_error(terminal)); + } + Err(HttpClientError::Closed) + } + + /// Close every live stream owned by one handler. The owner lock is released + /// before the registry lock is acquired, preserving the registry->owner + /// nesting order used by the reaper. + fn close_stream_owner(&self, owner: &StreamOwner) { + let ids: Vec = { + let mut owned = owner.lock().unwrap_or_else(|error| error.into_inner()); + owned.drain().collect() + }; + self.close_stream_ids(&ids); + } + + /// Close a selected set of live streams without disturbing other handles + /// owned by the same handler. + fn close_stream_ids(&self, ids: &[String]) { + if ids.is_empty() { + return; + } + + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + registry.prune_recent(Instant::now()); + for id in ids { + if let Some(mut slot) = registry.live.remove(id) { + slot.cancel.terminate(StreamTerminal::Closed); + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, id); + } + registry.forget_recent(id); + } + } + + /// Signal cancel, abort the reaper, drop any parked handle, and remove the + /// slot. Guaranteed (std mutex) — usable from Drop. Returns whether a slot + /// was present. + fn finish_stream_slot_sync(&self, handle_id: &str, terminal: StreamTerminal) -> bool { + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(Instant::now()); + let had_recent = registry.forget_recent(handle_id); + if let Some(mut slot) = registry.live.remove(handle_id) { + slot.cancel.terminate(terminal); + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + true + } else { + had_recent + } + } + + async fn finish_stream_slot(&self, handle_id: &str) -> bool { + self.finish_stream_slot_sync(handle_id, StreamTerminal::Closed) + } + + /// Remove a stream handle from its slot so a body read can await without + /// holding the global handle lock. The cancel watch stays alive so close/ + /// expire aborts the read. A recent clean EOF yields `Ok(None)`; unknown, + /// closed, and past-deadline handles remain errors. + fn take_stream(&self, handle_id: &str) -> Result, HttpClientError> { + let now = Instant::now(); + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(now); + if !registry.live.contains_key(handle_id) { + if let Some(terminal) = registry.take_recent(handle_id, now) { + return match terminal { + StreamTerminal::CleanEof => Ok(None), + terminal => Err(self.stream_terminal_error(terminal)), + }; + } + return Err(HttpClientError::Request(format!( + "Unknown or already-closed stream handle '{handle_id}'" + ))); + } + if let Some(terminal) = registry + .live + .get(handle_id) + .and_then(|slot| slot.cancel.terminal()) + { + if let Some(mut slot) = registry.live.remove(handle_id) { + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + } + return Err(self.stream_terminal_error(terminal)); + } + let past_deadline = registry + .live + .get(handle_id) + .expect("live stream checked above") + .deadline + .is_some_and(|deadline| deadline <= now); + if past_deadline { + if let Some(mut slot) = registry.live.remove(handle_id) { + let terminal = slot.cancel.terminate(StreamTerminal::Timeout); + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + return Err(self.stream_terminal_error(terminal)); + } + return Err(self.outbound_stream_timeout_error()); + } + let slot = registry + .live + .get_mut(handle_id) + .expect("live stream checked above"); + let cancel = Arc::clone(&slot.cancel); + match slot.handle.take() { + Some(handle) => Ok(Some(TakenStream { handle, cancel })), + None => Err(HttpClientError::Request(format!( + "Unknown or already-closed stream handle '{handle_id}'" + ))), + } + } + + /// Return a still-open stream handle after a body read. Refuses reinsertion + /// if the stream was closed/expired mid-read (cancel flag or missing slot). + fn put_stream( + &self, + handle_id: &str, + handle: HttpStreamHandle, + cancel: &StreamCancel, + ) -> Result<(), HttpClientError> { + // Observing upstream EOF is the linearization point for clean + // completion. Finalize that already-latched result before consulting + // the wall clock or generic terminal rejection below: a reaper/close + // may have removed the slot after EOF won, but it must not erase the + // one follow-up `nothing` read. + if handle.done { + let terminal = cancel.terminate(StreamTerminal::CleanEof); + if terminal == StreamTerminal::CleanEof { + drop(handle); + let now = Instant::now(); + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(now); + if let Some(mut slot) = registry.live.remove(handle_id) { + slot.cancel.terminate(StreamTerminal::CleanEof); + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + } + // `remember_recent` replaces an existing record for this ID, + // so terminalization leaves exactly one bounded one-shot + // result even when the racing reaper already recorded EOF. + registry.remember_recent(handle_id.to_string(), StreamTerminal::CleanEof, now); + return Ok(()); + } + } + + if let Some(terminal) = cancel.terminal() { + drop(handle); + // Ensure the slot is gone (reaper/close may already have removed it). + let _ = self.finish_stream_slot_sync(handle_id, terminal); + return Err(self.stream_terminal_error(terminal)); + } + let now = Instant::now(); + let mut registry = self + .stream_handles + .lock() + .unwrap_or_else(|e| e.into_inner()); + registry.prune_recent(now); + if !registry.live.contains_key(handle_id) { + drop(handle); + let terminal = cancel + .terminal() + .or_else(|| registry.take_recent(handle_id, now)); + return Err(terminal + .map(|reason| self.stream_terminal_error(reason)) + .unwrap_or(HttpClientError::Closed)); + } + let terminal = registry.live.get(handle_id).and_then(|slot| { + slot.cancel.terminal().or_else(|| { + if slot.deadline.is_some_and(|deadline| deadline <= now) { + Some(slot.cancel.terminate(StreamTerminal::Timeout)) + } else { + None + } + }) + }); + if let Some(terminal) = terminal { + drop(handle); + if let Some(mut slot) = registry.live.remove(handle_id) { + if let Some(abort) = slot.reaper_abort.take() { + abort.abort(); + } + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + } + return Err(self.stream_terminal_error(terminal)); + } + let slot = registry + .live + .get_mut(handle_id) + .expect("live stream checked above"); + slot.handle = Some(handle); + Ok(()) + } + + fn stream_terminal_error(&self, terminal: StreamTerminal) -> HttpClientError { + match terminal { + StreamTerminal::CleanEof => HttpClientError::Closed, + StreamTerminal::Timeout => self.outbound_stream_timeout_error(), + StreamTerminal::Closed => HttpClientError::Closed, + } + } + + fn outbound_stream_timeout_error(&self) -> HttpClientError { + HttpClientError::Timeout { + seconds: outbound_stream_effective_seconds(self.config.outbound_stream_max_seconds) + .unwrap_or(self.config.timeout_seconds.max(1)), + } + } + + /// Pull one network chunk into `handle.buffer`, bounded by the per-chunk + /// read deadline, the absolute total, and cooperative stream cancellation + /// (close/expire while reading). Returns `Ok(true)` when bytes were added, + /// `Ok(false)` at clean EOF (sets `handle.done`). + async fn stream_pull( + &self, + handle: &mut HttpStreamHandle, + budget: &Arc, + cancel: &StreamCancel, + ) -> Result { + use futures_util::StreamExt; + + if handle.done { + let terminal = cancel.terminate(StreamTerminal::CleanEof); + return if terminal == StreamTerminal::CleanEof { + Ok(false) + } else { + Err(self.stream_terminal_error(terminal)) + }; + } + let mut terminal_rx = cancel.subscribe(); + if let Some(terminal) = *terminal_rx.borrow() { + return Err(self.stream_terminal_error(terminal)); + } + let max_response_bytes = budget.limits().max_response_bytes; + let idle_timeout = Duration::from_secs(self.config.timeout_seconds.max(1)); + let configured_timeout = match handle.total_deadline { + Some(deadline) => { + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + return Err(self.outbound_stream_timeout_error()); + } + idle_timeout.min(remaining) + } + None => idle_timeout, + }; + + // Race the network read against stream cancellation so close-during- + // active-read aborts the upstream promptly. + let op = Self::run_http_with_budget(Arc::clone(budget), configured_timeout, async { + Ok::>>, HttpClientError>(handle.stream.next().await) + }); + tokio::pin!(op); + let next = tokio::select! { + // Prefer a simultaneously-ready body result so the terminal + // re-check below is the single deterministic arbiter: expiry still + // wins, and a ready chunk can never be reinserted after the reaper. + biased; + result = &mut op => result?, + changed = terminal_rx.changed() => { + let _ = changed; + let terminal = (*terminal_rx.borrow()).unwrap_or(StreamTerminal::Closed); + return Err(self.stream_terminal_error(terminal)); + } + }; + // Re-check after select so a simultaneously-ready body chunk cannot win + // over an already-recorded hard deadline and be reinserted. + if let Some(terminal) = cancel.terminal() { + return Err(self.stream_terminal_error(terminal)); + } + + match next { + Some(Ok(bytes)) => { + let actual = handle.bytes_read.saturating_add(bytes.len()); + if bytes.len() > max_response_bytes.saturating_sub(handle.bytes_read) { + return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { + limit: max_response_bytes, + actual, + })); + } + handle.bytes_read = actual; + handle.buffer.extend_from_slice(&bytes); + Ok(true) + } + Some(Err(e)) => Err(HttpClientError::Request(format!( + "Failed to read response chunk: {e}" + ))), + None => { + let terminal = cancel.terminate(StreamTerminal::CleanEof); + if terminal == StreamTerminal::CleanEof { + handle.done = true; + Ok(false) + } else { + Err(self.stream_terminal_error(terminal)) + } + } + } + } + + /// Return a `Timeout` error if the handle's absolute stream lifetime + /// (`outbound_stream_max_seconds`, tracked as `total_deadline`) has elapsed. + /// Called before serving locally-buffered bytes so the absolute lifetime is + /// enforced even when no network read is performed; `stream_pull` performs the + /// same check before each network read. + fn check_stream_deadline(&self, handle: &HttpStreamHandle) -> Result<(), HttpClientError> { + if let Some(deadline) = handle.total_deadline + && deadline.saturating_duration_since(Instant::now()).is_zero() + { + return Err(self.outbound_stream_timeout_error()); + } + Ok(()) + } + + /// Pull the next raw byte chunk from a streaming response. Returns + /// `Ok(None)` at clean end of stream (handle is finished). On error or EOF + /// the slot is removed and the reaper aborted so the upstream is released + /// and no sleeping timer remains. + async fn next_chunk( + &self, + handle_id: &str, + budget: Arc, + ) -> Result>, HttpClientError> { + let Some(TakenStream { mut handle, cancel }) = self.take_stream(handle_id)? else { + return Ok(None); + }; + + if let Err(e) = self.check_stream_deadline(&handle) { + let _ = self.finish_stream_slot(handle_id).await; + return Err(e); + } + + if !handle.buffer.is_empty() { + let chunk = std::mem::take(&mut handle.buffer); + self.put_stream(handle_id, handle, &cancel)?; + return Ok(Some(chunk)); + } + + match self.stream_pull(&mut handle, &budget, &cancel).await { + Ok(true) => { + let chunk = std::mem::take(&mut handle.buffer); + self.put_stream(handle_id, handle, &cancel)?; + Ok(Some(chunk)) + } + Ok(false) => { + let _ = self.finish_stream_slot(handle_id).await; + Ok(None) + } + Err(e) => { + let _ = self.finish_stream_slot(handle_id).await; + Err(e) + } + } + } + + /// Pull the next newline-delimited line (trailing `\n`, and a paired `\r`, + /// stripped) from a streaming response. A final unterminated line is + /// returned before EOF. Returns `Ok(None)` at clean end of stream. + async fn next_line( + &self, + handle_id: &str, + budget: Arc, + ) -> Result, HttpClientError> { + let Some(TakenStream { mut handle, cancel }) = self.take_stream(handle_id)? else { + return Ok(None); + }; + + loop { + let clean_eof_latched = + handle.done && cancel.terminal() == Some(StreamTerminal::CleanEof); + if !clean_eof_latched && let Err(e) = self.check_stream_deadline(&handle) { + let _ = self.finish_stream_slot(handle_id).await; + return Err(e); + } + + if let Some(pos) = handle.buffer.iter().position(|&b| b == b'\n') { + let mut line: Vec = handle.buffer.drain(..=pos).collect(); + line.pop(); // drop '\n' + if line.last() == Some(&b'\r') { + line.pop(); // drop paired '\r' (CRLF) + } + self.put_stream(handle_id, handle, &cancel)?; + return Ok(Some(String::from_utf8_lossy(&line).into_owned())); + } + + if handle.done { + // Final unterminated line (if any), then fully finish — do NOT + // leave a done slot + reaper parked for a subsequent read. + if handle.buffer.is_empty() { + let _ = self.finish_stream_slot(handle_id).await; + return Ok(None); + } + let mut line = std::mem::take(&mut handle.buffer); + if line.last() == Some(&b'\r') { + line.pop(); + } + // Preserve one lightweight clean-EOF result so the next wait + // binds `nothing`; `put_stream` removes all live stream state. + self.put_stream(handle_id, handle, &cancel)?; + return Ok(Some(String::from_utf8_lossy(&line).into_owned())); + } + + if let Err(e) = self.stream_pull(&mut handle, &budget, &cancel).await { + let _ = self.finish_stream_slot(handle_id).await; + return Err(e); + } + } + } + + /// Close a streaming response handle if present. Signals cancel (aborting + /// any active read), drops the handle, and aborts the reaper timer. + /// Idempotent. + async fn close_stream(&self, handle_id: &str) -> bool { + self.finish_stream_slot(handle_id).await + } + + /// Send a request and consume its body without ever buffering more than the + /// configured response ceiling. The budget passed here is deliberately the + /// interpreter's *live* budget, not construction-time IoClient state: the + /// REPL replaces its budget for every command. + async fn send_http_request( + &self, + request: reqwest::RequestBuilder, + method: &str, + budget: Arc, + ) -> Result<(u16, Vec<(String, String)>, String), HttpClientError> { + let method = method.to_string(); + let operation_budget = Arc::clone(&budget); + let operation = async move { + use futures_util::StreamExt; + + let response = request.send().await.map_err(|e| { + HttpClientError::Request(format!("Failed to send HTTP {method} request: {e}")) + })?; + + let status = response.status().as_u16(); + // Header names are normalized to lowercase for consistent access + // from WFL (e.g. resp.headers["content-type"]), and non-UTF8 + // values are converted lossily instead of dropped. + let response_headers = response + .headers() + .iter() + .map(|(name, value)| { + ( + name.as_str().to_ascii_lowercase(), + String::from_utf8_lossy(value.as_bytes()).into_owned(), + ) + }) + .collect(); + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + let max_response_bytes = operation_budget.limits().max_response_bytes; + if let Some(content_length) = response.content_length() { + let max_as_u64 = u64::try_from(max_response_bytes).unwrap_or(u64::MAX); + if content_length > max_as_u64 { + return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { + limit: max_response_bytes, + actual: usize::try_from(content_length).unwrap_or(usize::MAX), + })); + } + } + + // Do not retain a full raw byte buffer and then allocate a second, + // potentially larger UTF-8 string. Decode each network chunk into + // bounded scratch space, and enforce the same ceiling on both wire + // bytes and decoded UTF-8 bytes. Invalid UTF-8 alone can expand 3x + // when replaced with U+FFFD. + let initial_capacity = response + .content_length() + .and_then(|len| usize::try_from(len).ok()) + .unwrap_or(0) + .min(max_response_bytes) + .min(64 * 1024); + let encoding = Self::http_text_encoding(content_type.as_deref()); + let mut decoder = encoding.new_decoder(); + let mut body = String::with_capacity(initial_capacity); + let mut wire_bytes = 0_usize; + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|e| { + HttpClientError::Request(format!("Failed to read response body: {e}")) + })?; + let actual = wire_bytes.saturating_add(chunk.len()); + if chunk.len() > max_response_bytes.saturating_sub(wire_bytes) { + return Err(HttpClientError::Budget(BudgetExceeded::ResponseBytes { + limit: max_response_bytes, + actual, + })); + } + wire_bytes = actual; + Self::decode_http_chunk( + &mut decoder, + &chunk, + false, + &mut body, + max_response_bytes, + )?; + } + Self::decode_http_chunk(&mut decoder, &[], true, &mut body, max_response_bytes)?; + + Ok((status, response_headers, body)) + }; + + // A custom live budget may deliberately have no run-wide deadline. In + // a lifetime-exempt main loop, fall back to the interpreter's + // configured `timeout_seconds` (minimum one second) so the individual + // network operation is still finite and user-configurable. + let configured_timeout = Duration::from_secs(self.config.timeout_seconds.max(1)); + Self::run_http_with_budget(budget, configured_timeout, operation).await + } + + /// Select the response encoding while preserving reqwest's text behavior: /// honor a declared charset and default to UTF-8. fn http_text_encoding(content_type: Option<&str>) -> &'static encoding_rs::Encoding { let charset = content_type.and_then(|value| { @@ -1788,11 +2971,25 @@ impl IoClient { F: std::future::Future>, { let deadline = Self::outbound_http_deadline(&budget, configured_timeout)?; - let timeout_duration = match deadline { + // The budget/run-derived finite duration, if any. + let budget_duration = match deadline { OutboundHttpDeadline::None => None, OutboundHttpDeadline::Execution { remaining, .. } => Some(remaining), OutboundHttpDeadline::MainLoop { duration } => Some(duration), }; + // The operation is bounded by the SHORTER of the caller's configured + // timeout — which already encodes the stream's idle timeout AND its + // remaining absolute-total deadline (see `stream_pull`/`open_http_stream`) + // — and the run/budget deadline. Whichever is smaller decides both how + // long we wait and which error we report. `configured_timeout` is always + // finite, so the operation is bounded even when the budget has no + // run-wide deadline (previously that path discarded the stream deadline + // entirely and could wait out the whole run). + let budget_is_binding = matches!(budget_duration, Some(bd) if bd <= configured_timeout); + let timeout_duration = match budget_duration { + Some(bd) if budget_is_binding => bd, + _ => configured_timeout, + }; let cancellation_budget = Arc::clone(&budget); let cancellation = async move { @@ -1803,12 +3000,7 @@ impl IoClient { tokio::time::sleep(HTTP_CANCELLATION_POLL_INTERVAL).await; } }; - let timeout = async move { - match timeout_duration { - Some(duration) => tokio::time::sleep(duration).await, - None => std::future::pending::<()>().await, - } - }; + let timeout = async move { tokio::time::sleep(timeout_duration).await }; tokio::pin!(operation); tokio::pin!(cancellation); @@ -1816,16 +3008,29 @@ impl IoClient { tokio::select! { result = &mut operation => result, _ = &mut cancellation => Err(HttpClientError::Budget(BudgetExceeded::Cancelled)), - _ = &mut timeout => match deadline { - OutboundHttpDeadline::Execution { limit_secs, .. } => { - Err(HttpClientError::Budget(BudgetExceeded::Deadline { limit_secs })) - } - OutboundHttpDeadline::MainLoop { duration } => { - Err(HttpClientError::Timeout { seconds: duration.as_secs() }) - } - OutboundHttpDeadline::None => unreachable!("disabled timeout cannot complete"), - }, - } + _ = &mut timeout => { + if budget_is_binding { + // The run/budget deadline was the shorter bound. + match deadline { + OutboundHttpDeadline::Execution { limit_secs, .. } => { + Err(HttpClientError::Budget(BudgetExceeded::Deadline { limit_secs })) + } + OutboundHttpDeadline::MainLoop { duration } => { + Err(HttpClientError::Timeout { seconds: duration.as_secs() }) + } + OutboundHttpDeadline::None => { + unreachable!("budget cannot be binding when there is no budget deadline") + } + } + } else { + // The caller's configured timeout (stream idle / absolute + // total) was the shorter bound. + Err(HttpClientError::Timeout { + seconds: configured_timeout.as_secs().max(1), + }) + } + } + } } #[allow(dead_code)] @@ -2888,7 +4093,15 @@ impl Interpreter { web_servers: RefCell::new(HashMap::new()), // Initialize empty web servers map web_socket_servers: RefCell::new(HashMap::new()), // Initialize empty WebSocket servers map ws_connections: Arc::new(std::sync::Mutex::new(HashMap::new())), // Live WebSocket connections - pending_responses: RefCell::new(HashMap::new()), // Initialize empty pending responses map + pending_responses: Rc::new(RefCell::new(HashMap::new())), // Initialize empty pending responses map + server_response_streams: Rc::new(RefCell::new(HashMap::new())), + open_response_streams: Rc::new(RefCell::new(Vec::new())), + open_pending_requests: Rc::new(RefCell::new(Vec::new())), + open_http_streams: Rc::new(RefCell::new(Arc::new(std::sync::Mutex::new( + HashSet::new(), + )))), + accepted_request: Cell::new(false), + next_response_stream_id: std::cell::Cell::new(1), config, current_source_file: RefCell::new(None), // No source file initially loading_stack: RefCell::new(Vec::new()), // Empty loading stack @@ -3341,501 +4554,1367 @@ impl Interpreter { column, ErrorKind::Timeout, ), + HttpClientError::Closed => { + RuntimeError::new("Stream was closed while reading".to_string(), line, column) + } + HttpClientError::Disconnected => RuntimeError::with_kind( + "Client disconnected; upstream read cancelled".to_string(), + line, + column, + ErrorKind::Cancelled, + ), } } - /// Preserve ordinary file I/O failures while classifying byte-ceiling - /// breaches as catchable execution-budget resource errors. - fn file_read_error(&self, error: FileReadError, line: usize, column: usize) -> RuntimeError { - match error { - FileReadError::Io(message) => RuntimeError::new(message, line, column), - FileReadError::Budget(exceeded) => self.budget_error(exceeded, line, column), - } - } - - /// Map a pattern-VM error onto a `RuntimeError`. Budget breaches (step/state - /// ceilings, cancellation) surface as catchable `ResourceLimit` errors so a - /// ReDoS/cancellation during matching is not silently collapsed into a - /// non-match; structural pattern errors surface as general runtime errors. - fn pattern_error( + /// Resolve a `wait for next chunk|line from ` operand to a stream + /// handle id. Requires the streaming-response object bound by + /// `stream response as ` (reads its internal `_stream` id); a bare text + /// id is rejected so an arbitrary string can't be aimed at a stream handle. + async fn resolve_stream_handle( &self, - err: crate::pattern::PatternError, + source: &Expression, + env: &Rc>, line: usize, column: usize, - ) -> RuntimeError { - use crate::pattern::PatternError; - let kind = match err { - // A pattern that outruns the wall-clock deadline is a timeout, with - // the historic `[Timeout]` kind and message, so existing timeout - // handling/tests keep matching. - PatternError::Timeout { .. } => ErrorKind::Timeout, - PatternError::StepLimitExceeded - | PatternError::StateLimitExceeded - | PatternError::Cancelled => ErrorKind::ResourceLimit, - _ => ErrorKind::General, - }; - RuntimeError::with_kind(err.to_string(), line, column, kind) + ) -> Result { + let value = self.evaluate_expression(source, Rc::clone(env)).await?; + match &value { + Value::Object(obj) => match obj.borrow().get("_stream") { + Some(Value::Text(id)) => Ok(id.to_string()), + _ => Err(RuntimeError::new( + "Expected a streaming response handle (from `stream response as ...`), \ + but this value has no open stream" + .to_string(), + line, + column, + )), + }, + _ => Err(RuntimeError::new( + format!( + "Expected a streaming response handle (from `stream response as ...`), got {}", + value.type_name() + ), + line, + column, + )), + } } - /// Read a WFL source file (`load module`, `include from`, `execute file`) - /// under the shared source-size ceiling. Reads at most `max_source_size + 1` - /// bytes, so an oversized file is refused without ever allocating the whole - /// thing — this holds even when the file's metadata is unavailable, stale, - /// or reports `0` (special files), which a metadata-only check would miss. - async fn read_source_bounded( + /// Resolve a `write line|chunk`/`flush` operand to a server response-stream + /// handle id. Requires the object bound by `start streaming response as ...` + /// (reads its internal `_server_stream` id); a bare text id is rejected so an + /// arbitrary string can't be aimed at a server response stream. + async fn resolve_server_stream_handle( &self, - path: &std::path::Path, + target: &Expression, + env: &Rc>, line: usize, column: usize, ) -> Result { - use tokio::io::AsyncReadExt; - - let io_err = |e: std::io::Error| { - let kind = match e.kind() { - std::io::ErrorKind::NotFound => ErrorKind::FileNotFound, - std::io::ErrorKind::PermissionDenied => ErrorKind::PermissionDenied, - _ => ErrorKind::General, - }; - RuntimeError::with_kind( - format!("Cannot read source file '{}': {e}", path.display()), + let value = self.evaluate_expression(target, Rc::clone(env)).await?; + match &value { + Value::Object(obj) => match obj.borrow().get("_server_stream") { + Some(Value::Text(id)) => Ok(id.to_string()), + _ => Err(RuntimeError::new( + "Expected a server response stream (from `start streaming response as ...`), \ + but this value has no open stream" + .to_string(), + line, + column, + )), + }, + _ => Err(RuntimeError::new( + format!( + "Expected a server response stream (from `start streaming response as ...`), got {}", + value.type_name() + ), line, column, - kind, - ) - }; + )), + } + } - let max = self.budget.max_source_bytes(); - // Read one byte past the limit so exceeding it is detectable; the buffer - // never grows beyond `max + 1`. - let read_cap = (max as u64).saturating_add(1); - let file = tokio::fs::File::open(path).await.map_err(io_err)?; - let mut buf = Vec::new(); - file.take(read_cap) - .read_to_end(&mut buf) - .await - .map_err(io_err)?; + /// Swap the interpreter's per-execution run-state fields with `state`. + /// + /// Used by [`IsolatedHandler`] to make the run state poll-local under + /// concurrent execution: the interpreter's live fields and the parked + /// snapshot trade places, so exactly one handler's run state is installed at + /// a time. A plain field-by-field `mem::swap`, so it is its own inverse. + fn swap_run_state(&self, state: &mut RunState) { + std::mem::swap( + &mut *self.current_count.borrow_mut(), + &mut state.current_count, + ); + std::mem::swap( + &mut *self.in_count_loop.borrow_mut(), + &mut state.in_count_loop, + ); + let depth = self.call_depth.replace(state.call_depth); + state.call_depth = depth; + std::mem::swap(&mut *self.call_stack.borrow_mut(), &mut state.call_stack); + std::mem::swap( + &mut *self.current_block_overload_dups.borrow_mut(), + &mut state.block_overload_dups, + ); + std::mem::swap( + &mut *self.open_response_streams.borrow_mut(), + &mut state.open_response_streams, + ); + std::mem::swap( + &mut *self.open_pending_requests.borrow_mut(), + &mut state.open_pending_requests, + ); + std::mem::swap( + &mut *self.open_http_streams.borrow_mut(), + &mut state.open_http_streams, + ); + let accepted = self.accepted_request.replace(state.accepted_request); + state.accepted_request = accepted; + } - if let Err(exceeded) = self.budget.check_source_bytes(buf.len()) { - return Err(self.budget_error(exceeded, line, column)); - } + /// Drop each outbound streaming handle whose id is in `ids` from + /// `IoClient.stream_handles`, cancelling its in-flight upstream request + /// (dropping the reqwest body stream aborts the connection). Best-effort and + /// synchronous (usable from `Drop`): if the async lock is momentarily held, + /// the handles remain and are reclaimed at interpreter teardown. Idempotent — + /// an id already removed by EOF/error/explicit `close` is a no-op. + fn close_http_streams(&self, owner: &StreamOwner) { + self.io_client.close_stream_owner(owner); + } - String::from_utf8(buf).map_err(|_| { - RuntimeError::new( - format!("Source file '{}' is not valid UTF-8", path.display()), - line, - column, - ) - }) + /// Build an RAII guard that closes any outbound stream handles still tracked + /// as open when the guard drops — including when the `interpret()` future is + /// dropped/cancelled before reaching its normal handler-exit/program-cleanup + /// sites. On a normal run those sites have already drained the list, so the + /// guard is a no-op; on a dropped future it releases the leaked upstreams + /// (instead of leaking them until the interpreter itself is torn down). + fn outbound_stream_cleanup_guard(&self) -> OutboundStreamCleanup { + OutboundStreamCleanup { + io_client: Rc::clone(&self.io_client), + open_http_streams: Arc::clone(&self.open_http_streams.borrow()), + open_response_streams: Rc::clone(&self.open_response_streams), + server_response_streams: Rc::clone(&self.server_response_streams), + open_pending_requests: Rc::clone(&self.open_pending_requests), + pending_responses: Rc::clone(&self.pending_responses), + } } - fn assert_invariants(&self) { - debug_assert_eq!( - *self.in_count_loop.borrow(), - self.current_count.borrow().is_some() - ); + /// Drain and drop every outbound stream the current (serial) handler left + /// open. Called at the end of each serial `main loop` iteration and at program + /// exit, mirroring the concurrent path's per-handler `Drop`. + fn close_open_http_streams(&self) { + let owner = Arc::clone(&self.open_http_streams.borrow()); + self.close_http_streams(&owner); + } - debug_assert!(self.call_stack.borrow().len() < 10_000); + /// Stop tracking an outbound stream id as handler-owned — it has already left + /// `IoClient.stream_handles` (EOF, error, or an explicit `close`), so the + /// handler-exit cleanup must not try to (re-)drop it. + fn untrack_http_stream(&self, handle_id: &str) { + self.open_http_streams + .borrow_mut() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(handle_id); } - fn native_display(args: Vec) -> Result { - let mut line = String::new(); - for (i, arg) in args.iter().enumerate() { - if i > 0 { - line.push(' '); - } - line.push_str(&arg.to_string()); + /// Clone the sender of every downstream response stream this handler owns. A + /// client disconnect drops the stream's `Receiver`, so `Sender::closed()` on + /// a clone resolves — a proactive disconnect signal that a blocked upstream + /// read can select against, so a proxy handler wakes and cancels its upstream + /// the moment the browser goes away, instead of only discovering it at the + /// next downstream `write` or when the absolute stream deadline elapses. + /// Returns owned clones (no `RefCell` borrow is held across the later await). + fn downstream_disconnect_senders(&self) -> Vec>> { + let open = self.open_response_streams.borrow(); + if open.is_empty() { + return Vec::new(); } - io_capture::emit_line(&line); - Ok(Value::Null) + let map = self.server_response_streams.borrow(); + open.iter() + .filter_map(|id| map.get(id).map(|(tx, _)| tx.clone())) + .collect() } - pub async fn interpret(&mut self, program: &Program) -> Result> { - // Scope this run's budget as the TASK-local current budget, so leaf - // helpers with no budget parameter (the stdlib pattern builtins in - // particular) match under the run's configured ceilings and shared - // meters — and, crucially, so a library embedder that interleaves two - // interpreter futures on one thread never sees the other's budget or - // restores stale state across an `.await`. An `execute file` child that - // calls `interpret` again nests its own scope (same or child budget). - ExecutionBudget::scope(Arc::clone(&self.budget), self.interpret_inner(program)).await + /// Await until ANY of `senders`' downstream clients has disconnected (its + /// `Receiver` dropped). Never resolves if `senders` is empty — the caller + /// guards that case so `select!` still has a live branch. + async fn any_downstream_disconnected(senders: Vec>>) { + if senders.is_empty() { + std::future::pending::<()>().await; + return; + } + let closes: Vec<_> = senders.iter().map(|tx| Box::pin(tx.closed())).collect(); + let _ = futures_util::future::select_all(closes).await; } - /// Action names defined more than once in `statements` — the immediate - /// slice only, matching the same-scope overloading rule: two same-name - /// definitions in different blocks are independent actions, not - /// overloads. Returns `None` (no allocation kept) when nothing in the - /// slice is overloaded. - fn scan_block_overload_dups( - statements: &[Statement], - ) -> Option>> { - let mut counts: HashMap<&str, usize> = HashMap::new(); - let mut dups: std::collections::HashSet = std::collections::HashSet::new(); - for statement in statements { - if let Statement::ActionDefinition { name, .. } = statement { - let entry = counts.entry(name.as_str()).or_insert(0); - *entry += 1; - if *entry == 2 { - dups.insert(name.clone()); + /// Await until any of this handler's open pending requests has had its client + /// disconnect — the transport route task drops the oneshot receiver when the + /// client goes away, so the parked sender reports `is_closed()`. This is the + /// disconnect signal that is valid BEFORE `start streaming response` (when no + /// downstream response stream exists yet), so a handler blocked opening an + /// upstream head can still be cancelled by a browser disconnect. Polled (the + /// sender lives behind an `Arc>>` shared with the transport, + /// not an awaitable primitive); never resolves when the handler holds no + /// pending request, so the caller's `select!` still has a live branch. + async fn any_pending_request_disconnected(&self) { + loop { + // Compute in a tight scope so no `RefCell`/`Mutex` guard is held across + // the await below. `None` => nothing to watch. + let any_closed = { + let open = self.open_pending_requests.borrow(); + if open.is_empty() { + None + } else { + let pending = self.pending_responses.borrow(); + Some(open.iter().any(|id| match pending.get(id) { + Some(p) => match p.sender.try_lock() { + Ok(guard) => guard.as_ref().is_some_and(|s| s.is_closed()), + // Being responded to right now — not a disconnect. + Err(_) => false, + }, + // Owned by this handler yet absent from the map: the only + // removal that leaves an id in `open_pending_requests` is a + // sibling handler's `wait for request` global prune, which + // deletes ONLY closed (disconnected) senders — and a handler + // that answered its own request drops the id from + // `open_pending_requests` in the same step. So a missing + // owned id is a terminal disconnect, not "still connected"; + // reporting it as connected here is exactly the bug where a + // parked handler waits out its timeout after a sibling pruned + // its since-disconnected entry. + None => true, + })) + } + }; + match any_closed { + None => { + std::future::pending::<()>().await; + return; + } + Some(true) => return, + Some(false) => { + tokio::time::sleep(REQUEST_DISCONNECT_POLL_INTERVAL).await; } } } - if dups.is_empty() { - None - } else { - Some(Rc::new(dups)) + } + + /// Await until this handler's client has disconnected by EITHER signal: an + /// open downstream response stream's receiver dropped (post-`start streaming + /// response`, event-driven) OR an open pending request's oneshot receiver + /// dropped (pre-`start streaming response`, polled). Racing an upstream head + /// open / body read against this cancels it the moment the browser goes away, + /// whichever phase the handler is in. + async fn any_client_disconnected(&self, senders: Vec>>) { + tokio::select! { + _ = Self::any_downstream_disconnected(senders) => {} + _ = self.any_pending_request_disconnected() => {} } } - /// Speculatively arms `enforce_param_types` on any same-scope function - /// that `statements` will merge a new definition into — the merge makes - /// that member overloaded, so its temporal window starts when the block - /// starts executing, not at the later merge. Inherited (parent-scope) - /// names are not mergeable — defining over them is a shadowing error — - /// so only the local scope is consulted. The returned guard reverts the - /// arming on drop for members whose merge never actually executed. - fn arm_block_members( - statements: &[Statement], - env: &Rc>, - ) -> ArmedEnforcementGuard { - let mut armed = Vec::new(); - for statement in statements { - if let Statement::ActionDefinition { name, .. } = statement - && let Some(Value::Function(existing)) = env.borrow().get_local(name) + /// Close (drop the sender for) each server response stream whose handle id is + /// in `ids`, ending its body so the client stops waiting. Idempotent — an id + /// already closed by an explicit `close out` (or a disconnect) is a no-op — + /// so it is safe to call over a handler's full opened-stream list on exit. + fn close_response_streams(&self, ids: &[String]) { + if ids.is_empty() { + return; + } + let mut map = self.server_response_streams.borrow_mut(); + for id in ids { + map.remove(id); + } + } + + /// Drain and close every server response stream the current (serial) handler + /// left open. Called at the end of each serial `main loop` iteration and at + /// program exit, mirroring the concurrent path's per-handler `Drop`. + fn close_open_response_streams(&self) { + let ids = std::mem::take(&mut *self.open_response_streams.borrow_mut()); + self.close_response_streams(&ids); + } + + fn pending_response_disconnected_now(&self, request_id: &str) -> bool { + let owned = self + .open_pending_requests + .borrow() + .iter() + .any(|id| id == request_id); + if !owned { + return false; + } + + let pending = self.pending_responses.borrow(); + match pending.get(request_id) { + Some(entry) => match entry.sender.try_lock() { + Ok(sender) => sender.as_ref().is_none_or(|sender| sender.is_closed()), + Err(_) => false, + }, + None => true, + } + } + + /// Wait for one specific still-owned request to disconnect. Losing ownership + /// is deliberately not treated as cancellation: duplicate/forged responses + /// must retain their established general-error classification. + async fn pending_response_disconnected(&self, request_id: &str) { + loop { + if self.pending_response_disconnected_now(request_id) { + return; + } + if !self + .open_pending_requests + .borrow() + .iter() + .any(|id| id == request_id) { - let prior_enforce = existing.enforce_param_types.get(); - existing.enforce_param_types.set(true); - armed.push(ArmedMember { - env: Rc::clone(env), - name: name.clone(), - member: existing, - prior_enforce, - }); + std::future::pending::<()>().await; + return; } + tokio::time::sleep(REQUEST_DISCONNECT_POLL_INTERVAL).await; } - ArmedEnforcementGuard { armed } } - /// Enters a statement block for overload enforcement: computes the - /// block's own duplicate set and speculatively arms mergeable members - /// (see [`Self::arm_block_members`]). Both guards restore on drop. - fn enter_block_overloads( - &self, - statements: &[Statement], - env: &Rc>, - ) -> (BlockDupsScope<'_>, ArmedEnforcementGuard) { - let armed = Self::arm_block_members(statements, env); - ( - BlockDupsScope::enter( - &self.current_block_overload_dups, - Self::scan_block_overload_dups(statements), - ), - armed, - ) + fn first_pending_response_disconnected_now(&self, request_ids: &[String]) -> Option { + request_ids + .iter() + .find(|request_id| self.pending_response_disconnected_now(request_id)) + .cloned() } - /// The interpreter run body, executed inside the task-local budget scope - /// established by [`Interpreter::interpret`]. - async fn interpret_inner(&mut self, program: &Program) -> Result> { - // Reset per-run enforcement/loop state first, so a prior *terminal* - // budget breach (e.g. an uncaught timeout that unwound to the top) can't - // leak stale count-loop or depth state into this run — matters when one - // interpreter is reused (the REPL). Done before `assert_invariants` so - // the invariant holds regardless of how the previous run ended. - *self.in_count_loop.borrow_mut() = false; - *self.current_count.borrow_mut() = None; - // Reset to the inherited base depth (0 for a top-level run/REPL; the - // parent's live depth for an `execute file` child) so recursion - // accounting spans the execute-file boundary instead of granting the - // child a fresh full allowance. - self.call_depth.set(self.base_call_depth); - // NOTE: do NOT touch the shared budget's main-loop depth here. It is - // managed entirely by the RAII `MainLoopGuard`, so it never leaks (a - // mid-loop unwind drops the guard); and for an `execute file` child that - // shares the parent's budget, clearing it would wrongly cancel the - // parent's still-active main-loop exemption. - self.assert_invariants(); - // Names that will become overload sets in the top-level block enforce - // their declared types from the first definition on. Nested blocks - // get their own scan at block entry (`BlockDupsScope`), so - // enforcement stays scoped to the block that actually overloads. - // Same-scope members an incoming definition will merge with (a REPL - // interpreter reused across snippets) are armed the same way — and - // reverted on run exit if this run never executed the merge. - let _armed_members = Self::arm_block_members(&program.statements, &self.global_env); - *self.current_block_overload_dups.borrow_mut() = - Self::scan_block_overload_dups(&program.statements); - self.call_stack.borrow_mut().clear(); - - // Set up script arguments in the global environment - { - let mut env = self.global_env.borrow_mut(); + /// A response request operand can itself be asynchronous, so its target id + /// is not available until evaluation finishes. Watch only the pending + /// requests owned when evaluation starts; requests accepted by nested work + /// are cleaned as newly-created resources if this evaluation is cancelled. + async fn first_pending_response_disconnected(&self, request_ids: &[String]) -> String { + loop { + if let Some(request_id) = self.first_pending_response_disconnected_now(request_ids) { + return request_id; + } + let any_still_owned = { + let owned = self.open_pending_requests.borrow(); + request_ids + .iter() + .any(|request_id| owned.iter().any(|id| id == request_id)) + }; + if !any_still_owned { + std::future::pending::<()>().await; + } + tokio::time::sleep(REQUEST_DISCONNECT_POLL_INTERVAL).await; + } + } - // Create args list with all arguments - let args_list: Vec = self - .script_args + fn response_precommit_snapshot(&self) -> ResponsePrecommitSnapshot { + let http_owner = Arc::clone(&self.open_http_streams.borrow()); + let http_streams = http_owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + ResponsePrecommitSnapshot { + call_stack: self.call_stack.borrow().clone(), + call_depth: self.call_depth.get(), + current_count: *self.current_count.borrow(), + in_count_loop: *self.in_count_loop.borrow(), + http_owner, + http_streams, + response_streams: self + .open_response_streams + .borrow() .iter() - .map(|arg| Value::Text(Arc::from(arg.as_str()))) - .collect(); - let _ = env.define("args", Value::List(Rc::new(RefCell::new(args_list)))); - - // Parse and set up flags (arguments starting with - or --) - let mut flags = HashMap::new(); - let mut positional_args = Vec::new(); - let mut i = 0; + .cloned() + .collect(), + pending_requests: self + .open_pending_requests + .borrow() + .iter() + .cloned() + .collect(), + } + } - while i < self.script_args.len() { - let arg = &self.script_args[i]; - if arg.starts_with("--") { - let flag_name = arg.trim_start_matches("--"); - // Check if next argument is a value for this flag - if i + 1 < self.script_args.len() && !self.script_args[i + 1].starts_with("-") { - flags.insert( - flag_name.to_string(), - Value::Text(Arc::from(self.script_args[i + 1].as_str())), - ); - i += 2; - } else { - flags.insert(flag_name.to_string(), Value::Bool(true)); - i += 1; - } - } else if arg.starts_with("-") && arg.len() > 1 { - // Handle short flags like -f - let flag_name = arg.trim_start_matches("-"); - // Check if next argument is a value for this flag - if i + 1 < self.script_args.len() && !self.script_args[i + 1].starts_with("-") { - flags.insert( - flag_name.to_string(), - Value::Text(Arc::from(self.script_args[i + 1].as_str())), - ); - i += 2; - } else { - flags.insert(flag_name.to_string(), Value::Bool(true)); - i += 1; - } - } else { - positional_args.push(Value::Text(Arc::from(arg.as_str()))); - i += 1; - } - } + /// Restore poll-local state and release only resources opened by the + /// cancelled evaluation. Existing handler resources remain owned. + fn cancel_response_precommit(&self, request_id: &str, snapshot: ResponsePrecommitSnapshot) { + *self.call_stack.borrow_mut() = snapshot.call_stack; + self.call_depth.set(snapshot.call_depth); + *self.current_count.borrow_mut() = snapshot.current_count; + *self.in_count_loop.borrow_mut() = snapshot.in_count_loop; - // Convert flags HashMap to Value - let mut flags_map = HashMap::new(); - for (key, value) in flags { - flags_map.insert(key, value); - } + let new_http_streams: Vec = snapshot + .http_owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .iter() + .filter(|id| !snapshot.http_streams.contains(*id)) + .cloned() + .collect(); + self.io_client.close_stream_ids(&new_http_streams); - // Store positional arguments - let _ = env.define( - "positional_args", - Value::List(Rc::new(RefCell::new(positional_args.clone()))), - ); + let new_response_streams: Vec = self + .open_response_streams + .borrow() + .iter() + .filter(|id| !snapshot.response_streams.contains(*id)) + .cloned() + .collect(); + self.close_response_streams(&new_response_streams); + self.open_response_streams + .borrow_mut() + .retain(|id| snapshot.response_streams.contains(id)); - // Store argument count - let _ = env.define("arg_count", Value::Number(self.script_args.len() as f64)); + let new_pending_requests: Vec = self + .open_pending_requests + .borrow() + .iter() + .filter(|id| !snapshot.pending_requests.contains(*id)) + .cloned() + .collect(); + self.fail_unanswered_requests(&new_pending_requests); + self.open_pending_requests + .borrow_mut() + .retain(|id| id != request_id && snapshot.pending_requests.contains(id)); + self.pending_responses.borrow_mut().remove(request_id); + } - // Store program name (first argument or empty string) - let program_name = if self.script_args.is_empty() { - "wfl".to_string() - } else { - // Extract just the filename from the path - std::path::Path::new(&self.script_args[0]) - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned() - }; - let _ = env.define("program_name", Value::Text(Arc::from(program_name))); + /// Evaluate the response request operand while racing every request this + /// handler already owns. The target request id is not known until the + /// operand resolves, so this establishes the response-attempt snapshot that + /// is carried through field evaluation and commit. + async fn evaluate_response_request( + &self, + line: usize, + column: usize, + disconnect_message: &'static str, + evaluation: F, + ) -> Result<(T, ResponsePrecommitSnapshot), RuntimeError> + where + F: std::future::Future>, + { + let snapshot = self.response_precommit_snapshot(); + let mut watched_requests: Vec = snapshot.pending_requests.iter().cloned().collect(); + watched_requests.sort(); + let mut evaluation = Box::pin(evaluation); + let mut disconnected = + Box::pin(self.first_pending_response_disconnected(&watched_requests)); + let outcome = tokio::select! { + biased; + request_id = disconnected.as_mut() => (None, Some(request_id)), + result = evaluation.as_mut() => (Some(result), None), + }; + let disconnected_request = outcome + .1 + .or_else(|| self.first_pending_response_disconnected_now(&watched_requests)); - // Store current directory - let current_dir = std::env::current_dir() - .unwrap_or_default() - .to_string_lossy() - .into_owned(); - let _ = env.define("current_directory", Value::Text(Arc::from(current_dir))); + if let Some(result) = outcome.0 + && disconnected_request.is_none() + { + drop(evaluation); + drop(disconnected); + return result.map(|value| (value, snapshot)); + } - // Store the running script's absolute path and directory, - // the equivalent of Python's __file__ / dirname(abspath(__file__)). - // Always absolute or empty: empty when no script file is running, - // or when a relative script path can't be resolved because the - // current directory is unavailable. - let (script_path, script_directory) = match self.current_source_file.borrow().as_ref() { - Some(source_file) => { - let cwd = if source_file.is_absolute() { - // lexical_abspath ignores cwd for absolute paths - Some(PathBuf::new()) - } else { - std::env::current_dir().ok() - }; - match cwd { - Some(cwd) => { - let abs = lexical_abspath(source_file, &cwd); - let dir = std::path::Path::new(&abs) - .parent() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(); - (abs, dir) - } - None => (String::new(), String::new()), - } - } - None => (String::new(), String::new()), - }; - let _ = env.define("script_path", Value::Text(Arc::from(script_path))); - let _ = env.define("script_directory", Value::Text(Arc::from(script_directory))); + // Drop first so action/loop RAII runs before restoring the baseline. + drop(evaluation); + drop(disconnected); + let request_id = + disconnected_request.expect("disconnect watcher completed without a request id"); + self.cancel_response_precommit(&request_id, snapshot); + Err(RuntimeError::with_kind( + disconnect_message.to_string(), + line, + column, + ErrorKind::Cancelled, + )) + } - // Store flags as individual variables with flag_ prefix - for (key, value) in flags_map { - let _ = env.define(&format!("flag_{key}"), value); + /// Evaluate every fallible response field while racing the target request's + /// disconnect signal. Disconnect wins ties, and the evaluation future is + /// dropped before partially-entered interpreter state is restored. + async fn evaluate_response_precommit( + &self, + request_id: &str, + line: usize, + column: usize, + disconnect_message: &'static str, + snapshot: ResponsePrecommitSnapshot, + evaluation: F, + ) -> Result<(T, ResponsePrecommitSnapshot), RuntimeError> + where + F: std::future::Future>, + { + if let Err(error) = self + .ensure_pending_response_owned(request_id, line, column) + .await + { + if error.kind == ErrorKind::Cancelled { + self.cancel_response_precommit(request_id, snapshot); } + return Err(error); } + let mut evaluation = Box::pin(evaluation); + let mut disconnected = Box::pin(self.pending_response_disconnected(request_id)); + let outcome = tokio::select! { + biased; + _ = disconnected.as_mut() => None, + result = evaluation.as_mut() => Some(result), + }; + let disconnected_now = self.pending_response_disconnected_now(request_id); - // Use exec_trace for execution logs instead of println - if !self.step_mode { - exec_trace!( - "Starting script execution with {} statements...", - program.statements.len() - ); + if let Some(result) = outcome + && !disconnected_now + { + drop(evaluation); + drop(disconnected); + return result.map(|value| (value, snapshot)); } - exec_trace!("=== Starting program execution ==="); - let mut last_value = Value::Null; - let mut errors = Vec::new(); + // This ordering is intentional: dropping the future runs RAII guards + // before we overwrite any manually-restored run-state fields. + drop(evaluation); + drop(disconnected); + self.cancel_response_precommit(request_id, snapshot); + Err(RuntimeError::with_kind( + disconnect_message.to_string(), + line, + column, + ErrorKind::Cancelled, + )) + } - #[allow(unused_variables)] - for (i, statement) in program.statements.iter().enumerate() { - if !self.step_mode { - exec_trace!( - "Executing statement {}/{}...", - i + 1, - program.statements.len() - ); + /// Take a pending response sender into an RAII completion guard for + /// `respond` / `start streaming response`. + /// + /// Ownership is checked against `open_pending_requests` *before* clearing the + /// id: a sibling `wait for request` may globally prune a closed (client- + /// disconnected) sender, so the map entry is missing while this handler still + /// owns the request. That path is `ErrorKind::Cancelled`. A missing entry when + /// the handler no longer owns the id (duplicate respond after a successful + /// one, or a forged request id) remains a general error. + /// Check that this handler still owns `request_id` and a pending entry is + /// present (or was pruned as disconnected → Cancelled), WITHOUT taking the + /// sender. Used before evaluating respond/stream-head expressions so the + /// parked sender remains a disconnect signal for upstream work. + async fn ensure_pending_response_owned( + &self, + request_id: &str, + line: usize, + column: usize, + ) -> Result<(), RuntimeError> { + let was_owned = self + .open_pending_requests + .borrow() + .iter() + .any(|id| id == request_id); + if !was_owned { + return Err(RuntimeError::new( + "Request ID not found - response may have already been sent".to_string(), + line, + column, + )); + } + let present = self.pending_responses.borrow().contains_key(request_id); + if !present { + // Sibling prune of a closed sender while we still own the request. + return Err(RuntimeError::with_kind( + "Client disconnected before the response was sent".to_string(), + line, + column, + ErrorKind::Cancelled, + )); + } + // Optionally check is_closed early for a clearer cancel before heavy eval. + let closed = { + let pending = self.pending_responses.borrow(); + match pending.get(request_id) { + Some(p) => match p.sender.try_lock() { + Ok(g) => g.as_ref().is_none_or(|s| s.is_closed()), + Err(_) => false, + }, + None => true, } - exec_trace!("Executing statement {}/{}", i + 1, program.statements.len()); + }; + if closed { + return Err(RuntimeError::with_kind( + "Client disconnected before the response was sent".to_string(), + line, + column, + ErrorKind::Cancelled, + )); + } + Ok(()) + } - if let Err(err) = self.check_time() { - if !self.step_mode { - exec_trace!( - "Timeout reached at statement {}/{}", - i + 1, - program.statements.len() - ); - } - errors.push(err); - return Err(errors); + async fn take_pending_response_completion( + &self, + request_id: &str, + line: usize, + column: usize, + ) -> Result { + let was_owned = self + .open_pending_requests + .borrow() + .iter() + .any(|id| id == request_id); + let pending_entry = { + let mut pending = self.pending_responses.borrow_mut(); + pending.remove(request_id) + }; + // Answered (or definitively cancelled) now: drop it from the handler's + // unanswered-request tracking so the exit-time 500 fallback skips it. + self.open_pending_requests + .borrow_mut() + .retain(|id| id != request_id); + match pending_entry { + // The admission slot is released by the transport task when it + // finishes delivering this response (or on its timeout), so the + // completion guard carries only the response channel. + Some(entry) => match entry.sender.lock().await.take() { + Some(sender) => Ok(ResponseCompletion { + sender: Some(sender), + }), + None => Err(RuntimeError::new( + "Response already sent for this request".to_string(), + line, + column, + )), + }, + None if was_owned => { + // Sibling prune removed a closed sender, or the entry otherwise + // vanished while this handler still owned the request — treat as + // client disconnect / cooperative cancellation. + Err(RuntimeError::with_kind( + "Client disconnected before the response was sent".to_string(), + line, + column, + ErrorKind::Cancelled, + )) } + None => Err(RuntimeError::new( + "Request ID not found - response may have already been sent".to_string(), + line, + column, + )), + } + } - match self - .execute_statement(statement, Rc::clone(&self.global_env)) - .await + /// Answer 500 for each request id in `ids` that is still unanswered (its + /// sender is still parked in `pending_responses`). A request the handler + /// already answered is gone from the map, so its `remove` is a no-op — + /// idempotent and safe to call over a handler's full dequeued list on exit. + /// + /// Fully synchronous (a non-blocking `try_lock` plus a `oneshot` send) so it + /// runs from `IsolatedHandler`'s `Drop`. The sender's mutex is only ever held + /// during `respond`, which the finished handler is no longer inside, so the + /// `try_lock` succeeds; if it somehow does not, the transport's request + /// timeout remains the backstop. + fn fail_unanswered_requests(&self, ids: &[String]) { + if ids.is_empty() { + return; + } + let mut pending = self.pending_responses.borrow_mut(); + for id in ids { + if let Some(entry) = pending.remove(id) + && let Ok(mut guard) = entry.sender.try_lock() + && let Some(sender) = guard.take() { - Ok((value, control_flow)) => { + let _ = sender.send(HandlerReply::Buffered(WflHttpResponse { + content: b"Internal Server Error\n".to_vec(), + status: 500, + content_type: "text/plain; charset=utf-8".to_string(), + headers: HashMap::new(), + })); + } + } + } + + /// Drain and 500 any requests the current (serial) handler dequeued but never + /// answered. Called at the end of each serial `main loop` iteration and at + /// program exit, mirroring the concurrent path's per-handler `Drop`. + fn fail_open_pending_requests(&self) { + let ids = std::mem::take(&mut *self.open_pending_requests.borrow_mut()); + self.fail_unanswered_requests(&ids); + } + + /// Execute a `main loop concurrently:` body. Keeps up to + /// `CONCURRENT_HANDLER_LIMIT` iterations of `body` in flight at once, each in + /// its own isolated child scope, driven cooperatively on this single thread + /// (no threads, no `Send`/`Arc` across the interpreter core). A handler that + /// errors or panics is contained — its request is resolved with 500 by the + /// response-completion drop guard — and its siblings keep running. + /// + /// Each handler also carries an isolated [`RunState`] (count-loop variable, + /// recursion depth, call stack, block overload set) via [`IsolatedHandler`], + /// so one handler's loop/recursion bookkeeping can never leak into another + /// across an `await`. + async fn execute_concurrent_main_loop( + &self, + body: &[Statement], + env: &Rc>, + ) -> Result<(Value, ControlFlow), RuntimeError> { + use futures_util::FutureExt; + use futures_util::stream::{FuturesUnordered, StreamExt}; + + let cap = CONCURRENT_HANDLER_LIMIT.max(1); + let mut futs = FuturesUnordered::new(); + let mut last_value = Value::Null; + + // A handler that fails *before* it ever awaits (a deterministic bad + // expression, or `wait for request` on a server that was closed without + // the loop breaking) completes instantly, so a naive refill-and-repoll + // would spin the CPU (and the log) forever — and this loop is exempt from + // the wall-clock deadline. Count consecutive failures with no successful + // iteration in between: back off between them, and terminate the loop once + // it's clearly a structural failure rather than incidental handler errors. + let mut consecutive_failures: u32 = 0; + + loop { + self.check_time()?; + + // Refill to the concurrency cap. Each iteration gets a fresh isolated + // scope so concurrent requests never clobber each other's variables, + // and a fresh `RunState` (wrapped by `IsolatedHandler`) so their + // count-loop/recursion/call-stack bookkeeping stays poll-local. + while futs.len() < cap { + let scope = Environment::new_child_env(env); + let handler = + std::panic::AssertUnwindSafe(self.execute_block(body, scope)).catch_unwind(); + futs.push(IsolatedHandler { + interp: self, + state: RunState::fresh(self.base_call_depth), + inner: Box::pin(handler), + }); + } + + // With cap >= 1 the set is never empty, so `next()` never returns a + // `Ready(None)` that would busy-spin the loop. + match futs.next().await { + // `IsolatedHandler` yields `(inner, accepted_request)` so we can + // tell request-local outcomes from structural pre-request failures + // after the handler's run state (and open-pending list) is gone. + Some((Ok(Ok((value, flow))), _accepted)) => { + // A completed iteration (request handled) — not a failure. + consecutive_failures = 0; last_value = value; - if !self.step_mode { - exec_trace!( - "Statement {}/{} completed successfully", - i + 1, - program.statements.len() - ); + match flow { + ControlFlow::Break => break, + ControlFlow::Exit => return Ok((last_value, ControlFlow::Exit)), + ControlFlow::Return(val) => { + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::Continue | ControlFlow::None => {} } - - match control_flow { - ControlFlow::Break | ControlFlow::Continue | ControlFlow::Exit => { - exec_trace!("Warning: {:?} at top level ignored", control_flow); + } + // Expected / request-local outcomes must NEVER feed the structural + // consecutive-failure breaker: + // - `Cancelled`: client disconnect (cooperative cancellation) + // - finite `wait for request ... with timeout` expiry ONLY + // (not every ErrorKind::Timeout — e.g. pre-request structural + // timeouts still feed the breaker) + // - any error from a handler that already accepted a request + // + // Structural pre-request failures feed the breaker. + Some((Ok(Err(err)), accepted)) => { + match classify_concurrent_handler_error(&err, accepted) { + ConcurrentHandlerDisposition::RequestLocal => { + log::debug!( + "concurrent main loop: non-structural handler outcome \ + (kind={:?}, accepted_request={accepted}): {err}", + err.kind + ); } - ControlFlow::Return(val) => { - exec_trace!("Return at top level with value: {:?}", val); - last_value = val; - break; + ConcurrentHandlerDisposition::Structural => { + log::warn!("concurrent main loop: structural handler error: {err}"); + if self + .backoff_or_break_concurrent( + &mut consecutive_failures, + MAX_CONSECUTIVE_HANDLER_FAILURES, + ) + .await + { + break; + } } - ControlFlow::None => {} } } - Err(err) => { - if !self.step_mode { - exec_trace!( - "Error at statement {}/{}: {:?}", - i + 1, - program.statements.len(), - err + // Panic after accepting a request is contained and the request is + // answered 500 by the drop guard — request-local, not structural. + // Panic before accepting a request can hot-spin; count it toward + // the structural breaker. + Some((Err(_panic), accepted)) => { + if accepted { + log::warn!( + "concurrent main loop: handler panicked after accepting a request; \ + request answered 500" + ); + } else { + log::warn!( + "concurrent main loop: handler panicked before accepting a request" ); + if self + .backoff_or_break_concurrent( + &mut consecutive_failures, + MAX_CONSECUTIVE_HANDLER_FAILURES, + ) + .await + { + break; + } } - errors.push(err); - break; // Stop on first runtime error } + None => break, // unreachable while cap >= 1; end cleanly if reached } } - if errors.is_empty() { - let main_func_opt = { - match self.global_env.borrow().get("main") { - Some(Value::Function(main_func)) => Some(main_func.clone()), - // An overloaded `main` runs its zero-argument overload. - Some(Value::Overloaded(overloaded)) => overloaded - .overloads - .iter() - .find(|func| func.params.is_empty()) - .cloned(), - _ => None, - } - }; + Ok((last_value, ControlFlow::None)) + } - if let Some(main_func) = main_func_opt { - exec_trace!("Calling main function"); - match self.call_function(&main_func, vec![], 0, 0).await { - Ok(value) => { - exec_trace!("Main function returned: {:?}", value); - last_value = value - } - Err(err) => { - exec_trace!("Main function failed: {}", err); - errors.push(err); - } - } - } + /// Handle a failed concurrent-handler iteration: bump the consecutive-failure + /// counter, back off proportionally (capped) so instant failures cannot hot- + /// spin the CPU/log while the loop is deadline-exempt, and report whether the + /// caller should break the loop because failures have crossed the structural- + /// failure threshold (e.g. the server was closed without the loop breaking). + /// Returns `true` to break. + async fn backoff_or_break_concurrent(&self, consecutive_failures: &mut u32, max: u32) -> bool { + *consecutive_failures += 1; + if *consecutive_failures >= max { + log::error!( + "concurrent main loop: {consecutive_failures} consecutive handler failures with no successful request; stopping the loop to avoid a hot spin" + ); + return true; + } + // Small, capped backoff so a burst of instant failures yields the thread + // (and rate-limits the log) instead of spinning. + let backoff_ms = (*consecutive_failures).min(50) as u64; + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; + false + } - self.assert_invariants(); - Ok(last_value) - } else { - self.assert_invariants(); - Err(errors) + /// Preserve ordinary file I/O failures while classifying byte-ceiling + /// breaches as catchable execution-budget resource errors. + fn file_read_error(&self, error: FileReadError, line: usize, column: usize) -> RuntimeError { + match error { + FileReadError::Io(message) => RuntimeError::new(message, line, column), + FileReadError::Budget(exceeded) => self.budget_error(exceeded, line, column), } } - async fn execute_statement( + /// Map a pattern-VM error onto a `RuntimeError`. Budget breaches (step/state + /// ceilings, cancellation) surface as catchable `ResourceLimit` errors so a + /// ReDoS/cancellation during matching is not silently collapsed into a + /// non-match; structural pattern errors surface as general runtime errors. + fn pattern_error( &self, - stmt: &Statement, - env: Rc>, - ) -> Result<(Value, ControlFlow), RuntimeError> { - #[cfg(debug_assertions)] - exec_trace!("Executing statement: {}", stmt_type(stmt)); - Box::pin(self._execute_statement(stmt, env)).await + err: crate::pattern::PatternError, + line: usize, + column: usize, + ) -> RuntimeError { + use crate::pattern::PatternError; + let kind = match err { + // A pattern that outruns the wall-clock deadline is a timeout, with + // the historic `[Timeout]` kind and message, so existing timeout + // handling/tests keep matching. + PatternError::Timeout { .. } => ErrorKind::Timeout, + PatternError::StepLimitExceeded + | PatternError::StateLimitExceeded + | PatternError::Cancelled => ErrorKind::ResourceLimit, + _ => ErrorKind::General, + }; + RuntimeError::with_kind(err.to_string(), line, column, kind) } - async fn _execute_statement( + /// Read a WFL source file (`load module`, `include from`, `execute file`) + /// under the shared source-size ceiling. Reads at most `max_source_size + 1` + /// bytes, so an oversized file is refused without ever allocating the whole + /// thing — this holds even when the file's metadata is unavailable, stale, + /// or reports `0` (special files), which a metadata-only check would miss. + async fn read_source_bounded( &self, - stmt: &Statement, - env: Rc>, - ) -> Result<(Value, ControlFlow), RuntimeError> { - self.check_time()?; + path: &std::path::Path, + line: usize, + column: usize, + ) -> Result { + use tokio::io::AsyncReadExt; - // Cooperatively yield to the async runtime on a throttled stride so a - // tight CPU-bound loop periodically returns control to the executor, - // letting a `select!` deliver cooperative cancellation (e.g. the REPL's - // Ctrl-C → `budget.cancel()`). Driven by a dedicated per-statement - // counter that advances even inside a `main loop` (whose operation - // counter is exempt and whose body is not guaranteed to await anything - // that returns `Pending`), so a CPU-only main loop still yields. - let sched = self.sched_counter.get().wrapping_add(1); - self.sched_counter.set(sched); - if sched & (COOP_YIELD_STRIDE - 1) == 0 { - tokio::task::yield_now().await; + let io_err = |e: std::io::Error| { + let kind = match e.kind() { + std::io::ErrorKind::NotFound => ErrorKind::FileNotFound, + std::io::ErrorKind::PermissionDenied => ErrorKind::PermissionDenied, + _ => ErrorKind::General, + }; + RuntimeError::with_kind( + format!("Cannot read source file '{}': {e}", path.display()), + line, + column, + kind, + ) + }; + + let max = self.budget.max_source_bytes(); + // Read one byte past the limit so exceeding it is detectable; the buffer + // never grows beyond `max + 1`. + let read_cap = (max as u64).saturating_add(1); + let file = tokio::fs::File::open(path).await.map_err(io_err)?; + let mut buf = Vec::new(); + file.take(read_cap) + .read_to_end(&mut buf) + .await + .map_err(io_err)?; + + if let Err(exceeded) = self.budget.check_source_bytes(buf.len()) { + return Err(self.budget_error(exceeded, line, column)); } - let env_before = if self.step_mode { - self.global_env.borrow().values.clone() + String::from_utf8(buf).map_err(|_| { + RuntimeError::new( + format!("Source file '{}' is not valid UTF-8", path.display()), + line, + column, + ) + }) + } + + fn assert_invariants(&self) { + debug_assert_eq!( + *self.in_count_loop.borrow(), + self.current_count.borrow().is_some() + ); + + debug_assert!(self.call_stack.borrow().len() < 10_000); + } + + fn native_display(args: Vec) -> Result { + let mut line = String::new(); + for (i, arg) in args.iter().enumerate() { + if i > 0 { + line.push(' '); + } + line.push_str(&arg.to_string()); + } + io_capture::emit_line(&line); + Ok(Value::Null) + } + + pub async fn interpret(&mut self, program: &Program) -> Result> { + // Scope this run's budget as the TASK-local current budget, so leaf + // helpers with no budget parameter (the stdlib pattern builtins in + // particular) match under the run's configured ceilings and shared + // meters — and, crucially, so a library embedder that interleaves two + // interpreter futures on one thread never sees the other's budget or + // restores stale state across an `.await`. An `execute file` child that + // calls `interpret` again nests its own scope (same or child budget). + ExecutionBudget::scope(Arc::clone(&self.budget), self.interpret_inner(program)).await + } + + /// Action names defined more than once in `statements` — the immediate + /// slice only, matching the same-scope overloading rule: two same-name + /// definitions in different blocks are independent actions, not + /// overloads. Returns `None` (no allocation kept) when nothing in the + /// slice is overloaded. + fn scan_block_overload_dups( + statements: &[Statement], + ) -> Option>> { + let mut counts: HashMap<&str, usize> = HashMap::new(); + let mut dups: std::collections::HashSet = std::collections::HashSet::new(); + for statement in statements { + if let Statement::ActionDefinition { name, .. } = statement { + let entry = counts.entry(name.as_str()).or_insert(0); + *entry += 1; + if *entry == 2 { + dups.insert(name.clone()); + } + } + } + if dups.is_empty() { + None } else { - HashMap::new() - }; + Some(Rc::new(dups)) + } + } + + /// Speculatively arms `enforce_param_types` on any same-scope function + /// that `statements` will merge a new definition into — the merge makes + /// that member overloaded, so its temporal window starts when the block + /// starts executing, not at the later merge. Inherited (parent-scope) + /// names are not mergeable — defining over them is a shadowing error — + /// so only the local scope is consulted. The returned guard reverts the + /// arming on drop for members whose merge never actually executed. + fn arm_block_members( + statements: &[Statement], + env: &Rc>, + ) -> ArmedEnforcementGuard { + let mut armed = Vec::new(); + for statement in statements { + if let Statement::ActionDefinition { name, .. } = statement + && let Some(Value::Function(existing)) = env.borrow().get_local(name) + { + let prior_enforce = existing.enforce_param_types.get(); + existing.enforce_param_types.set(true); + armed.push(ArmedMember { + env: Rc::clone(env), + name: name.clone(), + member: existing, + prior_enforce, + }); + } + } + ArmedEnforcementGuard { armed } + } + + /// Enters a statement block for overload enforcement: computes the + /// block's own duplicate set and speculatively arms mergeable members + /// (see [`Self::arm_block_members`]). Both guards restore on drop. + fn enter_block_overloads( + &self, + statements: &[Statement], + env: &Rc>, + ) -> (BlockDupsScope<'_>, ArmedEnforcementGuard) { + let armed = Self::arm_block_members(statements, env); + ( + BlockDupsScope::enter( + &self.current_block_overload_dups, + Self::scan_block_overload_dups(statements), + ), + armed, + ) + } + + /// The interpreter run body, executed inside the task-local budget scope + /// established by [`Interpreter::interpret`]. + async fn interpret_inner(&mut self, program: &Program) -> Result> { + // Reset per-run enforcement/loop state first, so a prior *terminal* + // budget breach (e.g. an uncaught timeout that unwound to the top) can't + // leak stale count-loop or depth state into this run — matters when one + // interpreter is reused (the REPL). Done before `assert_invariants` so + // the invariant holds regardless of how the previous run ended. + *self.in_count_loop.borrow_mut() = false; + *self.current_count.borrow_mut() = None; + // A prior run that ended while a stream was open or a request was + // unanswered (REPL reuse) must not leave dangling entries. Actually CLOSE + // those streams and 500 those requests (draining the tracking), rather + // than only clearing the id lists — clearing alone would strand the + // still-open sender/receiver in `server_response_streams` / + // `pending_responses`, hanging the client and leaking the entry. + self.close_open_response_streams(); + self.fail_open_pending_requests(); + self.close_open_http_streams(); + // RAII: if THIS run's future is dropped/cancelled before its normal exit + // sites run, still close any outbound handles it opened (they would + // otherwise leak the upstream until the interpreter itself is dropped). + // On a normal run the exit sites drain the list first, so this is a no-op. + let _outbound_cleanup = self.outbound_stream_cleanup_guard(); + // Reset to the inherited base depth (0 for a top-level run/REPL; the + // parent's live depth for an `execute file` child) so recursion + // accounting spans the execute-file boundary instead of granting the + // child a fresh full allowance. + self.call_depth.set(self.base_call_depth); + // NOTE: do NOT touch the shared budget's main-loop depth here. It is + // managed entirely by the RAII `MainLoopGuard`, so it never leaks (a + // mid-loop unwind drops the guard); and for an `execute file` child that + // shares the parent's budget, clearing it would wrongly cancel the + // parent's still-active main-loop exemption. + self.assert_invariants(); + // Names that will become overload sets in the top-level block enforce + // their declared types from the first definition on. Nested blocks + // get their own scan at block entry (`BlockDupsScope`), so + // enforcement stays scoped to the block that actually overloads. + // Same-scope members an incoming definition will merge with (a REPL + // interpreter reused across snippets) are armed the same way — and + // reverted on run exit if this run never executed the merge. + let _armed_members = Self::arm_block_members(&program.statements, &self.global_env); + *self.current_block_overload_dups.borrow_mut() = + Self::scan_block_overload_dups(&program.statements); + self.call_stack.borrow_mut().clear(); + + // Set up script arguments in the global environment + { + let mut env = self.global_env.borrow_mut(); + + // Create args list with all arguments + let args_list: Vec = self + .script_args + .iter() + .map(|arg| Value::Text(Arc::from(arg.as_str()))) + .collect(); + let _ = env.define("args", Value::List(Rc::new(RefCell::new(args_list)))); + + // Parse and set up flags (arguments starting with - or --) + let mut flags = HashMap::new(); + let mut positional_args = Vec::new(); + let mut i = 0; + + while i < self.script_args.len() { + let arg = &self.script_args[i]; + if arg.starts_with("--") { + let flag_name = arg.trim_start_matches("--"); + // Check if next argument is a value for this flag + if i + 1 < self.script_args.len() && !self.script_args[i + 1].starts_with("-") { + flags.insert( + flag_name.to_string(), + Value::Text(Arc::from(self.script_args[i + 1].as_str())), + ); + i += 2; + } else { + flags.insert(flag_name.to_string(), Value::Bool(true)); + i += 1; + } + } else if arg.starts_with("-") && arg.len() > 1 { + // Handle short flags like -f + let flag_name = arg.trim_start_matches("-"); + // Check if next argument is a value for this flag + if i + 1 < self.script_args.len() && !self.script_args[i + 1].starts_with("-") { + flags.insert( + flag_name.to_string(), + Value::Text(Arc::from(self.script_args[i + 1].as_str())), + ); + i += 2; + } else { + flags.insert(flag_name.to_string(), Value::Bool(true)); + i += 1; + } + } else { + positional_args.push(Value::Text(Arc::from(arg.as_str()))); + i += 1; + } + } + + // Convert flags HashMap to Value + let mut flags_map = HashMap::new(); + for (key, value) in flags { + flags_map.insert(key, value); + } + + // Store positional arguments + let _ = env.define( + "positional_args", + Value::List(Rc::new(RefCell::new(positional_args.clone()))), + ); + + // Store argument count + let _ = env.define("arg_count", Value::Number(self.script_args.len() as f64)); + + // Store program name (first argument or empty string) + let program_name = if self.script_args.is_empty() { + "wfl".to_string() + } else { + // Extract just the filename from the path + std::path::Path::new(&self.script_args[0]) + .file_name() + .unwrap_or_default() + .to_string_lossy() + .into_owned() + }; + let _ = env.define("program_name", Value::Text(Arc::from(program_name))); + + // Store current directory + let current_dir = std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + let _ = env.define("current_directory", Value::Text(Arc::from(current_dir))); + + // Store the running script's absolute path and directory, + // the equivalent of Python's __file__ / dirname(abspath(__file__)). + // Always absolute or empty: empty when no script file is running, + // or when a relative script path can't be resolved because the + // current directory is unavailable. + let (script_path, script_directory) = match self.current_source_file.borrow().as_ref() { + Some(source_file) => { + let cwd = if source_file.is_absolute() { + // lexical_abspath ignores cwd for absolute paths + Some(PathBuf::new()) + } else { + std::env::current_dir().ok() + }; + match cwd { + Some(cwd) => { + let abs = lexical_abspath(source_file, &cwd); + let dir = std::path::Path::new(&abs) + .parent() + .map(|p| p.to_string_lossy().into_owned()) + .unwrap_or_default(); + (abs, dir) + } + None => (String::new(), String::new()), + } + } + None => (String::new(), String::new()), + }; + let _ = env.define("script_path", Value::Text(Arc::from(script_path))); + let _ = env.define("script_directory", Value::Text(Arc::from(script_directory))); + + // Store flags as individual variables with flag_ prefix + for (key, value) in flags_map { + let _ = env.define(&format!("flag_{key}"), value); + } + } + + // Use exec_trace for execution logs instead of println + if !self.step_mode { + exec_trace!( + "Starting script execution with {} statements...", + program.statements.len() + ); + } + exec_trace!("=== Starting program execution ==="); + + let mut last_value = Value::Null; + let mut errors = Vec::new(); + + #[allow(unused_variables)] + for (i, statement) in program.statements.iter().enumerate() { + if !self.step_mode { + exec_trace!( + "Executing statement {}/{}...", + i + 1, + program.statements.len() + ); + } + exec_trace!("Executing statement {}/{}", i + 1, program.statements.len()); + + if let Err(err) = self.check_time() { + if !self.step_mode { + exec_trace!( + "Timeout reached at statement {}/{}", + i + 1, + program.statements.len() + ); + } + errors.push(err); + // A mid-run timeout is still an exit path: finalize any open + // top-level streams and unanswered requests before returning. + self.close_open_response_streams(); + self.fail_open_pending_requests(); + self.close_open_http_streams(); + return Err(errors); + } + + match self + .execute_statement(statement, Rc::clone(&self.global_env)) + .await + { + Ok((value, control_flow)) => { + last_value = value; + if !self.step_mode { + exec_trace!( + "Statement {}/{} completed successfully", + i + 1, + program.statements.len() + ); + } + + match control_flow { + ControlFlow::Break | ControlFlow::Continue | ControlFlow::Exit => { + exec_trace!("Warning: {:?} at top level ignored", control_flow); + } + ControlFlow::Return(val) => { + exec_trace!("Return at top level with value: {:?}", val); + last_value = val; + break; + } + ControlFlow::None => {} + } + } + Err(err) => { + if !self.step_mode { + exec_trace!( + "Error at statement {}/{}: {:?}", + i + 1, + program.statements.len(), + err + ); + } + errors.push(err); + break; // Stop on first runtime error + } + } + } + + // Run the conventional `main` action (if any) before cleanup, so a + // stream/request opened by `main` is finalized by the drain below rather + // than leaking. + if errors.is_empty() { + let main_func_opt = { + match self.global_env.borrow().get("main") { + Some(Value::Function(main_func)) => Some(main_func.clone()), + // An overloaded `main` runs its zero-argument overload. + Some(Value::Overloaded(overloaded)) => overloaded + .overloads + .iter() + .find(|func| func.params.is_empty()) + .cloned(), + _ => None, + } + }; + + if let Some(main_func) = main_func_opt { + exec_trace!("Calling main function"); + match self.call_function(&main_func, vec![], 0, 0).await { + Ok(value) => { + exec_trace!("Main function returned: {:?}", value); + last_value = value + } + Err(err) => { + exec_trace!("Main function failed: {}", err); + errors.push(err); + } + } + } + } + + // Close any server response streams opened directly at top level or by + // `main` (outside a `main loop`, which already closes + // per-iteration/per-handler), and 500 any top-level request dequeued but + // never answered — on EVERY exit path (normal end, statement error, or a + // failing `main`), so a script that exits without `close` still finalizes + // the client's body rather than leaving it hanging until process death. + self.close_open_response_streams(); + self.fail_open_pending_requests(); + self.close_open_http_streams(); + + self.assert_invariants(); + if errors.is_empty() { + Ok(last_value) + } else { + Err(errors) + } + } + + async fn execute_statement( + &self, + stmt: &Statement, + env: Rc>, + ) -> Result<(Value, ControlFlow), RuntimeError> { + #[cfg(debug_assertions)] + exec_trace!("Executing statement: {}", stmt_type(stmt)); + Box::pin(self._execute_statement(stmt, env)).await + } + + async fn _execute_statement( + &self, + stmt: &Statement, + env: Rc>, + ) -> Result<(Value, ControlFlow), RuntimeError> { + self.check_time()?; + + // Cooperatively yield to the async runtime on a throttled stride so a + // tight CPU-bound loop periodically returns control to the executor, + // letting a `select!` deliver cooperative cancellation (e.g. the REPL's + // Ctrl-C → `budget.cancel()`). Driven by a dedicated per-statement + // counter that advances even inside a `main loop` (whose operation + // counter is exempt and whose body is not guaranteed to await anything + // that returns `Pending`), so a CPU-only main loop still yields. + let sched = self.sched_counter.get().wrapping_add(1); + self.sched_counter.set(sched); + if sched & (COOP_YIELD_STRIDE - 1) == 0 { + tokio::task::yield_now().await; + } + + let env_before = if self.step_mode { + self.global_env.borrow().values.clone() + } else { + HashMap::new() + }; let (line, column) = match stmt { Statement::VariableDeclaration { line, column, .. } => (*line, *column), @@ -3879,6 +5958,12 @@ impl Interpreter { Statement::HttpGetStatement { line, column, .. } => (*line, *column), Statement::HttpPostStatement { line, column, .. } => (*line, *column), Statement::HttpRequestStatement { line, column, .. } => (*line, *column), + Statement::HttpStreamStatement { line, column, .. } => (*line, *column), + Statement::WaitForNextChunkStatement { line, column, .. } => (*line, *column), + Statement::WaitForNextLineStatement { line, column, .. } => (*line, *column), + Statement::StartStreamingResponseStatement { line, column, .. } => (*line, *column), + Statement::StreamWriteStatement { line, column, .. } => (*line, *column), + Statement::FlushStreamStatement { line, column, .. } => (*line, *column), Statement::PushStatement { line, column, .. } => (*line, *column), Statement::CreateListStatement { line, column, .. } => (*line, *column), Statement::MapCreation { line, column, .. } => (*line, *column), @@ -3918,3234 +6003,4249 @@ impl Interpreter { Statement::ExpectStatement { line, column, .. } => (*line, *column), }; - let result = match stmt { - Statement::VariableDeclaration { - name, - value, - is_constant, + let result = match stmt { + Statement::VariableDeclaration { + name, + value, + is_constant, + line: _line, + column: _column, + } => { + let evaluated_value = self.evaluate_expression(value, Rc::clone(&env)).await?; + + #[cfg(debug_assertions)] + exec_var_declare!(name, &evaluated_value); + + // OPTIMIZATION: declare_variable handles scope traversal, shadowing detection, + // and definition/assignment in a single pass. + match env + .borrow_mut() + .declare_variable(name, evaluated_value, *is_constant) + { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, line, column)), + } + } + + Statement::Assignment { + name, + value, + line, + column, + } => { + let value = self.evaluate_expression(value, Rc::clone(&env)).await?; + #[cfg(debug_assertions)] + exec_var_assign!(name, &value); + match env.borrow_mut().assign(name, value.clone()) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + + Statement::IfStatement { + condition, + then_block, + else_block, + line: _line, + column: _column, + } => { + let condition_value = self.evaluate_expression(condition, Rc::clone(&env)).await?; + #[cfg(debug_assertions)] + exec_control_flow!("if condition", condition_value.is_truthy()); + + if condition_value.is_truthy() { + #[cfg(debug_assertions)] + let _guard = IndentGuard::new(); + #[cfg(debug_assertions)] + exec_block_enter!("if branch"); + let result = self.execute_block(then_block, Rc::clone(&env)).await; + #[cfg(debug_assertions)] + exec_block_exit!("if branch"); + result + } else if let Some(else_stmts) = else_block { + #[cfg(debug_assertions)] + let _guard = IndentGuard::new(); + #[cfg(debug_assertions)] + exec_block_enter!("else branch"); + let result = self.execute_block(else_stmts, Rc::clone(&env)).await; + #[cfg(debug_assertions)] + exec_block_exit!("else branch"); + result + } else { + Ok((Value::Null, ControlFlow::None)) + } + } + + Statement::SingleLineIf { + condition, + then_stmt, + else_stmt, + line: _line, + column: _column, + } => { + let condition_value = self.evaluate_expression(condition, Rc::clone(&env)).await?; + + if condition_value.is_truthy() { + self.execute_statement(then_stmt, Rc::clone(&env)).await + } else if let Some(else_stmt) = else_stmt { + self.execute_statement(else_stmt, Rc::clone(&env)).await + } else { + Ok((Value::Null, ControlFlow::None)) + } + } + + Statement::DisplayStatement { + value, + line: _line, + column: _column, + } => { + let value = self.evaluate_expression(value, Rc::clone(&env)).await?; + io_capture::emit_line(&value.to_string()); + Ok((Value::Null, ControlFlow::None)) + } + + Statement::ActionDefinition { + name, + parameters, + body, + return_type: _return_type, + line, + column, + } => { + let param_names: Vec = parameters.iter().map(|p| p.name.clone()).collect(); + let param_types: Vec> = + parameters.iter().map(|p| p.param_type.clone()).collect(); + + let function = FunctionValue { + name: Some(name.clone()), + params: param_names, + param_types, + body: body.clone(), + env: Rc::downgrade(&env), + line: *line, + column: *column, + // Pre-scanned: true when this name has several definitions + // in its block, so even the first member enforces its + // declared types during the window before the second + // definition executes ("the overloads defined so far"). + enforce_param_types: std::cell::Cell::new( + self.current_block_overload_dups + .borrow() + .as_ref() + .is_some_and(|dups| dups.contains(name)), + ), + }; + + // A same-scope redefinition of an action name merges into an + // overload set instead of erroring; every other collision + // keeps its existing error. + match env + .borrow_mut() + .define_or_merge_action(name, Rc::new(function)) + { + Ok(defined_value) => Ok((defined_value, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + + Statement::ReturnStatement { + value, + line: _line, + column: _column, + } => { + #[cfg(debug_assertions)] + exec_trace!("Executing return statement"); + + if let Some(expr) = value { + let result = self.evaluate_expression(expr, Rc::clone(&env)).await?; + Ok((result.clone(), ControlFlow::Return(result))) + } else { + Ok((Value::Null, ControlFlow::Return(Value::Null))) + } + } + + Statement::ExpressionStatement { + expression, + line: _line, + column: _column, + } => { + // Check if this is a bare action call (just the action name without parentheses) + if let Expression::Variable(name, var_line, var_column) = expression { + // Check if the variable refers to an action + // Extract lookup result so the Ref is dropped before call_function + let lookup = env.borrow().get(name); + if let Some(Value::Function(func)) = lookup { + // It's an action, so execute it as a call with no arguments + #[cfg(debug_assertions)] + exec_trace!("Executing bare action call: {}", name); + return self + .call_function(&func, vec![], *var_line, *var_column) + .await + .map(|value| (value, ControlFlow::None)); + } else if let Some(Value::Overloaded(overloaded)) = lookup { + // A bare overloaded name auto-calls its zero-argument + // overload, matching single-function behavior. + if let Some(func) = overloaded + .overloads + .iter() + .find(|func| func.params.is_empty()) + { + #[cfg(debug_assertions)] + exec_trace!("Executing bare overloaded action call: {}", name); + return self + .call_function(func, vec![], *var_line, *var_column) + .await + .map(|value| (value, ControlFlow::None)); + } + } + } + + // Regular expression evaluation + let value = self + .evaluate_expression(expression, Rc::clone(&env)) + .await?; + Ok((value, ControlFlow::None)) + } + + Statement::CountLoop { + start, + end, + step, + downward, + variable_name, + body, + line, + column, + } => { + // === CRITICAL FIX: Reset count loop state before starting === + let previous_count = *self.current_count.borrow(); + let was_in_count_loop = *self.in_count_loop.borrow(); + + // Force reset state to prevent inheriting stale values + *self.current_count.borrow_mut() = None; + *self.in_count_loop.borrow_mut() = false; + + crate::exec_trace!("Count loop: resetting state before evaluation"); + + let start_val = self.evaluate_expression(start, Rc::clone(&env)).await?; + let end_val = self.evaluate_expression(end, Rc::clone(&env)).await?; + + let (start_num, end_num) = match (start_val, end_val) { + (Value::Number(s), Value::Number(e)) => (s, e), + _ => { + return Err(RuntimeError::new( + "Count loop requires numeric start and end values".to_string(), + *line, + *column, + )); + } + }; + + let step_num = if let Some(step_expr) = step { + match self.evaluate_expression(step_expr, Rc::clone(&env)).await? { + Value::Number(n) => n, + _ => { + return Err(RuntimeError::new( + "Count loop step must be a number".to_string(), + *line, + *column, + )); + } + } + } else { + 1.0 + }; + + let mut count = start_num; + + let should_continue: Box bool> = if *downward { + Box::new(|count, end_num| count >= end_num) + } else { + Box::new(|count, end_num| count <= end_num) + }; + + let max_iterations = if end_num > 1000000.0 { + u64::MAX // Effectively no limit for large end values, rely on timeout instead + } else { + // Allow up to 10001 iterations to accommodate loops that need exactly 10000 + // (e.g., "count from 1 to 10000" requires 10000 iterations) + 10001 + }; + let mut iterations = 0; + + *self.in_count_loop.borrow_mut() = true; + + // Determine the variable name to use - custom name or default "count" + let loop_var_name = variable_name.as_deref().unwrap_or("count"); + let mut loop_env_recycle = None; + + while should_continue(count, end_num) && iterations < max_iterations { + self.check_time()?; + + *self.current_count.borrow_mut() = Some(count); + + // OPTIMIZATION: Recycle environment if possible + let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); + + // Make the loop variable available in the loop environment, + // shadowing any same-named variable from an outer scope. + // Use custom variable name if provided, otherwise default to "count" + loop_env + .borrow_mut() + .define_or_replace(loop_var_name, Value::Number(count)); + + let result = self.execute_block(body, Rc::clone(&loop_env)).await; + + // Save environment for potential recycling in next iteration + loop_env_recycle = Some(loop_env); + + match result { + Ok((_, control_flow)) => match control_flow { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of count loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing count loop"); + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from count loop"); + *self.current_count.borrow_mut() = previous_count; + *self.in_count_loop.borrow_mut() = was_in_count_loop; + return Ok((Value::Null, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from count loop with value: {:?}", val); + *self.current_count.borrow_mut() = previous_count; + *self.in_count_loop.borrow_mut() = was_in_count_loop; + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + }, + Err(e) => { + *self.current_count.borrow_mut() = previous_count; + *self.in_count_loop.borrow_mut() = was_in_count_loop; + return Err(e); + } + } + + if *downward { + count -= step_num; + } else { + count += step_num; + } + + iterations += 1; + } + + *self.current_count.borrow_mut() = previous_count; + *self.in_count_loop.borrow_mut() = was_in_count_loop; + + if iterations >= max_iterations { + return Err(RuntimeError::new( + format!("Count loop exceeded maximum iterations ({max_iterations})"), + *line, + *column, + )); + } + + Ok((Value::Null, ControlFlow::None)) + } + + Statement::ForEachLoop { + item_name, + collection, + reversed, + body, + line, + column, + } => { + let collection_val = self + .evaluate_expression(collection, Rc::clone(&env)) + .await?; + + match collection_val { + Value::List(list_rc) => { + let items: Vec = { + let list = list_rc.borrow(); + let indices: Vec = if *reversed { + (0..list.len()).rev().collect() + } else { + (0..list.len()).collect() + }; + indices.iter().map(|&i| list[i].clone()).collect() + }; + + let mut loop_env_recycle = None; + for item in items { + // OPTIMIZATION: Recycle environment if possible + let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); + + match loop_env.borrow_mut().define(item_name, item) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } + let result = self.execute_block(body, Rc::clone(&loop_env)).await?; + + // Save environment for potential recycling in next iteration + loop_env_recycle = Some(loop_env); + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of foreach loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing foreach loop"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from foreach loop"); + return Ok((Value::Null, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!( + "Returning from foreach loop with value: {:?}", + val + ); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } + } + } + Value::Object(obj_rc) => { + let items: Vec<(String, Value)> = { + let obj = obj_rc.borrow(); + obj.iter().map(|(k, v)| (k.clone(), v.clone())).collect() + }; + + let mut loop_env_recycle = None; + for (_, value) in items { + // OPTIMIZATION: Recycle environment if possible + let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); + + match loop_env.borrow_mut().define(item_name, value) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } + let result = self.execute_block(body, Rc::clone(&loop_env)).await?; + + // Save environment for potential recycling in next iteration + loop_env_recycle = Some(loop_env); + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of foreach loop (object)"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing foreach loop (object)"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from foreach loop (object)"); + return Ok((Value::Null, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!( + "Returning from foreach loop with value: {:?}", + val + ); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } + } + } + _ => { + return Err(RuntimeError::new( + format!("Cannot iterate over {}", collection_val.type_name()), + *line, + *column, + )); + } + } + + Ok((Value::Null, ControlFlow::None)) + } + + Statement::WhileLoop { + condition, + body, line: _line, column: _column, } => { - let evaluated_value = self.evaluate_expression(value, Rc::clone(&env)).await?; + let mut _last_value = Value::Null; - #[cfg(debug_assertions)] - exec_var_declare!(name, &evaluated_value); + while self + .evaluate_expression(condition, Rc::clone(&env)) + .await? + .is_truthy() + { + self.check_time()?; + let result = self.execute_block(body, Rc::clone(&env)).await?; + _last_value = result.0; + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of while loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing while loop"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from while loop"); + return Ok((_last_value, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from while loop with value: {:?}", val); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } + } + + Ok((_last_value, ControlFlow::None)) + } + + Statement::RepeatUntilLoop { + condition, + body, + line: _line, + column: _column, + } => { + let mut _last_value = Value::Null; + + loop { + self.check_time()?; + let result = self.execute_block(body, Rc::clone(&env)).await?; + _last_value = result.0; + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of repeat-until loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing repeat-until loop"); + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from repeat-until loop"); + return Ok((_last_value, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from repeat-until loop with value: {:?}", val); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } - // OPTIMIZATION: declare_variable handles scope traversal, shadowing detection, - // and definition/assignment in a single pass. - match env - .borrow_mut() - .declare_variable(name, evaluated_value, *is_constant) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, line, column)), + if self + .evaluate_expression(condition, Rc::clone(&env)) + .await? + .is_truthy() + { + break; + } } - } - Statement::Assignment { - name, - value, - line, - column, - } => { - let value = self.evaluate_expression(value, Rc::clone(&env)).await?; - #[cfg(debug_assertions)] - exec_var_assign!(name, &value); - match env.borrow_mut().assign(name, value.clone()) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } + Ok((_last_value, ControlFlow::None)) } - Statement::IfStatement { - condition, - then_block, - else_block, + Statement::ForeverLoop { + body, line: _line, column: _column, } => { - let condition_value = self.evaluate_expression(condition, Rc::clone(&env)).await?; #[cfg(debug_assertions)] - exec_control_flow!("if condition", condition_value.is_truthy()); + exec_trace!("Executing forever loop"); - if condition_value.is_truthy() { - #[cfg(debug_assertions)] - let _guard = IndentGuard::new(); - #[cfg(debug_assertions)] - exec_block_enter!("if branch"); - let result = self.execute_block(then_block, Rc::clone(&env)).await; - #[cfg(debug_assertions)] - exec_block_exit!("if branch"); - result - } else if let Some(else_stmts) = else_block { - #[cfg(debug_assertions)] - let _guard = IndentGuard::new(); - #[cfg(debug_assertions)] - exec_block_enter!("else branch"); - let result = self.execute_block(else_stmts, Rc::clone(&env)).await; - #[cfg(debug_assertions)] - exec_block_exit!("else branch"); - result - } else { - Ok((Value::Null, ControlFlow::None)) + let mut _last_value = Value::Null; + let mut loop_env_recycle = None; + + loop { + self.check_time()?; + + // OPTIMIZATION: Recycle environment if possible + let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); + let result = self.execute_block(body, Rc::clone(&loop_env)).await?; + _last_value = result.0; + + // Save environment for potential recycling in next iteration + loop_env_recycle = Some(loop_env); + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of forever loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing forever loop"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from forever loop"); + return Ok((_last_value, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from forever loop with value: {:?}", val); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } } + + Ok((_last_value, ControlFlow::None)) } - Statement::SingleLineIf { - condition, - then_stmt, - else_stmt, + Statement::MainLoop { + body, + concurrent, line: _line, column: _column, } => { - let condition_value = self.evaluate_expression(condition, Rc::clone(&env)).await?; + #[cfg(debug_assertions)] + exec_trace!("Executing main loop (timeout disabled)"); - if condition_value.is_truthy() { - self.execute_statement(then_stmt, Rc::clone(&env)).await - } else if let Some(else_stmt) = else_stmt { - self.execute_statement(else_stmt, Rc::clone(&env)).await - } else { - Ok((Value::Null, ControlFlow::None)) + // Enter the main loop's deadline exemption via an RAII guard on + // the shared budget. The guard restores the depth on EVERY exit — + // a normal end, an early `return` below, a caught error unwinding + // through `?`, or a nested main loop — so the exemption is never + // leaked or cleared while an outer loop is still active. A child + // `execute file` sharing this budget inherits the exemption too. + let _main_loop_guard = self.budget.enter_main_loop(); + + // `main loop concurrently:` runs body iterations cooperatively + // concurrently, each in its own isolated scope, so a slow handler + // (e.g. one streaming a slow upstream) does not block its + // siblings. Plain `main loop` stays strictly serial below. + if *concurrent { + return self.execute_concurrent_main_loop(body, &env).await; } - } - Statement::DisplayStatement { - value, - line: _line, - column: _column, - } => { - let value = self.evaluate_expression(value, Rc::clone(&env)).await?; - io_capture::emit_line(&value.to_string()); - Ok((Value::Null, ControlFlow::None)) - } + let mut _last_value = Value::Null; + let mut loop_env_recycle = None; - Statement::ActionDefinition { - name, - parameters, - body, - return_type: _return_type, - line, - column, - } => { - let param_names: Vec = parameters.iter().map(|p| p.name.clone()).collect(); - let param_types: Vec> = - parameters.iter().map(|p| p.param_type.clone()).collect(); + loop { + // check_time() skips the deadline while the main-loop depth > 0 + self.check_time()?; - let function = FunctionValue { - name: Some(name.clone()), - params: param_names, - param_types, - body: body.clone(), - env: Rc::downgrade(&env), - line: *line, - column: *column, - // Pre-scanned: true when this name has several definitions - // in its block, so even the first member enforces its - // declared types during the window before the second - // definition executes ("the overloads defined so far"). - enforce_param_types: std::cell::Cell::new( - self.current_block_overload_dups - .borrow() - .as_ref() - .is_some_and(|dups| dups.contains(name)), - ), - }; + // OPTIMIZATION: Recycle environment if possible + let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); + let result = self.execute_block(body, Rc::clone(&loop_env)).await; + // On every path (including the error path below): close any + // server response streams this iteration left open, and 500 + // any request it dequeued but never answered — so a handler + // that forgets `close out`/`respond` never leaves the client + // hanging or waiting out the request timeout. + self.close_open_response_streams(); + self.fail_open_pending_requests(); + self.close_open_http_streams(); + let result = result?; + _last_value = result.0; - // A same-scope redefinition of an action name merges into an - // overload set instead of erroring; every other collision - // keeps its existing error. - match env - .borrow_mut() - .define_or_merge_action(name, Rc::new(function)) - { - Ok(defined_value) => Ok((defined_value, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + // Save environment for potential recycling in next iteration + loop_env_recycle = Some(loop_env); + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of main loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing main loop"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from main loop"); + // `_main_loop_guard` drops here, restoring the depth. + return Ok((_last_value, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from main loop with value: {:?}", val); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } } + + // `_main_loop_guard` drops here on normal exit. + Ok((_last_value, ControlFlow::None)) } - Statement::ReturnStatement { - value, - line: _line, - column: _column, - } => { + Statement::BreakStatement { .. } => { #[cfg(debug_assertions)] - exec_trace!("Executing return statement"); + exec_trace!("Executing break statement"); + Ok((Value::Null, ControlFlow::Break)) + } - if let Some(expr) = value { - let result = self.evaluate_expression(expr, Rc::clone(&env)).await?; - Ok((result.clone(), ControlFlow::Return(result))) - } else { - Ok((Value::Null, ControlFlow::Return(Value::Null))) - } + Statement::ContinueStatement { .. } => { + #[cfg(debug_assertions)] + exec_trace!("Executing continue statement"); + Ok((Value::Null, ControlFlow::Continue)) + } + + Statement::ExitStatement { .. } => { + #[cfg(debug_assertions)] + exec_trace!("Executing exit statement"); + Ok((Value::Null, ControlFlow::Exit)) } - Statement::ExpressionStatement { - expression, - line: _line, - column: _column, - } => { - // Check if this is a bare action call (just the action name without parentheses) - if let Expression::Variable(name, var_line, var_column) = expression { - // Check if the variable refers to an action - // Extract lookup result so the Ref is dropped before call_function - let lookup = env.borrow().get(name); - if let Some(Value::Function(func)) = lookup { - // It's an action, so execute it as a call with no arguments - #[cfg(debug_assertions)] - exec_trace!("Executing bare action call: {}", name); - return self - .call_function(&func, vec![], *var_line, *var_column) - .await - .map(|value| (value, ControlFlow::None)); - } else if let Some(Value::Overloaded(overloaded)) = lookup { - // A bare overloaded name auto-calls its zero-argument - // overload, matching single-function behavior. - if let Some(func) = overloaded - .overloads - .iter() - .find(|func| func.params.is_empty()) + Statement::OpenFileStatement { + path, + variable_name, + mode, + line, + column, + } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file path, got {path_value:?}"), + *line, + *column, + )); + } + }; + + // Use the appropriate file open mode + match self + .io_client + .open_file_with_mode(&path_str, mode.clone()) + .await + { + Ok(handle) => { + match env + .borrow_mut() + .define(variable_name, Value::Text(handle.into())) { - #[cfg(debug_assertions)] - exec_trace!("Executing bare overloaded action call: {}", name); - return self - .call_function(func, vec![], *var_line, *var_column) - .await - .map(|value| (value, ControlFlow::None)); + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), } } + Err(e) => Err(e), } - - // Regular expression evaluation - let value = self - .evaluate_expression(expression, Rc::clone(&env)) - .await?; - Ok((value, ControlFlow::None)) } - - Statement::CountLoop { - start, - end, - step, - downward, + Statement::OpenDatabaseStatement { + url, variable_name, - body, line, column, } => { - // === CRITICAL FIX: Reset count loop state before starting === - let previous_count = *self.current_count.borrow(); - let was_in_count_loop = *self.in_count_loop.borrow(); - - // Force reset state to prevent inheriting stale values - *self.current_count.borrow_mut() = None; - *self.in_count_loop.borrow_mut() = false; - - crate::exec_trace!("Count loop: resetting state before evaluation"); - - let start_val = self.evaluate_expression(start, Rc::clone(&env)).await?; - let end_val = self.evaluate_expression(end, Rc::clone(&env)).await?; - - let (start_num, end_num) = match (start_val, end_val) { - (Value::Number(s), Value::Number(e)) => (s, e), + let url_value = self.evaluate_expression(url, Rc::clone(&env)).await?; + let url_str = match &url_value { + Value::Text(s) => s.clone(), _ => { return Err(RuntimeError::new( - "Count loop requires numeric start and end values".to_string(), + format!("Expected text for database URL, got {url_value:?}"), *line, *column, )); } }; - let step_num = if let Some(step_expr) = step { - match self.evaluate_expression(step_expr, Rc::clone(&env)).await? { - Value::Number(n) => n, - _ => { - return Err(RuntimeError::new( - "Count loop step must be a number".to_string(), - *line, - *column, - )); + match self.io_client.open_database(&url_str).await { + Ok(handle) => { + let define_result = env + .borrow_mut() + .define(variable_name, Value::Text(handle.as_str().into())); + match define_result { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => { + // Don't leave an unreachable pool behind when + // the variable binding fails. + let _ = self.io_client.close_database(&handle).await; + Err(RuntimeError::new(msg, *line, *column)) + } } } - } else { - 1.0 - }; - - let mut count = start_num; - - let should_continue: Box bool> = if *downward { - Box::new(|count, end_num| count >= end_num) - } else { - Box::new(|count, end_num| count <= end_num) - }; + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Statement::DatabaseQueryStatement { + db, + sql, + parameters, + variable_name, + kind, + line, + column, + } => { + let result = self + .evaluate_database_query( + db, + sql, + parameters.as_ref(), + *kind, + *line, + *column, + Rc::clone(&env), + ) + .await?; - let max_iterations = if end_num > 1000000.0 { - u64::MAX // Effectively no limit for large end values, rely on timeout instead - } else { - // Allow up to 10001 iterations to accommodate loops that need exactly 10000 - // (e.g., "count from 1 to 10000" requires 10000 iterations) - 10001 + match env.borrow_mut().define(variable_name, result) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + Statement::CloseDatabaseStatement { db, line, column } => { + let db_value = self.evaluate_expression(db, Rc::clone(&env)).await?; + let handle = match &db_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected a database handle, got {db_value:?}"), + *line, + *column, + )); + } }; - let mut iterations = 0; - - *self.in_count_loop.borrow_mut() = true; - - // Determine the variable name to use - custom name or default "count" - let loop_var_name = variable_name.as_deref().unwrap_or("count"); - let mut loop_env_recycle = None; - - while should_continue(count, end_num) && iterations < max_iterations { - self.check_time()?; - - *self.current_count.borrow_mut() = Some(count); - - // OPTIMIZATION: Recycle environment if possible - let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); - // Make the loop variable available in the loop environment, - // shadowing any same-named variable from an outer scope. - // Use custom variable name if provided, otherwise default to "count" - loop_env - .borrow_mut() - .define_or_replace(loop_var_name, Value::Number(count)); + self.io_client + .close_database(&handle) + .await + .map_err(|e| RuntimeError::new(e, *line, *column))?; - let result = self.execute_block(body, Rc::clone(&loop_env)).await; + Ok((Value::Null, ControlFlow::None)) + } + Statement::ReadFileStatement { + path, + variable_name, + line, + column, + } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file path or handle, got {path_value:?}"), + *line, + *column, + )); + } + }; - // Save environment for potential recycling in next iteration - loop_env_recycle = Some(loop_env); + let is_file_path = matches!(path, Expression::Literal(Literal::String(_), _, _)); - match result { - Ok((_, control_flow)) => match control_flow { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of count loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing count loop"); + if is_file_path { + match self.io_client.open_file(&path_str).await { + Ok(handle) => match self.io_client.read_file(&handle, &self.budget).await { + Ok(content) => { + // Capture the define result and drop the env + // borrow before the `close_file` await below. + let define_result = env + .borrow_mut() + .define(variable_name, Value::Text(content.into())); + match define_result { + Ok(_) => { + let _ = self.io_client.close_file(&handle).await; + Ok((Value::Null, ControlFlow::None)) + } + Err(msg) => { + let _ = self.io_client.close_file(&handle).await; + Err(RuntimeError::new(msg, *line, *column)) + } + } } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from count loop"); - *self.current_count.borrow_mut() = previous_count; - *self.in_count_loop.borrow_mut() = was_in_count_loop; - return Ok((Value::Null, ControlFlow::Exit)); + Err(e) => { + let _ = self.io_client.close_file(&handle).await; + Err(self.file_read_error(e, *line, *column)) } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from count loop with value: {:?}", val); - *self.current_count.borrow_mut() = previous_count; - *self.in_count_loop.borrow_mut() = was_in_count_loop; - return Ok((val.clone(), ControlFlow::Return(val))); + }, + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } else { + match self.io_client.read_file(&path_str, &self.budget).await { + Ok(content) => { + match env + .borrow_mut() + .define(variable_name, Value::Text(content.into())) + { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), } - ControlFlow::None => {} - }, - Err(e) => { - *self.current_count.borrow_mut() = previous_count; - *self.in_count_loop.borrow_mut() = was_in_count_loop; - return Err(e); } + Err(e) => Err(self.file_read_error(e, *line, *column)), } + } + } + Statement::WriteFileStatement { + file, + content, + mode, + line, + column, + } => { + let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; + let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; - if *downward { - count -= step_num; - } else { - count += step_num; + let file_str = match &file_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file handle, got {file_value:?}"), + *line, + *column, + )); } + }; - iterations += 1; - } + let content_str = match &content_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file content, got {content_value:?}"), + *line, + *column, + )); + } + }; - *self.current_count.borrow_mut() = previous_count; - *self.in_count_loop.borrow_mut() = was_in_count_loop; + match mode { + crate::parser::ast::WriteMode::Append => { + match self.io_client.append_file(&file_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + crate::parser::ast::WriteMode::Overwrite => { + match self.io_client.write_file(&file_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + } + } + Statement::CloseFileStatement { file, line, column } => { + let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; - if iterations >= max_iterations { - return Err(RuntimeError::new( - format!("Count loop exceeded maximum iterations ({max_iterations})"), + match &file_value { + // A bare text handle closes a file (existing behavior). + Value::Text(s) => match self.io_client.close_file(s).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + }, + // A streaming-response object closes its underlying stream. + // For a client upstream (`_stream`) this cancels the in-flight + // request; for a server response stream (`_server_stream`) + // this drops the body sender, ending the response. Closing a + // stream is always safe, even after EOF / already closed. + Value::Object(obj) => { + let (client_id, server_id) = { + let obj_ref = obj.borrow(); + let client = obj_ref.get("_stream").and_then(|v| match v { + Value::Text(s) => Some(s.to_string()), + _ => None, + }); + let server = obj_ref.get("_server_stream").and_then(|v| match v { + Value::Text(s) => Some(s.to_string()), + _ => None, + }); + (client, server) + }; + if let Some(id) = client_id { + self.io_client.close_stream(&id).await; + // Drop it from the handler's ownership tracking so the + // exit cleanup does not try to re-close it. + self.untrack_http_stream(&id); + Ok((Value::Null, ControlFlow::None)) + } else if let Some(id) = server_id { + // Dropping the sender ends the response body stream. + self.server_response_streams.borrow_mut().remove(&id); + // Drop it from the handler's auto-close tracking so + // the list stays bounded to actually-open streams. + self.open_response_streams.borrow_mut().retain(|s| s != &id); + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + "Cannot close this value: it is not a file handle or a \ + streaming response" + .to_string(), + *line, + *column, + )) + } + } + _ => Err(RuntimeError::new( + format!( + "Expected a file handle or streaming response, got {}", + file_value.type_name() + ), *line, *column, - )); + )), } - - Ok((Value::Null, ControlFlow::None)) } + Statement::CreateDirectoryStatement { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); + } + }; - Statement::ForEachLoop { - item_name, - collection, - reversed, - body, + match self.io_client.create_directory(&path_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Statement::CreateFileStatement { + path, + content, line, column, } => { - let collection_val = self - .evaluate_expression(collection, Rc::clone(&env)) - .await?; - - match collection_val { - Value::List(list_rc) => { - let items: Vec = { - let list = list_rc.borrow(); - let indices: Vec = if *reversed { - (0..list.len()).rev().collect() - } else { - (0..list.len()).collect() - }; - indices.iter().map(|&i| list[i].clone()).collect() - }; - - let mut loop_env_recycle = None; - for item in items { - // OPTIMIZATION: Recycle environment if possible - let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); - - match loop_env.borrow_mut().define(item_name, item) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), - } - let result = self.execute_block(body, Rc::clone(&loop_env)).await?; - - // Save environment for potential recycling in next iteration - loop_env_recycle = Some(loop_env); + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of foreach loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing foreach loop"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from foreach loop"); - return Ok((Value::Null, ControlFlow::Exit)); - } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!( - "Returning from foreach loop with value: {:?}", - val - ); - return Ok((val.clone(), ControlFlow::Return(val))); - } - ControlFlow::None => {} - } - } + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file path, got {path_value:?}"), + *line, + *column, + )); } - Value::Object(obj_rc) => { - let items: Vec<(String, Value)> = { - let obj = obj_rc.borrow(); - obj.iter().map(|(k, v)| (k.clone(), v.clone())).collect() - }; - - let mut loop_env_recycle = None; - for (_, value) in items { - // OPTIMIZATION: Recycle environment if possible - let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); - - match loop_env.borrow_mut().define(item_name, value) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), - } - let result = self.execute_block(body, Rc::clone(&loop_env)).await?; + }; - // Save environment for potential recycling in next iteration - loop_env_recycle = Some(loop_env); + let content_str = format!("{content_value}"); - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of foreach loop (object)"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing foreach loop (object)"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from foreach loop (object)"); - return Ok((Value::Null, ControlFlow::Exit)); - } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!( - "Returning from foreach loop with value: {:?}", - val - ); - return Ok((val.clone(), ControlFlow::Return(val))); - } - ControlFlow::None => {} - } - } + match self.io_client.create_file(&path_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Statement::DeleteFileStatement { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file path, got {path_value:?}"), + *line, + *column, + )); } + }; + + match self.io_client.delete_file(&path_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Statement::WriteToStatement { + content, + file, + line, + column, + } => { + let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; + let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; + + let file_str = match &file_value { + Value::Text(s) => s.clone(), _ => { return Err(RuntimeError::new( - format!("Cannot iterate over {}", collection_val.type_name()), + format!("Expected string for file handle, got {file_value:?}"), *line, *column, )); } - } + }; - Ok((Value::Null, ControlFlow::None)) - } + let content_str = format!("{content_value}"); - Statement::WhileLoop { - condition, - body, - line: _line, - column: _column, + match self.io_client.write_file(&file_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Statement::WriteContentStatement { + content, + target, + line, + column, } => { - let mut _last_value = Value::Null; + let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; + let target_value = self.evaluate_expression(target, Rc::clone(&env)).await?; - while self - .evaluate_expression(condition, Rc::clone(&env)) - .await? - .is_truthy() - { - self.check_time()?; - let result = self.execute_block(body, Rc::clone(&env)).await?; - _last_value = result.0; + let target_str = match &target_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file handle, got {target_value:?}"), + *line, + *column, + )); + } + }; - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of while loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing while loop"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from while loop"); - return Ok((_last_value, ControlFlow::Exit)); - } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from while loop with value: {:?}", val); - return Ok((val.clone(), ControlFlow::Return(val))); - } - ControlFlow::None => {} + let content_str = format!("{content_value}"); + + // Check if target is a file handle (starts with "file") or a file path + if target_str.starts_with("file") { + // This is a file handle, use append_file to respect the file's open mode + match self.io_client.append_file(&target_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } else { + // This is a file path, use write_file (overwrite mode) + match self.io_client.write_file(&target_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), } } - - Ok((_last_value, ControlFlow::None)) } - - Statement::RepeatUntilLoop { - condition, - body, - line: _line, - column: _column, + Statement::WriteBinaryStatement { + content, + target, + line, + column, } => { - let mut _last_value = Value::Null; + let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; + let target_value = self.evaluate_expression(target, Rc::clone(&env)).await?; - loop { - self.check_time()?; - let result = self.execute_block(body, Rc::clone(&env)).await?; - _last_value = result.0; + let target_str = match &target_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file handle, got {}", + target_value.type_name() + ), + *line, + *column, + )); + } + }; - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of repeat-until loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing repeat-until loop"); - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from repeat-until loop"); - return Ok((_last_value, ControlFlow::Exit)); + // 50MB limit to prevent memory exhaustion + const MAX_BINARY_WRITE: usize = 50 * 1024 * 1024; + let bytes = match &content_value { + Value::Binary(b) => b.clone(), + Value::List(items) => { + let items = items.borrow(); + if items.len() > MAX_BINARY_WRITE { + return Err(RuntimeError::new( + format!( + "Byte list length {} exceeds maximum allowed ({})", + items.len(), + MAX_BINARY_WRITE + ), + *line, + *column, + )); } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from repeat-until loop with value: {:?}", val); - return Ok((val.clone(), ControlFlow::Return(val))); + let mut bytes = Vec::with_capacity(items.len()); + for (i, item) in items.iter().enumerate() { + match item { + Value::Number(n) => { + if !n.is_finite() || n.fract() != 0.0 || *n < 0.0 || *n > 255.0 + { + return Err(RuntimeError::new( + format!( + "Invalid byte value at index {i}: {n} — must be an integer 0-255" + ), + *line, + *column, + )); + } + bytes.push(*n as u8); + } + _ => { + return Err(RuntimeError::new( + format!( + "Expected number in byte list at index {}, got {}", + i, + item.type_name() + ), + *line, + *column, + )); + } + } } - ControlFlow::None => {} + Arc::from(bytes) } - - if self - .evaluate_expression(condition, Rc::clone(&env)) - .await? - .is_truthy() - { - break; + _ => { + return Err(RuntimeError::new( + format!( + "Expected Binary or List for write binary, got {}", + content_value.type_name() + ), + *line, + *column, + )); } - } + }; - Ok((_last_value, ControlFlow::None)) + match self.io_client.write_binary(&target_str, &bytes).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } } - - Statement::ForeverLoop { - body, - line: _line, - column: _column, - } => { - #[cfg(debug_assertions)] - exec_trace!("Executing forever loop"); - - let mut _last_value = Value::Null; - let mut loop_env_recycle = None; - - loop { - self.check_time()?; - - // OPTIMIZATION: Recycle environment if possible - let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); - let result = self.execute_block(body, Rc::clone(&loop_env)).await?; - _last_value = result.0; - - // Save environment for potential recycling in next iteration - loop_env_recycle = Some(loop_env); - - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of forever loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing forever loop"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from forever loop"); - return Ok((_last_value, ControlFlow::Exit)); - } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from forever loop with value: {:?}", val); - return Ok((val.clone(), ControlFlow::Return(val))); - } - ControlFlow::None => {} + Statement::DeleteDirectoryStatement { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); } - } + }; - Ok((_last_value, ControlFlow::None)) + match self.io_client.delete_directory(&path_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } } - Statement::MainLoop { - body, - line: _line, - column: _column, + Statement::LoadModuleStatement { + path, line, column, .. } => { - #[cfg(debug_assertions)] - exec_trace!("Executing main loop (timeout disabled)"); + // 1. Evaluate path expression to string + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str: String = match &path_value { + Value::Text(s) => s.to_string(), + _ => { + return Err(RuntimeError::new( + format!("Module path must be a string, got {path_value:?}"), + *line, + *column, + )); + } + }; - // Enter the main loop's deadline exemption via an RAII guard on - // the shared budget. The guard restores the depth on EVERY exit — - // a normal end, an early `return` below, a caught error unwinding - // through `?`, or a nested main loop — so the exemption is never - // leaked or cleared while an outer loop is still active. A child - // `execute file` sharing this budget inherits the exemption too. - let _main_loop_guard = self.budget.enter_main_loop(); + // 2. Resolve absolute path + let resolved_path = self.resolve_module_path(&path_str, *line, *column).await?; - let mut _last_value = Value::Null; - let mut loop_env_recycle = None; + // 3. Check circular dependencies and the shared import-depth + // ceiling (loading_stack length is the depth already entered). + self.check_circular_dependency(&resolved_path, *line, *column)?; + if let Err(exceeded) = self + .budget + .check_import_depth(self.loading_stack.borrow().len()) + { + return Err(self.budget_error(exceeded, *line, *column)); + } - loop { - // check_time() skips the deadline while the main-loop depth > 0 - self.check_time()?; + // 4. Read file content under the shared source-size ceiling. + let content = self + .read_source_bounded(&resolved_path, *line, *column) + .await?; - // OPTIMIZATION: Recycle environment if possible - let loop_env = self.get_recycled_env(loop_env_recycle.take(), &env); - let result = self.execute_block(body, Rc::clone(&loop_env)).await?; - _last_value = result.0; + // 6. Parse module + use crate::lexer::lex_wfl_with_positions_checked; + use crate::parser::Parser; - // Save environment for potential recycling in next iteration - loop_env_recycle = Some(loop_env); + // Lex under the shared run budget: a deadline / cancellation / + // operation breach during nested source loading surfaces as a + // typed, catchable runtime error instead of a truncated token + // stream that could execute as if it were the whole file. + let tokens = lex_wfl_with_positions_checked(&content) + .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; + let mut parser = Parser::new(&tokens); + let program = parser.parse().map_err(|errors| { + // Use the parse error's position from the module file, not the load site + let first_error = errors.first(); + let (error_line, error_column) = + first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); + RuntimeError::new( + format!( + "Parse error in module '{}': {}", + resolved_path.display(), + first_error.map(|e| e.message.as_str()).unwrap_or("unknown") + ), + error_line, + error_column, + ) + })?; - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of main loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing main loop"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from main loop"); - // `_main_loop_guard` drops here, restoring the depth. - return Ok((_last_value, ControlFlow::Exit)); + // 7. Analyze semantics + use crate::analyzer::Analyzer; + + // Extract parent variables from current environment for module analyzer + let parent_vars = Self::extract_parent_variables(&env); + let mut analyzer = Analyzer::with_parent_variables(parent_vars); + if let Err(errors) = analyzer.analyze(&program) { + // Use the semantic error's position from the module file, not the load site + let first_error = errors.first(); + let (error_line, error_column) = + first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); + return Err(RuntimeError::new( + format!( + "Semantic error in module '{}': {}", + resolved_path.display(), + first_error.map(|e| e.to_string()).unwrap_or_default() + ), + error_line, + error_column, + )); + } + + // 8. Type check + use crate::typechecker::{TypeCheckError, TypeChecker}; + + // Use the analyzer with parent scope for type checking + let mut tc = TypeChecker::with_analyzer(analyzer); + if let Err(failure) = tc.check_types(&program) { + match failure { + // A shared-budget breach while type-checking the module is + // fatal: surface it as the catchable resource/timeout + // error rather than a "type error in module". + TypeCheckError::Budget(exceeded) => { + return Err(self.budget_error(exceeded, 0, 0)); } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from main loop with value: {:?}", val); - return Ok((val.clone(), ControlFlow::Return(val))); + TypeCheckError::Types(type_errors) => { + // Use the type error's position from the module file, not the load site + let first_error = type_errors.first(); + let (error_line, error_column) = + first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); + return Err(RuntimeError::new( + format!( + "Type error in module '{}': {}", + resolved_path.display(), + first_error.map(|e| e.to_string()).unwrap_or_default() + ), + error_line, + error_column, + )); } - ControlFlow::None => {} } } - // `_main_loop_guard` drops here on normal exit. - Ok((_last_value, ControlFlow::None)) - } - - Statement::BreakStatement { .. } => { - #[cfg(debug_assertions)] - exec_trace!("Executing break statement"); - Ok((Value::Null, ControlFlow::Break)) - } + // 8. Create isolated child environment + // This prevents mutations of containers (lists/objects) from affecting parent scope + use crate::interpreter::environment::Environment; + let module_env = Environment::new_isolated_child_env(&env); - Statement::ContinueStatement { .. } => { - #[cfg(debug_assertions)] - exec_trace!("Executing continue statement"); - Ok((Value::Null, ControlFlow::Continue)) - } + // 9. Create guard to ensure context restoration on scope exit + let previous_source = self.current_source_file.borrow().clone(); + let _guard = ModuleLoadGuard::new(self, resolved_path.clone(), previous_source); - Statement::ExitStatement { .. } => { - #[cfg(debug_assertions)] - exec_trace!("Executing exit statement"); - Ok((Value::Null, ControlFlow::Exit)) - } + // 10. Execute module in child scope + let result = self.execute_block(&program.statements, module_env).await; - Statement::OpenFileStatement { - path, - variable_name, - mode, - line, - column, - } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file path, got {path_value:?}"), - *line, - *column, - )); - } - }; + // Note: Context automatically restored when _guard drops at end of scope - // Use the appropriate file open mode - match self - .io_client - .open_file_with_mode(&path_str, mode.clone()) - .await - { - Ok(handle) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(handle.into())) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + // 11. Handle result + match result { + Ok((_, ControlFlow::None)) => Ok((Value::Null, ControlFlow::None)), + Ok((_, ControlFlow::Return(_))) => Err(RuntimeError::new( + "Cannot use 'return' in module scope".to_string(), + *line, + *column, + )), + Ok((_, ControlFlow::Break)) => Err(RuntimeError::new( + "Cannot use 'break' in module scope".to_string(), + *line, + *column, + )), + Ok((_, ControlFlow::Continue)) => Err(RuntimeError::new( + "Cannot use 'continue' in module scope".to_string(), + *line, + *column, + )), + Ok((_, ControlFlow::Exit)) => Err(RuntimeError::new( + "Cannot use 'exit' in module scope".to_string(), + *line, + *column, + )), + Err(e) => { + // Capture chain BEFORE guard drops (while current module is still on stack) + let chain = _guard.get_chain(); + if chain.len() > 1 { + // Only show chain if there are multiple modules + // Preserve the original error kind and use the original error's coordinates + Err(RuntimeError::with_kind( + format!( + "Error in module chain {}: {}", + chain.join(" → "), + e.message + ), + e.line, + e.column, + e.kind, + )) + } else { + Err(e) } } - Err(e) => Err(e), } } - Statement::OpenDatabaseStatement { - url, - variable_name, - line, - column, + + Statement::IncludeStatement { + path, line, column, .. } => { - let url_value = self.evaluate_expression(url, Rc::clone(&env)).await?; - let url_str = match &url_value { - Value::Text(s) => s.clone(), + // Include statement is like LoadModule but executes in parent scope instead of isolated child + + // 1. Evaluate path expression to string + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str: String = match &path_value { + Value::Text(s) => s.to_string(), _ => { return Err(RuntimeError::new( - format!("Expected text for database URL, got {url_value:?}"), + format!("Include path must be a string, got {path_value:?}"), *line, *column, )); } }; - match self.io_client.open_database(&url_str).await { - Ok(handle) => { - let define_result = env - .borrow_mut() - .define(variable_name, Value::Text(handle.as_str().into())); - match define_result { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => { - // Don't leave an unreachable pool behind when - // the variable binding fails. - let _ = self.io_client.close_database(&handle).await; - Err(RuntimeError::new(msg, *line, *column)) - } - } - } - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } - Statement::DatabaseQueryStatement { - db, - sql, - parameters, - variable_name, - kind, - line, - column, - } => { - let result = self - .evaluate_database_query( - db, - sql, - parameters.as_ref(), - *kind, - *line, - *column, - Rc::clone(&env), - ) - .await?; + // 2. Resolve absolute path + let resolved_path = self.resolve_module_path(&path_str, *line, *column).await?; - match env.borrow_mut().define(variable_name, result) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + // 3. Check circular dependencies and the shared import-depth + // ceiling (loading_stack length is the depth already entered). + self.check_circular_dependency(&resolved_path, *line, *column)?; + if let Err(exceeded) = self + .budget + .check_import_depth(self.loading_stack.borrow().len()) + { + return Err(self.budget_error(exceeded, *line, *column)); } - } - Statement::CloseDatabaseStatement { db, line, column } => { - let db_value = self.evaluate_expression(db, Rc::clone(&env)).await?; - let handle = match &db_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected a database handle, got {db_value:?}"), - *line, - *column, - )); - } - }; - self.io_client - .close_database(&handle) - .await - .map_err(|e| RuntimeError::new(e, *line, *column))?; + // 4. Read file content under the shared source-size ceiling. + let content = self + .read_source_bounded(&resolved_path, *line, *column) + .await?; - Ok((Value::Null, ControlFlow::None)) - } - Statement::ReadFileStatement { - path, - variable_name, - line, - column, - } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file path or handle, got {path_value:?}"), - *line, - *column, - )); - } - }; + // 5. Parse included file + use crate::lexer::lex_wfl_with_positions_checked; + use crate::parser::Parser; - let is_file_path = matches!(path, Expression::Literal(Literal::String(_), _, _)); + // Lex under the shared run budget: a deadline / cancellation / + // operation breach during nested source loading surfaces as a + // typed, catchable runtime error instead of a truncated token + // stream that could execute as if it were the whole file. + let tokens = lex_wfl_with_positions_checked(&content) + .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; + let mut parser = Parser::new(&tokens); + let program = parser.parse().map_err(|errors| { + let first_error = errors.first(); + let (error_line, error_column) = + first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); + RuntimeError::new( + format!( + "Parse error in included file '{}': {}", + resolved_path.display(), + first_error.map(|e| e.message.as_str()).unwrap_or("unknown") + ), + error_line, + error_column, + ) + })?; - if is_file_path { - match self.io_client.open_file(&path_str).await { - Ok(handle) => match self.io_client.read_file(&handle, &self.budget).await { - Ok(content) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(content.into())) - { - Ok(_) => { - let _ = self.io_client.close_file(&handle).await; - Ok((Value::Null, ControlFlow::None)) - } - Err(msg) => { - let _ = self.io_client.close_file(&handle).await; - Err(RuntimeError::new(msg, *line, *column)) - } - } - } - Err(e) => { - let _ = self.io_client.close_file(&handle).await; - Err(self.file_read_error(e, *line, *column)) - } - }, - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } else { - match self.io_client.read_file(&path_str, &self.budget).await { - Ok(content) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(content.into())) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } - } - Err(e) => Err(self.file_read_error(e, *line, *column)), - } - } - } - Statement::WriteFileStatement { - file, - content, - mode, - line, - column, - } => { - let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; - let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; + // 6. Analyze semantics + use crate::analyzer::Analyzer; - let file_str = match &file_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {file_value:?}"), - *line, - *column, - )); - } - }; + let parent_vars = Self::extract_parent_variables(&env); + let mut analyzer = Analyzer::with_parent_variables_mutable(parent_vars); + if let Err(errors) = analyzer.analyze(&program) { + let first_error = errors.first(); + let (error_line, error_column) = + first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); + return Err(RuntimeError::new( + format!( + "Semantic error in included file '{}': {}", + resolved_path.display(), + first_error.map(|e| e.to_string()).unwrap_or_default() + ), + error_line, + error_column, + )); + } - let content_str = match &content_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file content, got {content_value:?}"), - *line, - *column, - )); - } - }; + // 7. Type check. Type errors are reported as non-fatal + // warnings, exactly like the main-file pipeline (main.rs + // prints them and continues): `include from` executes in the + // parent scope, so included code must never be checked more + // strictly than the same code written in the main program + // (issues #551/#553). + use crate::diagnostics::DiagnosticReporter; + use crate::typechecker::{TypeCheckError, TypeChecker}; - match mode { - crate::parser::ast::WriteMode::Append => { - match self.io_client.append_file(&file_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), + let mut tc = TypeChecker::with_analyzer(analyzer); + if let Err(failure) = tc.check_types(&program) { + match failure { + // Ordinary type diagnostics stay non-fatal warnings here + // (included code must never be checked more strictly than + // the same code in the main file). A shared-budget breach + // is the exception: the deadline/cancellation/resource + // limit was hit while checking the included file, so the + // run must stop instead of executing it. + TypeCheckError::Budget(exceeded) => { + return Err(self.budget_error(exceeded, 0, 0)); } - } - crate::parser::ast::WriteMode::Overwrite => { - match self.io_client.write_file(&file_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), + TypeCheckError::Types(type_errors) => { + eprintln!( + "Type checking warnings in included file '{}':", + resolved_path.display() + ); + let mut reporter = DiagnosticReporter::new(); + let file_id = reporter + .add_file(resolved_path.display().to_string(), content.clone()); + for error in &type_errors { + let diagnostic = reporter.convert_type_error(file_id, error); + if reporter.report_diagnostic(file_id, &diagnostic).is_err() { + eprintln!("{error}"); + } + } } } } - } - Statement::CloseFileStatement { file, line, column } => { - let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; - - let file_str = match &file_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {file_value:?}"), - *line, - *column, - )); - } - }; - match self.io_client.close_file(&file_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } - Statement::CreateDirectoryStatement { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); - } - }; + // 8. Create guard for context tracking + let previous_source = self.current_source_file.borrow().clone(); + let _guard = ModuleLoadGuard::new(self, resolved_path.clone(), previous_source); - match self.io_client.create_directory(&path_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } - Statement::CreateFileStatement { - path, - content, - line, - column, - } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; + // 9. Execute included file in PARENT scope (key difference from load module) + // This allows containers/variables to be exposed to parent + let result = self + .execute_block(&program.statements, Rc::clone(&env)) + .await; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file path, got {path_value:?}"), - *line, - *column, - )); + // 10. Handle result + match result { + Ok((_, ControlFlow::None)) => Ok((Value::Null, ControlFlow::None)), + Ok((val, ControlFlow::Return(_))) => { + // Return statements in included files are allowed and simply return the value + // This enables utility functions in included files to use return statements + Ok((val, ControlFlow::None)) } - }; - - let content_str = format!("{content_value}"); - - match self.io_client.create_file(&path_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } - Statement::DeleteFileStatement { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file path, got {path_value:?}"), - *line, - *column, - )); + Ok((_, ControlFlow::Break)) => Err(RuntimeError::new( + "Cannot use 'break' in included file scope".to_string(), + *line, + *column, + )), + Ok((_, ControlFlow::Continue)) => Err(RuntimeError::new( + "Cannot use 'continue' in included file scope".to_string(), + *line, + *column, + )), + Ok((_, ControlFlow::Exit)) => Err(RuntimeError::new( + "Cannot use 'exit' in included file scope".to_string(), + *line, + *column, + )), + Err(e) => { + let chain = _guard.get_chain(); + if chain.len() > 1 { + Err(RuntimeError::with_kind( + format!( + "Error in include chain {}: {}", + chain.join(" → "), + e.message + ), + e.line, + e.column, + e.kind, + )) + } else { + Err(e) + } } - }; - - match self.io_client.delete_file(&path_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), } } - Statement::WriteToStatement { - content, - file, + + Statement::ExportStatement { + export_type, + name, line, column, + .. } => { - let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; - let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; - - let file_str = match &file_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {file_value:?}"), - *line, - *column, - )); - } - }; + // Export statement validates that the named item exists in current scope + // In V1, this is a foundation for future module namespace system - let content_str = format!("{content_value}"); + use crate::parser::ast::ExportType; - match self.io_client.write_file(&file_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), + match export_type { + ExportType::Container => { + // Check if container definition exists in local scope only + if let Some(value) = env.borrow().get_local(name) { + if matches!(value, Value::ContainerDefinition(_)) { + // Container exists - export is valid + // In future versions, this would add to export registry + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + format!("'{}' is not a container definition", name), + *line, + *column, + )) + } + } else { + // Check if it exists in parent scope to provide better error message + if env.borrow().get(name).is_some() { + Err(RuntimeError::new( + format!( + "Container '{}' is only defined in parent scope and cannot be exported", + name + ), + *line, + *column, + )) + } else { + Err(RuntimeError::new( + format!("Container '{}' not found in current scope", name), + *line, + *column, + )) + } + } + } + ExportType::Action => { + // Check if action definition exists in local scope only + if let Some(value) = env.borrow().get_local(name) { + if matches!(value, Value::Function(_) | Value::Overloaded(_)) { + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + format!("'{}' is not an action definition", name), + *line, + *column, + )) + } + } else { + // Check if it exists in parent scope to provide better error message + if env.borrow().get(name).is_some() { + Err(RuntimeError::new( + format!( + "Action '{}' is only defined in parent scope and cannot be exported", + name + ), + *line, + *column, + )) + } else { + Err(RuntimeError::new( + format!("Action '{}' not found in current scope", name), + *line, + *column, + )) + } + } + } + ExportType::Constant => { + // Check if the variable exists in local scope and is actually a constant + if let Some(_value) = env.borrow().get_local(name) { + if env.borrow().is_constant(name) { + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + format!( + "Variable '{}' is not a constant and cannot be exported as one", + name + ), + *line, + *column, + )) + } + } else { + // Check if it exists in parent scope to provide better error message + if env.borrow().get(name).is_some() { + Err(RuntimeError::new( + format!( + "Constant '{}' is only defined in parent scope and cannot be exported", + name + ), + *line, + *column, + )) + } else { + Err(RuntimeError::new( + format!("Constant '{}' not found in current scope", name), + *line, + *column, + )) + } + } + } } } - Statement::WriteContentStatement { - content, - target, - line, - column, + + Statement::WaitForStatement { + inner, + line: _line, + column: _column, } => { - let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; - let target_value = self.evaluate_expression(target, Rc::clone(&env)).await?; + match inner.as_ref() { + Statement::ExpressionStatement { + expression: Expression::Variable(var_name, _, _), + line: _, + column: _, + } => { + let max_attempts = 1000; // Prevent infinite waiting + for _ in 0..max_attempts { + if let Some(value) = env.borrow().get(var_name) + && !matches!(value, Value::Null) + { + return Ok((Value::Null, ControlFlow::None)); + } - let target_str = match &target_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {target_value:?}"), - *line, - *column, - )); - } - }; + tokio::time::sleep(std::time::Duration::from_millis(10)).await; - let content_str = format!("{content_value}"); + self.check_time()?; + } - // Check if target is a file handle (starts with "file") or a file path - if target_str.starts_with("file") { - // This is a file handle, use append_file to respect the file's open mode - match self.io_client.append_file(&target_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } else { - // This is a file path, use write_file (overwrite mode) - match self.io_client.write_file(&target_str, &content_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), + Err(RuntimeError::new( + format!("Timeout waiting for variable '{var_name}'"), + 0, + 0, + )) } - } - } - Statement::WriteBinaryStatement { - content, - target, - line, - column, - } => { - let content_value = self.evaluate_expression(content, Rc::clone(&env)).await?; - let target_value = self.evaluate_expression(target, Rc::clone(&env)).await?; + Statement::WriteFileStatement { + file, + content, + mode, + line, + column, + } => { + let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; + let content_value = + self.evaluate_expression(content, Rc::clone(&env)).await?; - let target_str = match &target_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string for file handle, got {}", - target_value.type_name() - ), - *line, - *column, - )); - } - }; + let file_str = match &file_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file handle, got {file_value:?}"), + *line, + *column, + )); + } + }; - // 50MB limit to prevent memory exhaustion - const MAX_BINARY_WRITE: usize = 50 * 1024 * 1024; - let bytes = match &content_value { - Value::Binary(b) => b.clone(), - Value::List(items) => { - let items = items.borrow(); - if items.len() > MAX_BINARY_WRITE { - return Err(RuntimeError::new( - format!( - "Byte list length {} exceeds maximum allowed ({})", - items.len(), - MAX_BINARY_WRITE - ), - *line, - *column, - )); - } - let mut bytes = Vec::with_capacity(items.len()); - for (i, item) in items.iter().enumerate() { - match item { - Value::Number(n) => { - if !n.is_finite() || n.fract() != 0.0 || *n < 0.0 || *n > 255.0 - { - return Err(RuntimeError::new( - format!( - "Invalid byte value at index {i}: {n} — must be an integer 0-255" - ), - *line, - *column, - )); + let content_str = match &content_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file content, got {content_value:?}" + ), + *line, + *column, + )); + } + }; + + exec_trace!("Writing to file: {}, content: {}", file_str, content_str); + match mode { + crate::parser::ast::WriteMode::Append => { + match self.io_client.append_file(&file_str, &content_str).await { + Ok(_) => { + exec_trace!("Successfully appended to file"); + Ok((Value::Null, ControlFlow::None)) + } + Err(e) => { + exec_trace!("Error appending to file: {}", e); + Err(RuntimeError::new(e, *line, *column)) } - bytes.push(*n as u8); } - _ => { - return Err(RuntimeError::new( - format!( - "Expected number in byte list at index {}, got {}", - i, - item.type_name() - ), - *line, - *column, - )); + } + crate::parser::ast::WriteMode::Overwrite => { + match self.io_client.write_file(&file_str, &content_str).await { + Ok(_) => { + exec_trace!("Successfully wrote to file"); + Ok((Value::Null, ControlFlow::None)) + } + Err(e) => { + exec_trace!("Error writing to file: {}", e); + Err(RuntimeError::new(e, *line, *column)) + } } } } - Arc::from(bytes) - } - _ => { - return Err(RuntimeError::new( - format!( - "Expected Binary or List for write binary, got {}", - content_value.type_name() - ), - *line, - *column, - )); } - }; + Statement::ReadFileStatement { + path, + variable_name, + line, + column, + } => { + exec_trace!("Executing wait for read file statement"); + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file path or handle, got {path_value:?}" + ), + *line, + *column, + )); + } + }; - match self.io_client.write_binary(&target_str, &bytes).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } - Statement::DeleteDirectoryStatement { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); + let is_file_path = + matches!(path, Expression::Literal(Literal::String(_), _, _)); + + if is_file_path { + match self.io_client.open_file(&path_str).await { + Ok(handle) => { + match self.io_client.read_file(&handle, &self.budget).await { + Ok(content) => { + // Capture the define result and drop the + // env borrow before the `close_file` await. + let define_result = env + .borrow_mut() + .define(variable_name, Value::Text(content.into())); + match define_result { + Ok(_) => { + let _ = + self.io_client.close_file(&handle).await; + Ok((Value::Null, ControlFlow::None)) + } + Err(msg) => { + let _ = + self.io_client.close_file(&handle).await; + Err(RuntimeError::new(msg, *line, *column)) + } + } + } + Err(e) => { + let _ = self.io_client.close_file(&handle).await; + Err(self.file_read_error(e, *line, *column)) + } + } + } + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } else { + match self.io_client.read_file(&path_str, &self.budget).await { + Ok(content) => { + match env + .borrow_mut() + .define(variable_name, Value::Text(content.into())) + { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + Err(e) => Err(self.file_read_error(e, *line, *column)), + } + } } - }; - - match self.io_client.delete_directory(&path_str).await { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), + _ => self.execute_statement(inner, Rc::clone(&env)).await, } } - - Statement::LoadModuleStatement { - path, line, column, .. + Statement::WaitForDurationStatement { + duration, + unit, + line, + column, } => { - // 1. Evaluate path expression to string - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str: String = match &path_value { - Value::Text(s) => s.to_string(), + let duration_value = self.evaluate_expression(duration, Rc::clone(&env)).await?; + let duration_ms = match &duration_value { + Value::Number(n) => match unit.as_str() { + "milliseconds" => *n as u64, + "seconds" => (*n * 1000.0) as u64, + _ => { + return Err(RuntimeError::new( + format!("Unsupported time unit: {}", unit), + *line, + *column, + )); + } + }, _ => { return Err(RuntimeError::new( - format!("Module path must be a string, got {path_value:?}"), + format!("Expected number for duration, got {duration_value:?}"), *line, *column, )); } }; - // 2. Resolve absolute path - let resolved_path = self.resolve_module_path(&path_str, *line, *column).await?; - - // 3. Check circular dependencies and the shared import-depth - // ceiling (loading_stack length is the depth already entered). - self.check_circular_dependency(&resolved_path, *line, *column)?; - if let Err(exceeded) = self - .budget - .check_import_depth(self.loading_stack.borrow().len()) - { - return Err(self.budget_error(exceeded, *line, *column)); - } - - // 4. Read file content under the shared source-size ceiling. - let content = self - .read_source_bounded(&resolved_path, *line, *column) + // While WebSocket servers are running, spend the wait window + // dispatching their events to the registered handler blocks. + // With no WebSocket servers this is an ordinary sleep, so the + // statement's timing semantics are unchanged. + self.pump_websocket_events(std::time::Duration::from_millis(duration_ms)) .await?; + Ok((Value::Null, ControlFlow::None)) + } + Statement::TryStatement { + body, + when_clauses, + otherwise_block, + finally_block, + line: _line, + column: _column, + } => { + let child_env = Environment::new_child_env(&env); - // 6. Parse module - use crate::lexer::lex_wfl_with_positions_checked; - use crate::parser::Parser; - - // Lex under the shared run budget: a deadline / cancellation / - // operation breach during nested source loading surfaces as a - // typed, catchable runtime error instead of a truncated token - // stream that could execute as if it were the whole file. - let tokens = lex_wfl_with_positions_checked(&content) - .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; - let mut parser = Parser::new(&tokens); - let program = parser.parse().map_err(|errors| { - // Use the parse error's position from the module file, not the load site - let first_error = errors.first(); - let (error_line, error_column) = - first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); - RuntimeError::new( - format!( - "Parse error in module '{}': {}", - resolved_path.display(), - first_error.map(|e| e.message.as_str()).unwrap_or("unknown") - ), - error_line, - error_column, - ) - })?; - - // 7. Analyze semantics - use crate::analyzer::Analyzer; - - // Extract parent variables from current environment for module analyzer - let parent_vars = Self::extract_parent_variables(&env); - let mut analyzer = Analyzer::with_parent_variables(parent_vars); - if let Err(errors) = analyzer.analyze(&program) { - // Use the semantic error's position from the module file, not the load site - let first_error = errors.first(); - let (error_line, error_column) = - first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); - return Err(RuntimeError::new( - format!( - "Semantic error in module '{}': {}", - resolved_path.display(), - first_error.map(|e| e.to_string()).unwrap_or_default() - ), - error_line, - error_column, - )); - } - - // 8. Type check - use crate::typechecker::{TypeCheckError, TypeChecker}; - - // Use the analyzer with parent scope for type checking - let mut tc = TypeChecker::with_analyzer(analyzer); - if let Err(failure) = tc.check_types(&program) { - match failure { - // A shared-budget breach while type-checking the module is - // fatal: surface it as the catchable resource/timeout - // error rather than a "type error in module". - TypeCheckError::Budget(exceeded) => { - return Err(self.budget_error(exceeded, 0, 0)); - } - TypeCheckError::Types(type_errors) => { - // Use the type error's position from the module file, not the load site - let first_error = type_errors.first(); - let (error_line, error_column) = - first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); - return Err(RuntimeError::new( - format!( - "Type error in module '{}': {}", - resolved_path.display(), - first_error.map(|e| e.to_string()).unwrap_or_default() - ), - error_line, - error_column, - )); - } - } - } - - // 8. Create isolated child environment - // This prevents mutations of containers (lists/objects) from affecting parent scope - use crate::interpreter::environment::Environment; - let module_env = Environment::new_isolated_child_env(&env); - - // 9. Create guard to ensure context restoration on scope exit - let previous_source = self.current_source_file.borrow().clone(); - let _guard = ModuleLoadGuard::new(self, resolved_path.clone(), previous_source); + let primary_result = match self.execute_block(body, Rc::clone(&child_env)).await { + Ok(val) => Ok(val), // Success path: just bubble result + Err(err) => { + // Find matching when clause based on error kind + let mut executed = false; + let mut result = Err(err.clone()); - // 10. Execute module in child scope - let result = self.execute_block(&program.statements, module_env).await; + for when_clause in when_clauses { + let matches = match &when_clause.error_type { + crate::parser::ast::ErrorType::General => true, // General catches all errors + crate::parser::ast::ErrorType::FileNotFound => { + err.kind == ErrorKind::FileNotFound + } + crate::parser::ast::ErrorType::PermissionDenied => { + err.kind == ErrorKind::PermissionDenied + } + crate::parser::ast::ErrorType::ProcessNotFound => { + err.kind == ErrorKind::ProcessNotFound + } + crate::parser::ast::ErrorType::ProcessSpawnFailed => { + err.kind == ErrorKind::ProcessSpawnFailed + } + crate::parser::ast::ErrorType::ProcessKillFailed => { + err.kind == ErrorKind::ProcessKillFailed + } + crate::parser::ast::ErrorType::CommandNotFound => { + err.kind == ErrorKind::CommandNotFound + } + }; - // Note: Context automatically restored when _guard drops at end of scope + if matches { + // Bind the error under the clause's name and the + // `error_message` alias, which is always available + // in error-handling clauses. + let error_text = Value::Text(err.message.into()); + { + let mut env_mut = child_env.borrow_mut(); + env_mut.define_or_replace( + &when_clause.error_name, + error_text.clone(), + ); + env_mut.define_or_replace("error_message", error_text); + } - // 11. Handle result - match result { - Ok((_, ControlFlow::None)) => Ok((Value::Null, ControlFlow::None)), - Ok((_, ControlFlow::Return(_))) => Err(RuntimeError::new( - "Cannot use 'return' in module scope".to_string(), - *line, - *column, - )), - Ok((_, ControlFlow::Break)) => Err(RuntimeError::new( - "Cannot use 'break' in module scope".to_string(), - *line, - *column, - )), - Ok((_, ControlFlow::Continue)) => Err(RuntimeError::new( - "Cannot use 'continue' in module scope".to_string(), - *line, - *column, - )), - Ok((_, ControlFlow::Exit)) => Err(RuntimeError::new( - "Cannot use 'exit' in module scope".to_string(), - *line, - *column, - )), - Err(e) => { - // Capture chain BEFORE guard drops (while current module is still on stack) - let chain = _guard.get_chain(); - if chain.len() > 1 { - // Only show chain if there are multiple modules - // Preserve the original error kind and use the original error's coordinates - Err(RuntimeError::with_kind( - format!( - "Error in module chain {}: {}", - chain.join(" → "), - e.message - ), - e.line, - e.column, - e.kind, - )) - } else { - Err(e) + result = self + .execute_block(&when_clause.body, Rc::clone(&child_env)) + .await; + executed = true; + break; + } + } + + // If no when clause matched and there's an otherwise block + if !executed && otherwise_block.is_some() { + result = self + .execute_block( + otherwise_block.as_ref().unwrap(), + Rc::clone(&child_env), + ) + .await; } + + result + } + }; + + // A `finally:` block runs on both the success and error paths, + // after any matching when/otherwise clause. If it raises its own + // error, that error wins; otherwise the primary result (the + // success value or the still-unhandled error) propagates. + if let Some(finally_stmts) = finally_block { + match self.execute_block(finally_stmts, child_env).await { + Ok(_) => primary_result, + Err(finally_err) => Err(finally_err), } + } else { + primary_result } } - - Statement::IncludeStatement { - path, line, column, .. + Statement::HttpGetStatement { + url, + variable_name, + line, + column, } => { - // Include statement is like LoadModule but executes in parent scope instead of isolated child - - // 1. Evaluate path expression to string - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str: String = match &path_value { - Value::Text(s) => s.to_string(), + let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; + let url_str = match &url_val { + Value::Text(s) => s.clone(), _ => { return Err(RuntimeError::new( - format!("Include path must be a string, got {path_value:?}"), + format!("Expected string for URL, got {url_val:?}"), *line, *column, )); } }; - // 2. Resolve absolute path - let resolved_path = self.resolve_module_path(&path_str, *line, *column).await?; - - // 3. Check circular dependencies and the shared import-depth - // ceiling (loading_stack length is the depth already entered). - self.check_circular_dependency(&resolved_path, *line, *column)?; - if let Err(exceeded) = self - .budget - .check_import_depth(self.loading_stack.borrow().len()) + match self + .io_client + .http_get(&url_str, Arc::clone(&self.budget)) + .await { - return Err(self.budget_error(exceeded, *line, *column)); - } - - // 4. Read file content under the shared source-size ceiling. - let content = self - .read_source_bounded(&resolved_path, *line, *column) - .await?; - - // 5. Parse included file - use crate::lexer::lex_wfl_with_positions_checked; - use crate::parser::Parser; - - // Lex under the shared run budget: a deadline / cancellation / - // operation breach during nested source loading surfaces as a - // typed, catchable runtime error instead of a truncated token - // stream that could execute as if it were the whole file. - let tokens = lex_wfl_with_positions_checked(&content) - .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; - let mut parser = Parser::new(&tokens); - let program = parser.parse().map_err(|errors| { - let first_error = errors.first(); - let (error_line, error_column) = - first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); - RuntimeError::new( - format!( - "Parse error in included file '{}': {}", - resolved_path.display(), - first_error.map(|e| e.message.as_str()).unwrap_or("unknown") - ), - error_line, - error_column, - ) - })?; - - // 6. Analyze semantics - use crate::analyzer::Analyzer; - - let parent_vars = Self::extract_parent_variables(&env); - let mut analyzer = Analyzer::with_parent_variables_mutable(parent_vars); - if let Err(errors) = analyzer.analyze(&program) { - let first_error = errors.first(); - let (error_line, error_column) = - first_error.map(|e| (e.line, e.column)).unwrap_or((1, 1)); - return Err(RuntimeError::new( - format!( - "Semantic error in included file '{}': {}", - resolved_path.display(), - first_error.map(|e| e.to_string()).unwrap_or_default() - ), - error_line, - error_column, - )); - } - - // 7. Type check. Type errors are reported as non-fatal - // warnings, exactly like the main-file pipeline (main.rs - // prints them and continues): `include from` executes in the - // parent scope, so included code must never be checked more - // strictly than the same code written in the main program - // (issues #551/#553). - use crate::diagnostics::DiagnosticReporter; - use crate::typechecker::{TypeCheckError, TypeChecker}; - - let mut tc = TypeChecker::with_analyzer(analyzer); - if let Err(failure) = tc.check_types(&program) { - match failure { - // Ordinary type diagnostics stay non-fatal warnings here - // (included code must never be checked more strictly than - // the same code in the main file). A shared-budget breach - // is the exception: the deadline/cancellation/resource - // limit was hit while checking the included file, so the - // run must stop instead of executing it. - TypeCheckError::Budget(exceeded) => { - return Err(self.budget_error(exceeded, 0, 0)); - } - TypeCheckError::Types(type_errors) => { - eprintln!( - "Type checking warnings in included file '{}':", - resolved_path.display() - ); - let mut reporter = DiagnosticReporter::new(); - let file_id = reporter - .add_file(resolved_path.display().to_string(), content.clone()); - for error in &type_errors { - let diagnostic = reporter.convert_type_error(file_id, error); - if reporter.report_diagnostic(file_id, &diagnostic).is_err() { - eprintln!("{error}"); - } - } + Ok(body) => { + match env + .borrow_mut() + .define(variable_name, Value::Text(body.into())) + { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), } } + Err(error) => Err(self.http_client_error(error, *line, *column)), } + } + Statement::HttpPostStatement { + url, + data, + variable_name, + line, + column, + } => { + let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; + let data_val = self.evaluate_expression(data, Rc::clone(&env)).await?; - // 8. Create guard for context tracking - let previous_source = self.current_source_file.borrow().clone(); - let _guard = ModuleLoadGuard::new(self, resolved_path.clone(), previous_source); - - // 9. Execute included file in PARENT scope (key difference from load module) - // This allows containers/variables to be exposed to parent - let result = self - .execute_block(&program.statements, Rc::clone(&env)) - .await; + let url_str = match &url_val { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for URL, got {url_val:?}"), + *line, + *column, + )); + } + }; - // 10. Handle result - match result { - Ok((_, ControlFlow::None)) => Ok((Value::Null, ControlFlow::None)), - Ok((val, ControlFlow::Return(_))) => { - // Return statements in included files are allowed and simply return the value - // This enables utility functions in included files to use return statements - Ok((val, ControlFlow::None)) + let data_str = match &data_val { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for data, got {data_val:?}"), + *line, + *column, + )); } - Ok((_, ControlFlow::Break)) => Err(RuntimeError::new( - "Cannot use 'break' in included file scope".to_string(), - *line, - *column, - )), - Ok((_, ControlFlow::Continue)) => Err(RuntimeError::new( - "Cannot use 'continue' in included file scope".to_string(), - *line, - *column, - )), - Ok((_, ControlFlow::Exit)) => Err(RuntimeError::new( - "Cannot use 'exit' in included file scope".to_string(), - *line, - *column, - )), - Err(e) => { - let chain = _guard.get_chain(); - if chain.len() > 1 { - Err(RuntimeError::with_kind( - format!( - "Error in include chain {}: {}", - chain.join(" → "), - e.message - ), - e.line, - e.column, - e.kind, - )) - } else { - Err(e) + }; + + match self + .io_client + .http_post(&url_str, &data_str, Arc::clone(&self.budget)) + .await + { + Ok(body) => { + match env + .borrow_mut() + .define(variable_name, Value::Text(body.into())) + { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), } } + Err(error) => Err(self.http_client_error(error, *line, *column)), } } - - Statement::ExportStatement { - export_type, - name, + Statement::HttpRequestStatement { + url, + method, + headers, + body, + variable_name, + full_response, line, column, - .. } => { - // Export statement validates that the named item exists in current scope - // In V1, this is a foundation for future module namespace system - - use crate::parser::ast::ExportType; + let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; + let url_str = match &url_val { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for URL, got {url_val:?}"), + *line, + *column, + )); + } + }; - match export_type { - ExportType::Container => { - // Check if container definition exists in local scope only - if let Some(value) = env.borrow().get_local(name) { - if matches!(value, Value::ContainerDefinition(_)) { - // Container exists - export is valid - // In future versions, this would add to export registry - Ok((Value::Null, ControlFlow::None)) - } else { - Err(RuntimeError::new( - format!("'{}' is not a container definition", name), - *line, - *column, - )) - } - } else { - // Check if it exists in parent scope to provide better error message - if env.borrow().get(name).is_some() { - Err(RuntimeError::new( - format!( - "Container '{}' is only defined in parent scope and cannot be exported", - name - ), - *line, - *column, - )) - } else { - Err(RuntimeError::new( - format!("Container '{}' not found in current scope", name), + let method_str = match method { + Some(method_expr) => { + let method_val = self + .evaluate_expression(method_expr, Rc::clone(&env)) + .await?; + match &method_val { + Value::Text(s) => s.trim().to_ascii_uppercase(), + _ => { + return Err(RuntimeError::new( + format!("Expected text for HTTP method, got {method_val:?}"), *line, *column, - )) + )); } } } - ExportType::Action => { - // Check if action definition exists in local scope only - if let Some(value) = env.borrow().get_local(name) { - if matches!(value, Value::Function(_) | Value::Overloaded(_)) { - Ok((Value::Null, ControlFlow::None)) - } else { - Err(RuntimeError::new( - format!("'{}' is not an action definition", name), - *line, - *column, - )) - } - } else { - // Check if it exists in parent scope to provide better error message - if env.borrow().get(name).is_some() { - Err(RuntimeError::new( - format!( - "Action '{}' is only defined in parent scope and cannot be exported", - name - ), - *line, - *column, - )) - } else { - Err(RuntimeError::new( - format!("Action '{}' not found in current scope", name), - *line, - *column, - )) + None => "GET".to_string(), + }; + + let mut header_list: Vec<(String, String)> = Vec::new(); + if let Some(headers_expr) = headers { + let headers_val = self + .evaluate_expression(headers_expr, Rc::clone(&env)) + .await?; + match &headers_val { + Value::Object(obj) => { + for (name, value) in obj.borrow().iter() { + let value_str = match value { + Value::Text(s) => s.to_string(), + Value::Number(_) | Value::Bool(_) => value.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Header '{name}' must be text, got {}", + value.type_name() + ), + *line, + *column, + )); + } + }; + header_list.push((name.clone(), value_str)); } + // HashMap iteration order is random; sort for + // deterministic requests and error messages + header_list.sort(); + } + _ => { + return Err(RuntimeError::new( + format!( + "Expected a map for headers, got {}", + headers_val.type_name() + ), + *line, + *column, + )); } } - ExportType::Constant => { - // Check if the variable exists in local scope and is actually a constant - if let Some(_value) = env.borrow().get_local(name) { - if env.borrow().is_constant(name) { - Ok((Value::Null, ControlFlow::None)) - } else { - Err(RuntimeError::new( - format!( - "Variable '{}' is not a constant and cannot be exported as one", - name - ), - *line, - *column, - )) - } - } else { - // Check if it exists in parent scope to provide better error message - if env.borrow().get(name).is_some() { - Err(RuntimeError::new( + } + + let body_str = match body { + Some(body_expr) => { + let body_val = self.evaluate_expression(body_expr, Rc::clone(&env)).await?; + match &body_val { + Value::Text(s) => Some(s.to_string()), + Value::Number(_) | Value::Bool(_) => Some(body_val.to_string()), + _ => { + return Err(RuntimeError::new( format!( - "Constant '{}' is only defined in parent scope and cannot be exported", - name + "Expected text for request body, got {}", + body_val.type_name() ), *line, *column, - )) - } else { - Err(RuntimeError::new( - format!("Constant '{}' not found in current scope", name), - *line, - *column, - )) + )); } } } - } - } + None => None, + }; - Statement::WaitForStatement { - inner, - line: _line, - column: _column, - } => { - match inner.as_ref() { - Statement::ExpressionStatement { - expression: Expression::Variable(var_name, _, _), - line: _, - column: _, - } => { - let max_attempts = 1000; // Prevent infinite waiting - for _ in 0..max_attempts { - if let Some(value) = env.borrow().get(var_name) - && !matches!(value, Value::Null) - { - return Ok((Value::Null, ControlFlow::None)); + match self + .io_client + .http_request( + &method_str, + &url_str, + &header_list, + body_str, + Arc::clone(&self.budget), + ) + .await + { + Ok((status, response_headers, response_body)) => { + let value = if *full_response { + let mut headers_map = HashMap::new(); + for (name, value) in response_headers { + headers_map.insert(name, Value::Text(value.into())); } - tokio::time::sleep(std::time::Duration::from_millis(10)).await; + let mut response_map = HashMap::new(); + response_map.insert("status".to_string(), Value::Number(status as f64)); + response_map.insert( + "ok".to_string(), + Value::Bool((200..300).contains(&status)), + ); + response_map + .insert("body".to_string(), Value::Text(response_body.into())); + response_map.insert( + "headers".to_string(), + Value::Object(Rc::new(RefCell::new(headers_map))), + ); + Value::Object(Rc::new(RefCell::new(response_map))) + } else { + Value::Text(response_body.into()) + }; - self.check_time()?; + match env.borrow_mut().define(variable_name, value) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), } - - Err(RuntimeError::new( - format!("Timeout waiting for variable '{var_name}'"), - 0, - 0, - )) } - Statement::WriteFileStatement { - file, - content, - mode, - line, - column, - } => { - let file_value = self.evaluate_expression(file, Rc::clone(&env)).await?; - let content_value = - self.evaluate_expression(content, Rc::clone(&env)).await?; - - let file_str = match &file_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {file_value:?}"), - *line, - *column, - )); - } - }; + Err(error) => Err(self.http_client_error(error, *line, *column)), + } + } + Statement::HttpStreamStatement { + url, + method, + headers, + body, + variable_name, + line, + column, + } => { + let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; + let url_str = match &url_val { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for URL, got {url_val:?}"), + *line, + *column, + )); + } + }; - let content_str = match &content_value { - Value::Text(s) => s.clone(), + let method_str = match method { + Some(method_expr) => { + let method_val = self + .evaluate_expression(method_expr, Rc::clone(&env)) + .await?; + match &method_val { + Value::Text(s) => s.trim().to_ascii_uppercase(), _ => { return Err(RuntimeError::new( - format!( - "Expected string for file content, got {content_value:?}" - ), + format!("Expected text for HTTP method, got {method_val:?}"), *line, *column, )); } - }; + } + } + None => "GET".to_string(), + }; - exec_trace!("Writing to file: {}, content: {}", file_str, content_str); - match mode { - crate::parser::ast::WriteMode::Append => { - match self.io_client.append_file(&file_str, &content_str).await { - Ok(_) => { - exec_trace!("Successfully appended to file"); - Ok((Value::Null, ControlFlow::None)) - } - Err(e) => { - exec_trace!("Error appending to file: {}", e); - Err(RuntimeError::new(e, *line, *column)) - } - } - } - crate::parser::ast::WriteMode::Overwrite => { - match self.io_client.write_file(&file_str, &content_str).await { - Ok(_) => { - exec_trace!("Successfully wrote to file"); - Ok((Value::Null, ControlFlow::None)) - } - Err(e) => { - exec_trace!("Error writing to file: {}", e); - Err(RuntimeError::new(e, *line, *column)) + let mut header_list: Vec<(String, String)> = Vec::new(); + if let Some(headers_expr) = headers { + let headers_val = self + .evaluate_expression(headers_expr, Rc::clone(&env)) + .await?; + match &headers_val { + Value::Object(obj) => { + for (name, value) in obj.borrow().iter() { + let value_str = match value { + Value::Text(s) => s.to_string(), + Value::Number(_) | Value::Bool(_) => value.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Header '{name}' must be text, got {}", + value.type_name() + ), + *line, + *column, + )); } - } + }; + header_list.push((name.clone(), value_str)); } + header_list.sort(); + } + _ => { + return Err(RuntimeError::new( + format!( + "Expected a map for headers, got {}", + headers_val.type_name() + ), + *line, + *column, + )); } } - Statement::ReadFileStatement { - path, - variable_name, - line, - column, - } => { - exec_trace!("Executing wait for read file statement"); - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), + } + + let body_str = match body { + Some(body_expr) => { + let body_val = self.evaluate_expression(body_expr, Rc::clone(&env)).await?; + match &body_val { + Value::Text(s) => Some(s.to_string()), + Value::Number(_) | Value::Bool(_) => Some(body_val.to_string()), _ => { return Err(RuntimeError::new( format!( - "Expected string for file path or handle, got {path_value:?}" + "Expected text for request body, got {}", + body_val.type_name() ), *line, *column, )); } - }; + } + } + None => None, + }; - let is_file_path = - matches!(path, Expression::Literal(Literal::String(_), _, _)); + // Race the head open (connect + await response head) against a + // client disconnect, so a browser that goes away while the upstream + // withholds its head cancels the open promptly (dropping `open_fut` + // aborts the upstream connection) instead of waiting out the head + // timeout. Valid before `start streaming response` via the pending + // request's oneshot (see `any_pending_request_disconnected`). + let open_fut = self.io_client.open_http_stream( + &method_str, + &url_str, + &header_list, + body_str, + Arc::clone(&self.budget), + ); + let disconnect = self.any_client_disconnected(self.downstream_disconnect_senders()); + let opened = { + tokio::pin!(open_fut); + tokio::pin!(disconnect); + tokio::select! { + r = &mut open_fut => r, + _ = &mut disconnect => Err(HttpClientError::Disconnected), + } + }; + match opened { + Ok((status, response_headers, handle_id)) => { + // Track the outbound handle as handler-owned so it is + // dropped (cancelling the upstream) if the handler ends + // without closing/exhausting it. + let owner = Arc::clone(&self.open_http_streams.borrow()); + self.io_client + .claim_stream_owner(&handle_id, &owner) + .map_err(|error| self.http_client_error(error, *line, *column))?; + let mut headers_map = HashMap::new(); + for (name, value) in response_headers { + headers_map.insert(name, Value::Text(value.into())); + } - if is_file_path { - match self.io_client.open_file(&path_str).await { - Ok(handle) => { - match self.io_client.read_file(&handle, &self.budget).await { - Ok(content) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(content.into())) - { - Ok(_) => { - let _ = - self.io_client.close_file(&handle).await; - Ok((Value::Null, ControlFlow::None)) - } - Err(msg) => { - let _ = - self.io_client.close_file(&handle).await; - Err(RuntimeError::new(msg, *line, *column)) - } - } - } - Err(e) => { - let _ = self.io_client.close_file(&handle).await; - Err(self.file_read_error(e, *line, *column)) - } - } - } - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } - } else { - match self.io_client.read_file(&path_str, &self.budget).await { - Ok(content) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(content.into())) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } - } - Err(e) => Err(self.file_read_error(e, *line, *column)), - } + let mut stream_map = HashMap::new(); + stream_map.insert("status".to_string(), Value::Number(status as f64)); + stream_map + .insert("ok".to_string(), Value::Bool((200..300).contains(&status))); + stream_map.insert( + "headers".to_string(), + Value::Object(Rc::new(RefCell::new(headers_map))), + ); + // Internal id used by `wait for next chunk|line` and + // `close`. Underscore-prefixed to signal "not for + // direct program use", mirroring request objects. + stream_map.insert("_stream".to_string(), Value::Text(handle_id.into())); + + let value = Value::Object(Rc::new(RefCell::new(stream_map))); + // define_or_replace so a `main loop` handler that rebinds + // `upstream` on each request works. + env.borrow_mut().define_or_replace(variable_name, value); + Ok((Value::Null, ControlFlow::None)) + } + Err(error) => Err(self.http_client_error(error, *line, *column)), + } + } + Statement::WaitForNextChunkStatement { + source, + variable_name, + line, + column, + } => { + let handle_id = self + .resolve_stream_handle(source, &env, *line, *column) + .await?; + // Race the upstream read against a client disconnect (either an + // open downstream response stream, or — if the handler has not + // called `start streaming response` yet — the pending request's + // oneshot) so a blocked proxy read is cancelled promptly when the + // browser goes away. + let disconnect = self.any_client_disconnected(self.downstream_disconnect_senders()); + let read = self + .io_client + .next_chunk(&handle_id, Arc::clone(&self.budget)); + let outcome = { + tokio::pin!(read); + tokio::pin!(disconnect); + tokio::select! { + r = &mut read => r, + _ = &mut disconnect => { + // Client gone: dropping `read` above already cancels + // the upstream; close the handle too in case the read + // had not yet taken it, and report the disconnect as a + // cooperative cancellation (not a handler failure). + self.io_client.close_stream(&handle_id).await; + Err(HttpClientError::Disconnected) + } + } + }; + match outcome { + // Raw bytes as Binary so callers can handle any payload. + // define_or_replace (not define) so re-reading into the same + // variable across a loop refreshes it, matching + // `wait for request ... as req`. + Ok(Some(bytes)) => { + let value = Value::Binary(Arc::from(bytes.as_slice())); + env.borrow_mut().define_or_replace(variable_name, value); + Ok((Value::Null, ControlFlow::None)) + } + // Clean EOF binds `nothing` so `check if chunk is nothing` ends the loop. + Ok(None) => { + // The handle left the map at EOF — stop owning it. + self.untrack_http_stream(&handle_id); + env.borrow_mut() + .define_or_replace(variable_name, Value::Null); + Ok((Value::Null, ControlFlow::None)) + } + Err(error) => { + // The read dropped the handle (timeout/cancel/error). + self.untrack_http_stream(&handle_id); + Err(self.http_client_error(error, *line, *column)) + } + } + } + Statement::WaitForNextLineStatement { + source, + variable_name, + line, + column, + } => { + let handle_id = self + .resolve_stream_handle(source, &env, *line, *column) + .await?; + // Race the upstream read against a client disconnect by EITHER + // signal (open downstream stream OR the pre-response pending + // request's oneshot), exactly like the `wait for next chunk` + // handler — a line read blocked before `start streaming response` + // must also be cancelled the moment the browser goes away. + let disconnect = self.any_client_disconnected(self.downstream_disconnect_senders()); + let read = self + .io_client + .next_line(&handle_id, Arc::clone(&self.budget)); + let outcome = { + tokio::pin!(read); + tokio::pin!(disconnect); + tokio::select! { + r = &mut read => r, + _ = &mut disconnect => { + self.io_client.close_stream(&handle_id).await; + Err(HttpClientError::Disconnected) } } - _ => self.execute_statement(inner, Rc::clone(&env)).await, + }; + match outcome { + Ok(Some(line_text)) => { + let value = Value::Text(line_text.into()); + env.borrow_mut().define_or_replace(variable_name, value); + Ok((Value::Null, ControlFlow::None)) + } + Ok(None) => { + self.untrack_http_stream(&handle_id); + env.borrow_mut() + .define_or_replace(variable_name, Value::Null); + Ok((Value::Null, ControlFlow::None)) + } + Err(error) => { + self.untrack_http_stream(&handle_id); + Err(self.http_client_error(error, *line, *column)) + } + } + } + Statement::RepeatWhileLoop { + condition, + body, + line: _line, + column: _column, + } => { + let loop_env = Environment::new_child_env(&env); + let mut _last_value = Value::Null; + + loop { + self.check_time()?; + + let condition_value = self + .evaluate_expression(condition, Rc::clone(&loop_env)) + .await?; + + if !condition_value.is_truthy() { + break; + } + + let result = self.execute_block(body, Rc::clone(&loop_env)).await?; + _last_value = result.0; + + match result.1 { + ControlFlow::Break => { + #[cfg(debug_assertions)] + exec_trace!("Breaking out of repeat-while loop"); + break; + } + ControlFlow::Continue => { + #[cfg(debug_assertions)] + exec_trace!("Continuing repeat-while loop"); + continue; + } + ControlFlow::Exit => { + #[cfg(debug_assertions)] + exec_trace!("Exiting from repeat-while loop"); + return Ok((_last_value, ControlFlow::Exit)); + } + ControlFlow::Return(val) => { + #[cfg(debug_assertions)] + exec_trace!("Returning from repeat-while loop"); + return Ok((val.clone(), ControlFlow::Return(val))); + } + ControlFlow::None => {} + } + } + + Ok((Value::Null, ControlFlow::None)) + } + Statement::PushStatement { + list, + value, + line, + column, + } => { + let list_val = self.evaluate_expression(list, Rc::clone(&env)).await?; + let value_val = self.evaluate_expression(value, Rc::clone(&env)).await?; + + match list_val { + Value::List(list_rc) => { + list_rc.borrow_mut().push(value_val); + Ok((Value::Null, ControlFlow::None)) + } + _ => Err(RuntimeError::new( + format!("Cannot push to non-list value: {list_val:?}"), + *line, + *column, + )), } } - Statement::WaitForDurationStatement { - duration, - unit, + Statement::CreateListStatement { + name, + initial_values, line, column, } => { - let duration_value = self.evaluate_expression(duration, Rc::clone(&env)).await?; - let duration_ms = match &duration_value { - Value::Number(n) => match unit.as_str() { - "milliseconds" => *n as u64, - "seconds" => (*n * 1000.0) as u64, - _ => { - return Err(RuntimeError::new( - format!("Unsupported time unit: {}", unit), - *line, - *column, - )); - } - }, - _ => { - return Err(RuntimeError::new( - format!("Expected number for duration, got {duration_value:?}"), - *line, - *column, - )); - } - }; + // Create a new list with initial values + let mut list_items = Vec::new(); + for value_expr in initial_values { + let value = self + .evaluate_expression(value_expr, Rc::clone(&env)) + .await?; + list_items.push(value); + } + + let list_value = Value::List(Rc::new(RefCell::new(list_items))); + match env.borrow_mut().define(name, list_value) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } - // While WebSocket servers are running, spend the wait window - // dispatching their events to the registered handler blocks. - // With no WebSocket servers this is an ordinary sleep, so the - // statement's timing semantics are unchanged. - self.pump_websocket_events(std::time::Duration::from_millis(duration_ms)) - .await?; Ok((Value::Null, ControlFlow::None)) } - Statement::TryStatement { - body, - when_clauses, - otherwise_block, - finally_block, - line: _line, - column: _column, + Statement::MapCreation { + name, + entries, + line, + column, } => { - let child_env = Environment::new_child_env(&env); + // Create a new map/object with initial entries + let mut map = std::collections::HashMap::new(); + for (key, value_expr) in entries { + let value = self + .evaluate_expression(value_expr, Rc::clone(&env)) + .await?; + map.insert(key.clone(), value); + } - let primary_result = match self.execute_block(body, Rc::clone(&child_env)).await { - Ok(val) => Ok(val), // Success path: just bubble result - Err(err) => { - // Find matching when clause based on error kind - let mut executed = false; - let mut result = Err(err.clone()); + let map_value = Value::Object(Rc::new(RefCell::new(map))); + match env.borrow_mut().define(name, map_value) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } - for when_clause in when_clauses { - let matches = match &when_clause.error_type { - crate::parser::ast::ErrorType::General => true, // General catches all errors - crate::parser::ast::ErrorType::FileNotFound => { - err.kind == ErrorKind::FileNotFound - } - crate::parser::ast::ErrorType::PermissionDenied => { - err.kind == ErrorKind::PermissionDenied - } - crate::parser::ast::ErrorType::ProcessNotFound => { - err.kind == ErrorKind::ProcessNotFound - } - crate::parser::ast::ErrorType::ProcessSpawnFailed => { - err.kind == ErrorKind::ProcessSpawnFailed - } - crate::parser::ast::ErrorType::ProcessKillFailed => { - err.kind == ErrorKind::ProcessKillFailed - } - crate::parser::ast::ErrorType::CommandNotFound => { - err.kind == ErrorKind::CommandNotFound - } - }; + Ok((Value::Null, ControlFlow::None)) + } + Statement::CreateDateStatement { + name, + value, + line, + column, + } => { + let date_value = if let Some(expr) = value { + // Evaluate the expression to get the date + self.evaluate_expression(expr, Rc::clone(&env)).await? + } else { + // Default to today's date + let today = chrono::Local::now().date_naive(); + Value::Date(Rc::new(today)) + }; - if matches { - // Bind the error under the clause's name and the - // `error_message` alias, which is always available - // in error-handling clauses. - let error_text = Value::Text(err.message.into()); - { - let mut env_mut = child_env.borrow_mut(); - env_mut.define_or_replace( - &when_clause.error_name, - error_text.clone(), - ); - env_mut.define_or_replace("error_message", error_text); - } + match env.borrow_mut().define(name, date_value) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } + Ok((Value::Null, ControlFlow::None)) + } + Statement::CreateTimeStatement { + name, + value, + line, + column, + } => { + let time_value = if let Some(expr) = value { + // Evaluate the expression to get the time + self.evaluate_expression(expr, Rc::clone(&env)).await? + } else { + // Default to current time + let now = chrono::Local::now().time(); + Value::Time(Rc::new(now)) + }; - result = self - .execute_block(&when_clause.body, Rc::clone(&child_env)) - .await; - executed = true; - break; - } - } + match env.borrow_mut().define(name, time_value) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } + Ok((Value::Null, ControlFlow::None)) + } + Statement::AddToListStatement { + value, + list_name, + line, + column, + } => { + // Evaluate the value to add + let value_to_add = self.evaluate_expression(value, Rc::clone(&env)).await?; - // If no when clause matched and there's an otherwise block - if !executed && otherwise_block.is_some() { - result = self - .execute_block( - otherwise_block.as_ref().unwrap(), - Rc::clone(&child_env), - ) - .await; - } + // Get the list from the environment + let list_val = env.borrow().get(list_name).ok_or_else(|| { + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) + })?; - result + match list_val { + Value::List(list_rc) => { + list_rc.borrow_mut().push(value_to_add); + Ok((Value::Null, ControlFlow::None)) } - }; + Value::Number(_) => { + // This is actually arithmetic add + // Convert to arithmetic operation + let current = list_val; + if let (Value::Number(n1), Value::Number(n2)) = (¤t, &value_to_add) { + let result = Value::Number(n1 + n2); + env.borrow_mut() + .assign(list_name, result) + .map_err(|e| RuntimeError::new(e, *line, *column))?; + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + "Cannot add non-numeric value to number".to_string(), + *line, + *column, + )) + } + } + _ => Err(RuntimeError::new( + format!("Cannot add to non-list value: {list_val:?}"), + *line, + *column, + )), + } + } + Statement::RemoveFromListStatement { + value, + list_name, + line, + column, + } => { + // Evaluate the value to remove + let value_to_remove = self.evaluate_expression(value, Rc::clone(&env)).await?; - // A `finally:` block runs on both the success and error paths, - // after any matching when/otherwise clause. If it raises its own - // error, that error wins; otherwise the primary result (the - // success value or the still-unhandled error) propagates. - if let Some(finally_stmts) = finally_block { - match self.execute_block(finally_stmts, child_env).await { - Ok(_) => primary_result, - Err(finally_err) => Err(finally_err), + // Get the list from the environment + let list_val = env.borrow().get(list_name).ok_or_else(|| { + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) + })?; + + match list_val { + Value::List(list_rc) => { + let mut list = list_rc.borrow_mut(); + // Remove the first occurrence of the value + if let Some(pos) = list.iter().position(|v| v == &value_to_remove) { + list.remove(pos); + } + Ok((Value::Null, ControlFlow::None)) } - } else { - primary_result + _ => Err(RuntimeError::new( + format!("Cannot remove from non-list value: {list_val:?}"), + *line, + *column, + )), } } - Statement::HttpGetStatement { - url, - variable_name, + Statement::ClearListStatement { + list_name, line, column, } => { - let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; - let url_str = match &url_val { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for URL, got {url_val:?}"), - *line, - *column, - )); - } - }; + // Get the list from the environment + let list_val = env.borrow().get(list_name).ok_or_else(|| { + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) + })?; - match self - .io_client - .http_get(&url_str, Arc::clone(&self.budget)) - .await - { - Ok(body) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(body.into())) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } + match list_val { + Value::List(list_rc) => { + list_rc.borrow_mut().clear(); + Ok((Value::Null, ControlFlow::None)) } - Err(error) => Err(self.http_client_error(error, *line, *column)), + _ => Err(RuntimeError::new( + format!("Cannot clear non-list value: {list_val:?}"), + *line, + *column, + )), } } - Statement::HttpPostStatement { - url, - data, - variable_name, + // Container-related statements + Statement::ContainerDefinition { + name, + extends, + implements, + properties, + methods, + events, + static_properties: _static_properties, + static_methods: _static_methods, line, column, } => { - let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; - let data_val = self.evaluate_expression(data, Rc::clone(&env)).await?; + // Create a new container definition + let mut container_properties = HashMap::new(); + let mut container_methods = HashMap::new(); - let url_str = match &url_val { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for URL, got {url_val:?}"), - *line, - *column, - )); - } - }; + for prop in properties { + let property_type_str = prop + .property_type + .as_ref() + .map(|ast_type| format!("{ast_type:?}")); - let data_str = match &data_val { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for data, got {data_val:?}"), - *line, - *column, - )); + let default_val = match &prop.default_value { + Some(expr) => { + // Evaluate the default expression to get a Value + (self._evaluate_expression(expr, env.clone()).await).ok() + } + None => None, + }; + + let value_prop = value::PropertyDefinition { + name: prop.name.clone(), + property_type: property_type_str, + default_value: default_val, + validation_rules: Vec::new(), + is_static: false, + is_public: true, + line: prop.line, + column: prop.column, + }; + container_properties.insert(prop.name.clone(), value_prop); + } + + for method in methods { + if let Statement::ActionDefinition { + name, + parameters, + body, + line, + column, + .. + } = method + { + let container_method = ContainerMethodValue { + name: name.clone(), + params: parameters.iter().map(|p| p.name.clone()).collect(), + body: body.clone(), + is_static: false, + is_public: true, + env: Rc::downgrade(&env), + line: *line, + column: *column, + }; + // TODO(#638): container methods do not support + // overloading — a repeated method name silently keeps + // the last definition here. Route same-name methods + // through an overload set (see + // `Environment::define_or_merge_action` / + // `select_overload` for the action equivalent). + container_methods.insert(name.clone(), container_method); } + } + + // Process events + let mut container_events = HashMap::new(); + for event in events { + let container_event = ContainerEventValue { + name: event.name.clone(), + params: event.parameters.iter().map(|p| p.name.clone()).collect(), + handlers: Vec::new(), + line: event.line, + column: event.column, + }; + container_events.insert(event.name.clone(), container_event); + } + + let container_def = ContainerDefinitionValue { + name: name.clone(), + extends: extends.clone(), + implements: implements.clone(), + properties: container_properties, + methods: container_methods, + events: container_events, + static_properties: HashMap::new(), // Future feature + static_methods: HashMap::new(), // Future feature + line: *line, + column: *column, }; - match self - .io_client - .http_post(&url_str, &data_str, Arc::clone(&self.budget)) - .await - { - Ok(body) => { - match env - .borrow_mut() - .define(variable_name, Value::Text(body.into())) - { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } - } - Err(error) => Err(self.http_client_error(error, *line, *column)), + // Create the container definition value + let container_value = Value::ContainerDefinition(Rc::new(container_def)); + + // Store the container definition in the environment + match env.borrow_mut().define(name, container_value.clone()) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), } + + Ok((container_value, ControlFlow::None)) } - Statement::HttpRequestStatement { - url, - method, - headers, - body, - variable_name, - full_response, + Statement::ContainerInstantiation { + container_type, + instance_name, + arguments, + property_initializers, line, column, } => { - let url_val = self.evaluate_expression(url, Rc::clone(&env)).await?; - let url_str = match &url_val { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for URL, got {url_val:?}"), - *line, - *column, - )); - } - }; + // Create container instance with inheritance support + let mut instance = self.create_container_instance_with_inheritance( + container_type, + &env, + *line, + *column, + )?; - let method_str = match method { - Some(method_expr) => { - let method_val = self - .evaluate_expression(method_expr, Rc::clone(&env)) - .await?; - match &method_val { - Value::Text(s) => s.trim().to_ascii_uppercase(), - _ => { - return Err(RuntimeError::new( - format!("Expected text for HTTP method, got {method_val:?}"), - *line, - *column, - )); - } - } - } - None => "GET".to_string(), - }; + // Process property initializers (override inherited properties) + for initializer in property_initializers { + let init_value = self + ._evaluate_expression(&initializer.value, env.clone()) + .await?; + instance + .properties + .insert(initializer.name.clone(), init_value); + } + + let instance_value = Value::ContainerInstance(Rc::new(RefCell::new(instance))); + + // Store the instance in the environment + match env + .borrow_mut() + .define(instance_name, instance_value.clone()) + { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + } - let mut header_list: Vec<(String, String)> = Vec::new(); - if let Some(headers_expr) = headers { - let headers_val = self - .evaluate_expression(headers_expr, Rc::clone(&env)) - .await?; - match &headers_val { - Value::Object(obj) => { - for (name, value) in obj.borrow().iter() { - let value_str = match value { - Value::Text(s) => s.to_string(), - Value::Number(_) | Value::Bool(_) => value.to_string(), - _ => { - return Err(RuntimeError::new( - format!( - "Header '{name}' must be text, got {}", - value.type_name() - ), - *line, - *column, - )); - } - }; - header_list.push((name.clone(), value_str)); - } - // HashMap iteration order is random; sort for - // deterministic requests and error messages - header_list.sort(); - } + // Call constructor method if arguments are provided + if !arguments.is_empty() { + // Look up the container definition to find the initialize method + let container_def = match env.borrow().get(container_type) { + Some(Value::ContainerDefinition(def)) => def.clone(), _ => { return Err(RuntimeError::new( - format!( - "Expected a map for headers, got {}", - headers_val.type_name() - ), + format!("Container '{container_type}' not found"), *line, *column, )); } - } - } + }; - let body_str = match body { - Some(body_expr) => { - let body_val = self.evaluate_expression(body_expr, Rc::clone(&env)).await?; - match &body_val { - Value::Text(s) => Some(s.to_string()), - Value::Number(_) | Value::Bool(_) => Some(body_val.to_string()), - _ => { - return Err(RuntimeError::new( - format!( - "Expected text for request body, got {}", - body_val.type_name() - ), - *line, - *column, - )); - } - } - } - None => None, - }; + // Check if the container has an "initialize" method + if let Some(init_method) = container_def.methods.get("initialize") { + // Create a function value from the initialize method + let init_function = FunctionValue { + name: Some("initialize".to_string()), + params: init_method.params.clone(), + param_types: vec![None; init_method.params.len()], + body: init_method.body.clone(), + env: init_method.env.clone(), + line: init_method.line, + column: init_method.column, + enforce_param_types: std::cell::Cell::new(false), + }; - match self - .io_client - .http_request( - &method_str, - &url_str, - &header_list, - body_str, - Arc::clone(&self.budget), - ) - .await - { - Ok((status, response_headers, response_body)) => { - let value = if *full_response { - let mut headers_map = HashMap::new(); - for (name, value) in response_headers { - headers_map.insert(name, Value::Text(value.into())); - } + // Create a new environment for the constructor execution + let init_env = Environment::new_child_env(&env); - let mut response_map = HashMap::new(); - response_map.insert("status".to_string(), Value::Number(status as f64)); - response_map.insert( - "ok".to_string(), - Value::Bool((200..300).contains(&status)), - ); - response_map - .insert("body".to_string(), Value::Text(response_body.into())); - response_map.insert( - "headers".to_string(), - Value::Object(Rc::new(RefCell::new(headers_map))), - ); - Value::Object(Rc::new(RefCell::new(response_map))) - } else { - Value::Text(response_body.into()) - }; + // Add 'this' to the environment (the instance being constructed) + let _ = init_env.borrow_mut().define("this", instance_value.clone()); - match env.borrow_mut().define(variable_name, value) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + // Evaluate the arguments + let mut arg_values = Vec::with_capacity(arguments.len()); + for arg in arguments { + let arg_val = self.evaluate_expression(&arg.value, env.clone()).await?; + arg_values.push(arg_val); } + + // Call the initialize method + self.call_function(&init_function, arg_values, *line, *column) + .await?; + } else if !arguments.is_empty() { + return Err(RuntimeError::new( + format!( + "Container '{container_type}' does not have an initialize method but arguments were provided" + ), + *line, + *column, + )); } - Err(error) => Err(self.http_client_error(error, *line, *column)), } + + Ok((instance_value, ControlFlow::None)) } - Statement::RepeatWhileLoop { - condition, - body, + Statement::InterfaceDefinition { + name, + extends, + required_actions, line: _line, column: _column, } => { - let loop_env = Environment::new_child_env(&env); - let mut _last_value = Value::Null; - - loop { - self.check_time()?; + // Create a new interface definition + let mut interface_required_actions = HashMap::new(); - let condition_value = self - .evaluate_expression(condition, Rc::clone(&loop_env)) - .await?; + for action in required_actions { + let value_action = value::ActionSignature { + name: action.name.clone(), + params: action.parameters.iter().map(|p| p.name.clone()).collect(), + line: action.line, + column: action.column, + }; + interface_required_actions.insert(action.name.clone(), value_action); + } - if !condition_value.is_truthy() { - break; - } + let interface_def = InterfaceDefinitionValue { + name: name.clone(), + extends: extends.clone(), + required_actions: interface_required_actions, + line: *_line, + column: *_column, + }; - let result = self.execute_block(body, Rc::clone(&loop_env)).await?; - _last_value = result.0; + let interface_value = Value::InterfaceDefinition(Rc::new(interface_def)); - match result.1 { - ControlFlow::Break => { - #[cfg(debug_assertions)] - exec_trace!("Breaking out of repeat-while loop"); - break; - } - ControlFlow::Continue => { - #[cfg(debug_assertions)] - exec_trace!("Continuing repeat-while loop"); - continue; - } - ControlFlow::Exit => { - #[cfg(debug_assertions)] - exec_trace!("Exiting from repeat-while loop"); - return Ok((_last_value, ControlFlow::Exit)); - } - ControlFlow::Return(val) => { - #[cfg(debug_assertions)] - exec_trace!("Returning from repeat-while loop"); - return Ok((val.clone(), ControlFlow::Return(val))); - } - ControlFlow::None => {} - } + // Store the interface definition in the environment + match env.borrow_mut().define(name, interface_value.clone()) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *_line, *_column)), } - Ok((Value::Null, ControlFlow::None)) - } - Statement::PushStatement { - list, - value, - line, - column, - } => { - let list_val = self.evaluate_expression(list, Rc::clone(&env)).await?; - let value_val = self.evaluate_expression(value, Rc::clone(&env)).await?; - - match list_val { - Value::List(list_rc) => { - list_rc.borrow_mut().push(value_val); - Ok((Value::Null, ControlFlow::None)) - } - _ => Err(RuntimeError::new( - format!("Cannot push to non-list value: {list_val:?}"), - *line, - *column, - )), - } + Ok((interface_value, ControlFlow::None)) } - Statement::CreateListStatement { + Statement::EventDefinition { name, - initial_values, - line, - column, + parameters, + line: _line, + column: _column, } => { - // Create a new list with initial values - let mut list_items = Vec::new(); - for value_expr in initial_values { - let value = self - .evaluate_expression(value_expr, Rc::clone(&env)) - .await?; - list_items.push(value); - } + // Create a new event definition + let event_def = ContainerEventValue { + name: name.clone(), + params: parameters.iter().map(|p| p.name.clone()).collect(), + handlers: Vec::new(), + line: *_line, + column: *_column, + }; - let list_value = Value::List(Rc::new(RefCell::new(list_items))); - match env.borrow_mut().define(name, list_value) { + let event_value = Value::ContainerEvent(Rc::new(event_def)); + + // Store the event definition in the environment + match env.borrow_mut().define(name, event_value.clone()) { Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + Err(msg) => return Err(RuntimeError::new(msg, *_line, *_column)), } - Ok((Value::Null, ControlFlow::None)) + Ok((event_value, ControlFlow::None)) } - Statement::MapCreation { + Statement::EventTrigger { name, - entries, - line, - column, + arguments, + line: _line, + column: _column, } => { - // Create a new map/object with initial entries - let mut map = std::collections::HashMap::new(); - for (key, value_expr) in entries { - let value = self - .evaluate_expression(value_expr, Rc::clone(&env)) + // Look up the event + let event = match env.borrow().get(name) { + Some(Value::ContainerEvent(event)) => event.clone(), + _ => { + return Err(RuntimeError::new( + format!("Event '{name}' not found"), + *_line, + *_column, + )); + } + }; + + // Evaluate the arguments + let mut arg_values = Vec::with_capacity(arguments.len()); + for arg in arguments { + let arg_val = self + .evaluate_expression(&arg.value, Rc::clone(&env)) .await?; - map.insert(key.clone(), value); + arg_values.push(arg_val); } - let map_value = Value::Object(Rc::new(RefCell::new(map))); - match env.borrow_mut().define(name, map_value) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + // Execute all event handlers + for handler in &event.handlers { + // Create a new environment for the handler + let handler_env = Environment::new_child_env(&env); + + // Bind arguments to parameters. Use define_direct so a + // parameter shadows any same-named global rather than being + // rejected as already-defined-in-outer-scope (#582). + for (i, param_name) in event.params.iter().enumerate() { + if i < arg_values.len() { + let _ = handler_env + .borrow_mut() + .define_direct(param_name, arg_values[i].clone()); + } else { + let _ = handler_env + .borrow_mut() + .define_direct(param_name, Value::Null); + } + } + + // Execute the handler + self.execute_block(&handler.body, handler_env).await?; } Ok((Value::Null, ControlFlow::None)) } - Statement::CreateDateStatement { - name, - value, - line, - column, + Statement::EventHandler { + event_source, + event_name, + handler_body, + line: _line, + column: _column, } => { - let date_value = if let Some(expr) = value { - // Evaluate the expression to get the date - self.evaluate_expression(expr, Rc::clone(&env)).await? - } else { - // Default to today's date - let today = chrono::Local::now().date_naive(); - Value::Date(Rc::new(today)) - }; + // Evaluate the event source + let source_val = self + .evaluate_expression(event_source, Rc::clone(&env)) + .await?; - match env.borrow_mut().define(name, date_value) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + // Check if the source is a container instance + if let Value::ContainerInstance(instance_rc) = &source_val { + let instance = instance_rc.borrow(); + let container_type = instance.container_type.clone(); + + // Look up the container definition + let container_def = match env.borrow().get(&container_type) { + Some(Value::ContainerDefinition(def)) => def.clone(), + _ => { + return Err(RuntimeError::new( + format!("Container '{container_type}' not found"), + *_line, + *_column, + )); + } + }; + + // Look up the event + if let Some(event) = container_def.events.get(event_name) { + // Create a new event handler + let handler = EventHandler { + body: handler_body.clone(), + env: Rc::downgrade(&env), + line: *_line, + column: *_column, + }; + + // Create a new event with the handler added + let mut handlers = event.handlers.clone(); + handlers.push(handler); + + // Create a new event value + let new_event = ContainerEventValue { + name: event.name.clone(), + params: event.params.clone(), + handlers, + line: event.line, + column: event.column, + }; + + // Store the updated event in the environment + let event_value = Value::ContainerEvent(Rc::new(new_event)); + let _ = env.borrow_mut().define(event_name, event_value.clone()); + + Ok((Value::Null, ControlFlow::None)) + } else { + Err(RuntimeError::new( + format!( + "Event '{event_name}' not found in container '{container_type}'" + ), + *_line, + *_column, + )) + } + } else { + Err(RuntimeError::new( + "Cannot add event handler to non-container value".to_string(), + *_line, + *_column, + )) } - Ok((Value::Null, ControlFlow::None)) } - Statement::CreateTimeStatement { - name, - value, + Statement::ParentMethodCall { + method_name, + arguments, line, column, } => { - let time_value = if let Some(expr) = value { - // Evaluate the expression to get the time - self.evaluate_expression(expr, Rc::clone(&env)).await? - } else { - // Default to current time - let now = chrono::Local::now().time(); - Value::Time(Rc::new(now)) + // Get the current container instance (this) + let this_val = match env.borrow().get("this") { + Some(val) => val.clone(), + None => { + return Err(RuntimeError::new( + "Parent method call can only be used inside a container method" + .to_string(), + *line, + *column, + )); + } }; - match env.borrow_mut().define(name, time_value) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), - } - Ok((Value::Null, ControlFlow::None)) - } - Statement::AddToListStatement { - value, - list_name, - line, - column, - } => { - // Evaluate the value to add - let value_to_add = self.evaluate_expression(value, Rc::clone(&env)).await?; + // Check if this is a container instance + if let Value::ContainerInstance(instance_rc) = &this_val { + // Clone the parent Rc out so the instance's RefCell borrow does + // not span the awaited method call below (a sibling handler + // could otherwise re-borrow the same instance across the yield). + let parent_opt = instance_rc.borrow().parent.clone(); - // Get the list from the environment - let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) - })?; + // Check if the instance has a parent + if let Some(parent_rc) = parent_opt { + // Read the parent's type, then release its borrow too — no + // container RefCell borrow is held across the `.await`s. + let parent_type = parent_rc.borrow().container_type.clone(); - match list_val { - Value::List(list_rc) => { - list_rc.borrow_mut().push(value_to_add); - Ok((Value::Null, ControlFlow::None)) - } - Value::Number(_) => { - // This is actually arithmetic add - // Convert to arithmetic operation - let current = list_val; - if let (Value::Number(n1), Value::Number(n2)) = (¤t, &value_to_add) { - let result = Value::Number(n1 + n2); - env.borrow_mut() - .assign(list_name, result) - .map_err(|e| RuntimeError::new(e, *line, *column))?; - Ok((Value::Null, ControlFlow::None)) + // Look up the parent container definition + let parent_def = match env.borrow().get(&parent_type) { + Some(Value::ContainerDefinition(def)) => def.clone(), + _ => { + return Err(RuntimeError::new( + format!("Parent container '{parent_type}' not found"), + *line, + *column, + )); + } + }; + + // Look up the method in the parent + if let Some(method_val) = parent_def.methods.get(method_name) { + // Create a function value from the method + let function = FunctionValue { + name: Some(method_val.name.clone()), + params: method_val.params.clone(), + param_types: vec![None; method_val.params.len()], + body: method_val.body.clone(), + env: method_val.env.clone(), + line: method_val.line, + column: method_val.column, + enforce_param_types: std::cell::Cell::new(false), + }; + + // Create a new environment for the method execution + let method_env = Environment::new_child_env(&env); + + // Add 'this' to the environment (the current instance, not the parent) + let _ = method_env.borrow_mut().define("this", this_val.clone()); + + // Evaluate the arguments + let mut arg_values = Vec::with_capacity(arguments.len()); + for arg in arguments { + let arg_val = self + .evaluate_expression(&arg.value, Rc::clone(&env)) + .await?; + arg_values.push(arg_val); + } + + // Call the function + let result = self + .call_function(&function, arg_values, *line, *column) + .await?; + + Ok((result, ControlFlow::None)) } else { Err(RuntimeError::new( - "Cannot add non-numeric value to number".to_string(), + format!( + "Method '{method_name}' not found in parent container '{parent_type}'" + ), *line, *column, )) } + } else { + Err(RuntimeError::new( + "Cannot call parent method: no parent container".to_string(), + *line, + *column, + )) } - _ => Err(RuntimeError::new( - format!("Cannot add to non-list value: {list_val:?}"), + } else { + Err(RuntimeError::new( + "Parent method call can only be used inside a container method".to_string(), *line, *column, - )), + )) } } - Statement::RemoveFromListStatement { - value, - list_name, + Statement::PatternDefinition { + name, + pattern, line, column, + .. } => { - // Evaluate the value to remove - let value_to_remove = self.evaluate_expression(value, Rc::clone(&env)).await?; - - // Get the list from the environment - let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) - })?; - - match list_val { - Value::List(list_rc) => { - let mut list = list_rc.borrow_mut(); - // Remove the first occurrence of the value - if let Some(pos) = list.iter().position(|v| v == &value_to_remove) { - list.remove(pos); + // Compile the pattern AST into bytecode with environment access for list references + let compiled_pattern = { + let env_borrow = env.borrow(); + CompiledPattern::compile_with_env(pattern, &env_borrow) + }; + match compiled_pattern { + Ok(compiled_pattern) => { + // Store the compiled pattern in the environment + let pattern_value = Value::Pattern(Rc::new(compiled_pattern)); + match env.borrow_mut().define(name, pattern_value.clone()) { + Ok(_) => {} + Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), } - Ok((Value::Null, ControlFlow::None)) - } - _ => Err(RuntimeError::new( - format!("Cannot remove from non-list value: {list_val:?}"), - *line, - *column, - )), - } - } - Statement::ClearListStatement { - list_name, - line, - column, - } => { - // Get the list from the environment - let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) - })?; - - match list_val { - Value::List(list_rc) => { - list_rc.borrow_mut().clear(); - Ok((Value::Null, ControlFlow::None)) + Ok((pattern_value, ControlFlow::None)) } - _ => Err(RuntimeError::new( - format!("Cannot clear non-list value: {list_val:?}"), - *line, - *column, - )), + Err(compile_error) => Err(RuntimeError { + kind: ErrorKind::General, + message: format!("Failed to compile pattern '{name}': {compile_error}"), + line: *line, + column: *column, + }), } } - // Container-related statements - Statement::ContainerDefinition { - name, - extends, - implements, - properties, - methods, - events, - static_properties: _static_properties, - static_methods: _static_methods, + Statement::ListenStatement { + port, + server_name, + tls, + redirect_to_port, line, column, } => { - // Create a new container definition - let mut container_properties = HashMap::new(); - let mut container_methods = HashMap::new(); - - for prop in properties { - let property_type_str = prop - .property_type - .as_ref() - .map(|ast_type| format!("{ast_type:?}")); - - let default_val = match &prop.default_value { - Some(expr) => { - // Evaluate the default expression to get a Value - (self._evaluate_expression(expr, env.clone()).await).ok() - } - None => None, - }; - - let value_prop = value::PropertyDefinition { - name: prop.name.clone(), - property_type: property_type_str, - default_value: default_val, - validation_rules: Vec::new(), - is_static: false, - is_public: true, - line: prop.line, - column: prop.column, - }; - container_properties.insert(prop.name.clone(), value_prop); - } - - for method in methods { - if let Statement::ActionDefinition { - name, - parameters, - body, - line, - column, - .. - } = method - { - let container_method = ContainerMethodValue { - name: name.clone(), - params: parameters.iter().map(|p| p.name.clone()).collect(), - body: body.clone(), - is_static: false, - is_public: true, - env: Rc::downgrade(&env), - line: *line, - column: *column, - }; - // TODO(#638): container methods do not support - // overloading — a repeated method name silently keeps - // the last definition here. Route same-name methods - // through an overload set (see - // `Environment::define_or_merge_action` / - // `select_overload` for the action equivalent). - container_methods.insert(name.clone(), container_method); + let port_val = self.evaluate_expression(port, Rc::clone(&env)).await?; + let port_num = match &port_val { + Value::Number(n) => *n as u16, + _ => { + return Err(RuntimeError::new( + format!("Expected number for port, got {port_val:?}"), + *line, + *column, + )); } - } - - // Process events - let mut container_events = HashMap::new(); - for event in events { - let container_event = ContainerEventValue { - name: event.name.clone(), - params: event.parameters.iter().map(|p| p.name.clone()).collect(), - handlers: Vec::new(), - line: event.line, - column: event.column, - }; - container_events.insert(event.name.clone(), container_event); - } - - let container_def = ContainerDefinitionValue { - name: name.clone(), - extends: extends.clone(), - implements: implements.clone(), - properties: container_properties, - methods: container_methods, - events: container_events, - static_properties: HashMap::new(), // Future feature - static_methods: HashMap::new(), // Future feature - line: *line, - column: *column, }; - // Create the container definition value - let container_value = Value::ContainerDefinition(Rc::new(container_def)); - - // Store the container definition in the environment - match env.borrow_mut().define(name, container_value.clone()) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), - } - - Ok((container_value, ControlFlow::None)) - } - Statement::ContainerInstantiation { - container_type, - instance_name, - arguments, - property_initializers, - line, - column, - } => { - // Create container instance with inheritance support - let mut instance = self.create_container_instance_with_inheritance( - container_type, - &env, - *line, - *column, - )?; - - // Process property initializers (override inherited properties) - for initializer in property_initializers { - let init_value = self - ._evaluate_expression(&initializer.value, env.clone()) - .await?; - instance - .properties - .insert(initializer.name.clone(), init_value); - } + // Create request/response channels. The request queue is bounded + // (Phase 0, PR-0c) so a flood of accepted-but-unhandled requests + // sheds with 503 instead of growing memory without bound. + let queue_bound = self.budget.max_pending_requests(); + let (request_sender, request_receiver) = + mpsc::channel::(queue_bound); + let request_receiver = Arc::new(tokio::sync::Mutex::new(request_receiver)); - let instance_value = Value::ContainerInstance(Rc::new(RefCell::new(instance))); + // Create warp routes that handle all HTTP methods and paths. + // In-flight admission uses the shared ExecutionBudget's global + // request cap (RequestGuard), so a flood is bounded across every + // listener — not per-server. The guard is acquired *before* the + // body is read and held until the handler answers, the request + // times out, or the client disconnects, so a dequeued request + // can no longer pin memory indefinitely. Body size is enforced + // *while streaming* (below), which bounds chunked bodies that + // carry no Content-Length. + let request_sender_clone = request_sender.clone(); + let max_body_size = self.budget.max_request_body_bytes(); + let max_body_size_u64 = max_body_size as u64; + let request_timeout = self.budget.max_request_duration(); + let admit_budget = Arc::clone(&self.budget); + let routes = warp::any() + .and(warp::method()) + .and(warp::path::full()) + .and(warp::query::raw().or(warp::any().map(String::new)).unify()) + .and(warp::header::headers_cloned()) + // Fast path: reject when the client advertises an oversized + // body, before we admit it or read a byte. Optional header so + // GETs (and chunked bodies) without Content-Length are still + // admitted and then bounded by the streaming check below. + .and( + warp::header::optional::("content-length").and_then( + move |len: Option| async move { + if let Some(len) = len + && len > max_body_size_u64 + { + return Err(warp::reject::custom(PayloadTooLarge)); + } + Ok::<(), warp::Rejection>(()) + }, + ), + ) + // Admission control: reserve a global in-flight slot *before* + // the body is read. At the ceiling, shed with 503 (via the + // `Overloaded` rejection) without reading a body. + .and({ + let admit_budget = Arc::clone(&admit_budget); + warp::any().and_then(move || { + let admit_budget = Arc::clone(&admit_budget); + async move { + admit_budget + .try_acquire_request() + .ok_or_else(|| warp::reject::custom(Overloaded)) + } + }) + }) + .and(warp::body::stream()) + .and(warp::addr::remote()) + .and_then( + move |method: warp::http::Method, + path: warp::path::FullPath, + query: String, + headers: warp::http::HeaderMap, + (), + guard: crate::exec::budget::RequestGuard, + body_stream, + remote_addr: Option| { + let sender = request_sender_clone.clone(); + async move { + // Hold the admission slot for this request's WHOLE + // transport lifetime by binding the guard into this + // future: it drops when the future completes — on a + // delivered response, a response/body timeout, or a + // client disconnect (warp cancels the future) — + // releasing the in-flight slot INDEPENDENTLY of any + // later admitted request. This future stays alive + // awaiting the response (below) even after the + // interpreter dequeues the request, so the slot is + // still held during handling — it is not released at + // dequeue. Parking the guard in the interpreter's + // pending map instead pinned it until a *future* + // dequeued request pruned it, which could never + // happen once the cap was full — permanently wedging + // admission. The bounded request mpsc separately + // caps still-queued bodies, so releasing here does + // not let queued-body memory grow unbounded. + let _admission_guard = guard; - // Store the instance in the environment - match env - .borrow_mut() - .define(instance_name, instance_value.clone()) - { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), - } + // One deadline for the whole accepted-request + // lifetime (body read + handler response), set at + // admission. Applying it to the *body read* is + // what stops a slow "trickle" upload (a chunked + // body dribbled under the size cap forever) from + // pinning its global in-flight slot: without this, + // only the response wait was bounded. + let overall_deadline = request_timeout + .map(|dur| tokio::time::Instant::now() + dur); - // Call constructor method if arguments are provided - if !arguments.is_empty() { - // Look up the container definition to find the initialize method - let container_def = match env.borrow().get(container_type) { - Some(Value::ContainerDefinition(def)) => def.clone(), - _ => { - return Err(RuntimeError::new( - format!("Container '{container_type}' not found"), - *line, - *column, - )); - } - }; + // Enforce the body limit while streaming so a + // chunked body (no Content-Length) is bounded too, + // and bound the read by the shared deadline. + let read_fut = read_body_capped(body_stream, max_body_size); + let body_read = match overall_deadline { + Some(dl) => match tokio::time::timeout_at(dl, read_fut).await { + Ok(r) => r, + Err(_) => { + log::warn!( + "web server request from {} did not finish its body in time; shedding 408", + remote_addr + .map(|a| a.ip().to_string()) + .unwrap_or_else(|| "unknown".to_string()), + ); + return Ok(request_timeout_response() + .map(warp::hyper::Body::from)); + } + }, + None => read_fut.await, + }; + let body_bytes = match body_read { + Ok(bytes) => bytes, + Err(BodyReadError::TooLarge) => { + return Ok( + payload_too_large_response().map(warp::hyper::Body::from) + ); + } + Err(BodyReadError::Io) => { + return Err(warp::reject::custom(ServerError( + "Failed to read request body".to_string(), + ))); + } + }; - // Check if the container has an "initialize" method - if let Some(init_method) = container_def.methods.get("initialize") { - // Create a function value from the initialize method - let init_function = FunctionValue { - name: Some("initialize".to_string()), - params: init_method.params.clone(), - param_types: vec![None; init_method.params.len()], - body: init_method.body.clone(), - env: init_method.env.clone(), - line: init_method.line, - column: init_method.column, - enforce_param_types: std::cell::Cell::new(false), - }; + // Generate unique request ID + let request_id = uuid::Uuid::new_v4().to_string(); - // Create a new environment for the constructor execution - let init_env = Environment::new_child_env(&env); + // Extract client IP + let client_ip = remote_addr + .map(|addr| addr.ip().to_string()) + .unwrap_or_else(|| "unknown".to_string()); - // Add 'this' to the environment (the instance being constructed) - let _ = init_env.borrow_mut().define("this", instance_value.clone()); + // Convert headers to HashMap + let mut header_map = HashMap::new(); + for (name, value) in headers.iter() { + if let Ok(value_str) = value.to_str() { + header_map.insert(name.to_string(), value_str.to_string()); + } + } - // Evaluate the arguments - let mut arg_values = Vec::with_capacity(arguments.len()); - for arg in arguments { - let arg_val = self.evaluate_expression(&arg.value, env.clone()).await?; - arg_values.push(arg_val); - } + // Create response channel + let (response_sender, response_receiver) = + oneshot::channel::(); - // Call the initialize method - self.call_function(&init_function, arg_values, *line, *column) - .await?; - } else if !arguments.is_empty() { - return Err(RuntimeError::new( - format!( - "Container '{container_type}' does not have an initialize method but arguments were provided" - ), - *line, - *column, - )); - } - } + // Create the WFL request. The admission guard is + // NOT moved in — it stays bound to this transport + // future (see `_admission_guard` above), which + // outlives the enqueue and awaits the response, so + // the slot is held through handling and released + // when this future ends (respond/timeout/disconnect). + let wfl_request = WflHttpRequest { + id: request_id, + method: method.to_string(), + path: path.as_str().to_string(), + query, + client_ip, + body: body_bytes, + headers: header_map, + response_sender: Arc::new(tokio::sync::Mutex::new(Some( + response_sender, + ))), + }; - Ok((instance_value, ControlFlow::None)) - } - Statement::InterfaceDefinition { - name, - extends, - required_actions, - line: _line, - column: _column, - } => { - // Create a new interface definition - let mut interface_required_actions = HashMap::new(); + // Send request to WFL interpreter. The queue is + // bounded: a full queue means the interpreter is + // saturated, so shed with 503 rather than + // buffering unbounded work. `try_send` never + // blocks the transport task. + match sender.try_send(wfl_request) { + Ok(()) => {} + Err(mpsc::error::TrySendError::Full(shed)) => { + log::warn!( + "web server request queue full (capacity {}); shedding {} {} from {} with 503", + sender.max_capacity(), + shed.method, + shed.path, + shed.client_ip + ); + return Ok( + overloaded_response().map(warp::hyper::Body::from) + ); + } + Err(mpsc::error::TrySendError::Closed(_)) => { + return Err(warp::reject::custom(ServerError( + "Request channel closed".to_string(), + ))); + } + } - for action in required_actions { - let value_action = value::ActionSignature { - name: action.name.clone(), - params: action.parameters.iter().map(|p| p.name.clone()).collect(), - line: action.line, - column: action.column, - }; - interface_required_actions.insert(action.name.clone(), value_action); - } + // Wait for the handler's response, bounded by the + // *same* deadline as the body read so a handler + // that never answers frees its in-flight slot with + // a 504. Dropping `response_receiver` here closes + // its oneshot sender, which the interpreter + // observes (`is_closed`) to skip/prune the + // abandoned request rather than run zombie work. + let received = match overall_deadline { + Some(dl) => { + match tokio::time::timeout_at(dl, response_receiver).await { + Ok(r) => r, + Err(_) => { + log::warn!( + "web server request from {} timed out awaiting handler; shedding 504", + remote_addr + .map(|a| a.ip().to_string()) + .unwrap_or_else(|| "unknown".to_string()), + ); + return Ok(gateway_timeout_response() + .map(warp::hyper::Body::from)); + } + } + } + None => response_receiver.await, + }; - let interface_def = InterfaceDefinitionValue { - name: name.clone(), - extends: extends.clone(), - required_actions: interface_required_actions, - line: *_line, - column: *_column, - }; + match received { + Ok(HandlerReply::Buffered(response)) => { + let status_code = + warp::http::StatusCode::from_u16(response.status) + .unwrap_or(warp::http::StatusCode::OK); - let interface_value = Value::InterfaceDefinition(Rc::new(interface_def)); + // Content is already raw bytes (text responses + // stored their UTF-8 encoding, binary responses + // their verbatim bytes), so Content-Length is the + // exact byte count of the body. + let content_bytes = response.content; + let content_length = content_bytes.len(); - // Store the interface definition in the environment - match env.borrow_mut().define(name, interface_value.clone()) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *_line, *_column)), - } + let mut reply_builder = warp::http::Response::builder() + .status(status_code) + .header("Content-Type", response.content_type) + .header("Content-Length", content_length); - Ok((interface_value, ControlFlow::None)) - } - Statement::EventDefinition { - name, - parameters, - line: _line, - column: _column, - } => { - // Create a new event definition - let event_def = ContainerEventValue { - name: name.clone(), - params: parameters.iter().map(|p| p.name.clone()).collect(), - handlers: Vec::new(), - line: *_line, - column: *_column, - }; + // Add additional headers + for (name, value) in response.headers { + reply_builder = reply_builder.header(name, value); + } - let event_value = Value::ContainerEvent(Rc::new(event_def)); + match reply_builder.body(warp::hyper::Body::from(content_bytes)) { + Ok(response) => Ok(response), + Err(_) => Err(warp::reject::custom(ServerError( + "Failed to build response".to_string(), + ))), + } + } + Ok(HandlerReply::Streaming { + status, + content_type, + headers, + body, + }) => { + let status_code = warp::http::StatusCode::from_u16(status) + .unwrap_or(warp::http::StatusCode::OK); + + // No Content-Length: the body length is unknown + // up front. hyper frames it with chunked + // transfer-encoding and writes each chunk as it + // arrives off the bounded channel. + let mut reply_builder = warp::http::Response::builder() + .status(status_code) + .header("Content-Type", content_type); + for (name, value) in headers { + reply_builder = reply_builder.header(name, value); + } - // Store the event definition in the environment - match env.borrow_mut().define(name, event_value.clone()) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *_line, *_column)), - } + // Turn the chunk receiver into a body stream. + // When the client disconnects, hyper drops this + // body, dropping `body`, which makes the + // handler's next `write` fail (the interpreter + // observes the closed channel) — that is how a + // browser disconnect cancels the handler. + let stream = futures_util::stream::unfold( + body, + |mut rx| async move { + rx.recv() + .await + .map(|chunk| (Ok::, std::io::Error>(chunk), rx)) + }, + ); + match reply_builder + .body(warp::hyper::Body::wrap_stream(stream)) + { + Ok(response) => Ok(response), + Err(_) => Err(warp::reject::custom(ServerError( + "Failed to build streaming response".to_string(), + ))), + } + } + Err(_) => Err(warp::reject::custom(ServerError( + "Response channel closed".to_string(), + ))), + } + } + }, + ) + .recover(handle_overloaded); - Ok((event_value, ControlFlow::None)) - } - Statement::EventTrigger { - name, - arguments, - line: _line, - column: _column, - } => { - // Look up the event - let event = match env.borrow().get(name) { - Some(Value::ContainerEvent(event)) => event.clone(), - _ => { + // Parse the bind address from config + let bind_addr: IpAddr = match self.config.web_server_bind_address.parse() { + Ok(addr) => addr, + Err(_) => { return Err(RuntimeError::new( - format!("Event '{name}' not found"), - *_line, - *_column, + format!( + "Invalid web_server_bind_address in config: '{}'. Expected a valid IP address (e.g., '127.0.0.1' or '0.0.0.0')", + self.config.web_server_bind_address + ), + *line, + *column, )); } }; - // Evaluate the arguments - let mut arg_values = Vec::with_capacity(arguments.len()); - for arg in arguments { - let arg_val = self - .evaluate_expression(&arg.value, Rc::clone(&env)) + if let Some(target_port_expr) = redirect_to_port { + // Redirect server: answers every request natively with a + // 301 to the HTTPS port. Requests never reach the WFL + // request loop, so `wait for request` on this server + // never fires. + let target_val = self + .evaluate_expression(target_port_expr, Rc::clone(&env)) .await?; - arg_values.push(arg_val); - } - - // Execute all event handlers - for handler in &event.handlers { - // Create a new environment for the handler - let handler_env = Environment::new_child_env(&env); - - // Bind arguments to parameters. Use define_direct so a - // parameter shadows any same-named global rather than being - // rejected as already-defined-in-outer-scope (#582). - for (i, param_name) in event.params.iter().enumerate() { - if i < arg_values.len() { - let _ = handler_env - .borrow_mut() - .define_direct(param_name, arg_values[i].clone()); - } else { - let _ = handler_env - .borrow_mut() - .define_direct(param_name, Value::Null); + // Reject non-integer and out-of-range values instead of + // letting the float->u16 cast saturate to a wrong port + let target_port = match &target_val { + Value::Number(n) if n.fract() == 0.0 && *n >= 1.0 && *n <= 65535.0 => { + *n as u16 } - } - - // Execute the handler - self.execute_block(&handler.body, handler_env).await?; - } - - Ok((Value::Null, ControlFlow::None)) - } - Statement::EventHandler { - event_source, - event_name, - handler_body, - line: _line, - column: _column, - } => { - // Evaluate the event source - let source_val = self - .evaluate_expression(event_source, Rc::clone(&env)) - .await?; - - // Check if the source is a container instance - if let Value::ContainerInstance(instance_rc) = &source_val { - let instance = instance_rc.borrow(); - let container_type = instance.container_type.clone(); - - // Look up the container definition - let container_def = match env.borrow().get(&container_type) { - Some(Value::ContainerDefinition(def)) => def.clone(), _ => { return Err(RuntimeError::new( - format!("Container '{container_type}' not found"), - *_line, - *_column, + format!( + "Expected a whole number between 1 and 65535 for redirect target port, got {target_val:?}" + ), + *line, + *column, )); } }; - // Look up the event - if let Some(event) = container_def.events.get(event_name) { - // Create a new event handler - let handler = EventHandler { - body: handler_body.clone(), - env: Rc::downgrade(&env), - line: *_line, - column: *_column, - }; + let fallback_host = match bind_addr { + IpAddr::V6(v6) => format!("[{v6}]"), + IpAddr::V4(v4) => v4.to_string(), + }; + let redirect_routes = warp::any() + .and(warp::header::optional::("host")) + .and(warp::path::full()) + .and(warp::query::raw().or(warp::any().map(String::new)).unify()) + .map( + move |host: Option, + path: warp::path::FullPath, + query: String| { + let host_value = host.unwrap_or_else(|| fallback_host.clone()); + let mut location = + format!("https://{}", strip_host_port(&host_value)); + if target_port != 443 { + location.push_str(&format!(":{target_port}")); + } + location.push_str(path.as_str()); + if !query.is_empty() { + location.push('?'); + location.push_str(&query); + } + warp::http::Response::builder() + .status(warp::http::StatusCode::MOVED_PERMANENTLY) + .header("Location", location) + .header("Content-Length", 0) + .body(Vec::new()) + .unwrap_or_else(|_| { + let mut resp = warp::http::Response::new(Vec::new()); + *resp.status_mut() = + warp::http::StatusCode::INTERNAL_SERVER_ERROR; + resp + }) + }, + ); - // Create a new event with the handler added - let mut handlers = event.handlers.clone(); - handlers.push(handler); + match warp::serve(redirect_routes).try_bind_ephemeral((bind_addr, port_num)) { + Ok((addr, server)) => { + let server_handle = tokio::spawn(server); - // Create a new event value - let new_event = ContainerEventValue { - name: event.name.clone(), - params: event.params.clone(), - handlers, - line: event.line, - column: event.column, - }; + // Registered like any other server so `close server` + // works; its request channels are never fed. + let wfl_server = WflWebServer { + request_receiver: request_receiver.clone(), + request_sender: request_sender.clone(), + server_handle: Some(server_handle), + }; + self.web_servers + .borrow_mut() + .insert(server_name.clone(), wfl_server); - // Store the updated event in the environment - let event_value = Value::ContainerEvent(Rc::new(new_event)); - let _ = env.borrow_mut().define(event_name, event_value.clone()); + let server_value = Value::Text(Arc::from(format!( + "WebServer::{}:{}", + addr.ip(), + addr.port() + ))); + + println!( + "Redirect server is listening on port {} (redirecting to HTTPS port {})", + addr.port(), + target_port + ); - Ok((Value::Null, ControlFlow::None)) - } else { - Err(RuntimeError::new( - format!( - "Event '{event_name}' not found in container '{container_type}'" - ), - *_line, - *_column, - )) - } - } else { - Err(RuntimeError::new( - "Cannot add event handler to non-container value".to_string(), - *_line, - *_column, - )) - } - } - Statement::ParentMethodCall { - method_name, - arguments, - line, - column, - } => { - // Get the current container instance (this) - let this_val = match env.borrow().get("this") { - Some(val) => val.clone(), - None => { - return Err(RuntimeError::new( - "Parent method call can only be used inside a container method" - .to_string(), + match env.borrow_mut().define(server_name, server_value) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + Err(e) => Err(RuntimeError::new( + format!("Failed to start web server on port {}: {}", port_num, e), *line, *column, - )); + )), } - }; - - // Check if this is a container instance - if let Value::ContainerInstance(instance_rc) = &this_val { - let instance = instance_rc.borrow(); - - // Check if the instance has a parent - if let Some(parent_rc) = &instance.parent { - let parent = parent_rc.borrow(); - let parent_type = parent.container_type.clone(); - - // Look up the parent container definition - let parent_def = match env.borrow().get(&parent_type) { - Some(Value::ContainerDefinition(def)) => def.clone(), - _ => { + } else if let Some(tls_config) = tls { + // HTTPS server. Certificate/key paths come from the listen + // statement itself, falling back to .wflcfg for the bare + // `secured` form. + let cert_path = match &tls_config.cert_path { + Some(expr) => { + let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; + match &v { + Value::Text(t) => t.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected text for TLS certificate path, got {v:?}" + ), + *line, + *column, + )); + } + } + } + None => match &self.config.web_server_tls_cert_file { + Some(path) => path.clone(), + None => { return Err(RuntimeError::new( - format!("Parent container '{parent_type}' not found"), + "This listen statement is marked 'secured' but no certificate is configured. Either write 'secured with certificate \"cert.pem\" and key \"key.pem\"' or set web_server_tls_cert_file and web_server_tls_key_file in .wflcfg".to_string(), *line, *column, )); } - }; + }, + }; + let key_path = match &tls_config.key_path { + Some(expr) => { + let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; + match &v { + Value::Text(t) => t.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected text for TLS private key path, got {v:?}" + ), + *line, + *column, + )); + } + } + } + None => match &self.config.web_server_tls_key_file { + Some(path) => path.clone(), + None => { + return Err(RuntimeError::new( + "This listen statement is marked 'secured' but no private key is configured. Either write 'secured with certificate \"cert.pem\" and key \"key.pem\"' or set web_server_tls_cert_file and web_server_tls_key_file in .wflcfg".to_string(), + *line, + *column, + )); + } + }, + }; - // Look up the method in the parent - if let Some(method_val) = parent_def.methods.get(method_name) { - // Create a function value from the method - let function = FunctionValue { - name: Some(method_val.name.clone()), - params: method_val.params.clone(), - param_types: vec![None; method_val.params.len()], - body: method_val.body.clone(), - env: method_val.env.clone(), - line: method_val.line, - column: method_val.column, - enforce_param_types: std::cell::Cell::new(false), - }; + // Validate up front for actionable errors; warp would + // otherwise surface a bad certificate as a generic + // bind-time failure. + if let Err(msg) = validate_tls_pem_files(&cert_path, &key_path) { + return Err(RuntimeError::new(msg, *line, *column)); + } - // Create a new environment for the method execution - let method_env = Environment::new_child_env(&env); + // try_bind_with_graceful_shutdown is the only TlsServer + // constructor that returns bind/TLS errors instead of + // panicking inside the spawned task; the never-completing + // signal keeps the server running until `close server` + // aborts its task. + match warp::serve(routes) + .tls() + .cert_path(&cert_path) + .key_path(&key_path) + .try_bind_with_graceful_shutdown( + (bind_addr, port_num), + std::future::pending::<()>(), + ) { + Ok((addr, server)) => { + let server_handle = tokio::spawn(server); - // Add 'this' to the environment (the current instance, not the parent) - let _ = method_env.borrow_mut().define("this", this_val.clone()); + let wfl_server = WflWebServer { + request_receiver: request_receiver.clone(), + request_sender: request_sender.clone(), + server_handle: Some(server_handle), + }; + self.web_servers + .borrow_mut() + .insert(server_name.clone(), wfl_server); - // Evaluate the arguments - let mut arg_values = Vec::with_capacity(arguments.len()); - for arg in arguments { - let arg_val = self - .evaluate_expression(&arg.value, Rc::clone(&env)) - .await?; - arg_values.push(arg_val); - } + let server_value = Value::Text(Arc::from(format!( + "WebServer::{}:{}", + addr.ip(), + addr.port() + ))); - // Call the function - let result = self - .call_function(&function, arg_values, *line, *column) - .await?; + println!("Secure server is listening on port {}", addr.port()); - Ok((result, ControlFlow::None)) - } else { - Err(RuntimeError::new( - format!( - "Method '{method_name}' not found in parent container '{parent_type}'" - ), - *line, - *column, - )) + match env.borrow_mut().define(server_name, server_value) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } } - } else { - Err(RuntimeError::new( - "Cannot call parent method: no parent container".to_string(), + Err(e) => Err(RuntimeError::new( + format!( + "Failed to start secure web server on port {}: {}", + port_num, e + ), *line, *column, - )) + )), } } else { - Err(RuntimeError::new( - "Parent method call can only be used inside a container method".to_string(), - *line, - *column, - )) - } - } - Statement::PatternDefinition { - name, - pattern, - line, - column, - .. - } => { - // Compile the pattern AST into bytecode with environment access for list references - let compiled_pattern = { - let env_borrow = env.borrow(); - CompiledPattern::compile_with_env(pattern, &env_borrow) - }; - match compiled_pattern { - Ok(compiled_pattern) => { - // Store the compiled pattern in the environment - let pattern_value = Value::Pattern(Rc::new(compiled_pattern)); - match env.borrow_mut().define(name, pattern_value.clone()) { - Ok(_) => {} - Err(msg) => return Err(RuntimeError::new(msg, *line, *column)), + // Plain HTTP server (unchanged behavior) + let server_task = warp::serve(routes).try_bind_ephemeral((bind_addr, port_num)); + + match server_task { + Ok((addr, server)) => { + // Spawn the server in the background + let server_handle = tokio::spawn(server); + + // Create WFL web server object + let wfl_server = WflWebServer { + request_receiver: request_receiver.clone(), + request_sender: request_sender.clone(), + server_handle: Some(server_handle), + }; + + // Store the server in the interpreter + self.web_servers + .borrow_mut() + .insert(server_name.clone(), wfl_server); + + // Create a server value with the actual address + let server_value = Value::Text(Arc::from(format!( + "WebServer::{}:{}", + addr.ip(), + addr.port() + ))); + + println!("Server is listening on port {}", addr.port()); + + match env.borrow_mut().define(server_name, server_value) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } } - Ok((pattern_value, ControlFlow::None)) + Err(e) => Err(RuntimeError::new( + format!("Failed to start web server on port {}: {}", port_num, e), + *line, + *column, + )), } - Err(compile_error) => Err(RuntimeError { - kind: ErrorKind::General, - message: format!("Failed to compile pattern '{name}': {compile_error}"), - line: *line, - column: *column, - }), } } - Statement::ListenStatement { - port, - server_name, - tls, - redirect_to_port, + Statement::WaitForRequestStatement { + server, + request_name, + timeout, line, column, } => { - let port_val = self.evaluate_expression(port, Rc::clone(&env)).await?; - let port_num = match &port_val { - Value::Number(n) => *n as u16, + // Look up the server by name + let server_name = match self.evaluate_expression(server, Rc::clone(&env)).await? { + Value::Text(name) => { + // Extract server name from "WebServer::host:port" format + let name_str = name.as_ref(); + if name_str.starts_with("WebServer::") { + // Find the server by matching the exact server value + let web_servers = self.web_servers.borrow(); + + // Search through all servers to find which one has this exact value + let mut found_server = None; + for server_name in web_servers.keys() { + // Get the stored value for this server name + if let Some(Value::Text(stored_text)) = + env.borrow().get(server_name) + && stored_text.as_ref() == name_str + { + // Found the matching server + found_server = Some(server_name.clone()); + break; + } + } + + // Return the found server or use first server as fallback + if let Some(server_name) = found_server { + server_name + } else if let Some((found_name, _)) = web_servers.iter().next() { + found_name.clone() + } else { + return Err(RuntimeError::new( + "No web servers found".to_string(), + *line, + *column, + )); + } + } else { + name_str.to_string() + } + } _ => { return Err(RuntimeError::new( - format!("Expected number for port, got {port_val:?}"), + "Expected server name as text".to_string(), *line, *column, )); } }; - // Create request/response channels. The request queue is bounded - // (Phase 0, PR-0c) so a flood of accepted-but-unhandled requests - // sheds with 503 instead of growing memory without bound. - let queue_bound = self.budget.max_pending_requests(); - let (request_sender, request_receiver) = - mpsc::channel::(queue_bound); - let request_receiver = Arc::new(tokio::sync::Mutex::new(request_receiver)); + // Get the server's request receiver + let request_receiver = { + let web_servers = self.web_servers.borrow(); + if let Some(server) = web_servers.get(&server_name) { + server.request_receiver.clone() + } else { + return Err(RuntimeError::new( + format!("Web server '{}' not found", server_name), + *line, + *column, + )); + } + }; - // Create warp routes that handle all HTTP methods and paths. - // In-flight admission uses the shared ExecutionBudget's global - // request cap (RequestGuard), so a flood is bounded across every - // listener — not per-server. The guard is acquired *before* the - // body is read and held until the handler answers, the request - // times out, or the client disconnects, so a dequeued request - // can no longer pin memory indefinitely. Body size is enforced - // *while streaming* (below), which bounds chunked bodies that - // carry no Content-Length. - let request_sender_clone = request_sender.clone(); - let max_body_size = self.budget.max_request_body_bytes(); - let max_body_size_u64 = max_body_size as u64; - let request_timeout = self.budget.max_request_duration(); - let admit_budget = Arc::clone(&self.budget); - let routes = warp::any() - .and(warp::method()) - .and(warp::path::full()) - .and(warp::query::raw().or(warp::any().map(String::new)).unify()) - .and(warp::header::headers_cloned()) - // Fast path: reject when the client advertises an oversized - // body, before we admit it or read a byte. Optional header so - // GETs (and chunked bodies) without Content-Length are still - // admitted and then bounded by the streaming check below. - .and( - warp::header::optional::("content-length").and_then( - move |len: Option| async move { - if let Some(len) = len - && len > max_body_size_u64 - { - return Err(warp::reject::custom(PayloadTooLarge)); - } - Ok::<(), warp::Rejection>(()) - }, - ), - ) - // Admission control: reserve a global in-flight slot *before* - // the body is read. At the ceiling, shed with 503 (via the - // `Overloaded` rejection) without reading a body. - .and({ - let admit_budget = Arc::clone(&admit_budget); - warp::any().and_then(move || { - let admit_budget = Arc::clone(&admit_budget); - async move { - admit_budget - .try_acquire_request() - .ok_or_else(|| warp::reject::custom(Overloaded)) - } - }) - }) - .and(warp::body::stream()) - .and(warp::addr::remote()) - .and_then( - move |method: warp::http::Method, - path: warp::path::FullPath, - query: String, - headers: warp::http::HeaderMap, - (), - guard: crate::exec::budget::RequestGuard, - body_stream, - remote_addr: Option| { - let sender = request_sender_clone.clone(); - async move { - // Hold the admission slot for this request's WHOLE - // transport lifetime by binding the guard into this - // future: it drops when the future completes — on a - // delivered response, a response/body timeout, or a - // client disconnect (warp cancels the future) — - // releasing the in-flight slot INDEPENDENTLY of any - // later admitted request. This future stays alive - // awaiting the response (below) even after the - // interpreter dequeues the request, so the slot is - // still held during handling — it is not released at - // dequeue. Parking the guard in the interpreter's - // pending map instead pinned it until a *future* - // dequeued request pruned it, which could never - // happen once the cap was full — permanently wedging - // admission. The bounded request mpsc separately - // caps still-queued bodies, so releasing here does - // not let queued-body memory grow unbounded. - let _admission_guard = guard; + // Wait for a request to come in (with optional timeout) + let request = { + let mut receiver = request_receiver.lock().await; - // One deadline for the whole accepted-request - // lifetime (body read + handler response), set at - // admission. Applying it to the *body read* is - // what stops a slow "trickle" upload (a chunked - // body dribbled under the size cap forever) from - // pinning its global in-flight slot: without this, - // only the response wait was bounded. - let overall_deadline = request_timeout - .map(|dur| tokio::time::Instant::now() + dur); + // Evaluate timeout if provided + let timeout_duration = if let Some(timeout_expr) = timeout { + let timeout_val = self + .evaluate_expression(timeout_expr, Rc::clone(&env)) + .await?; + match timeout_val { + // Reject fractional values that would truncate to 0 ms + // (e.g. 0.5) and hot-spin forever with zero-duration + // timeouts. Require at least 1 millisecond. + Value::Number(ms) if ms >= 1.0 => { + Some(std::time::Duration::from_millis(ms as u64)) + } + Value::Number(ms) if ms > 0.0 => { + return Err(RuntimeError::new( + format!( + "Timeout must be at least 1 millisecond (got {ms} ms); \ + fractional values below 1 would truncate to zero and spin" + ), + *line, + *column, + )); + } + _ => { + return Err(RuntimeError::new( + "Timeout must be a positive number (milliseconds)".to_string(), + *line, + *column, + )); + } + } + } else { + None + }; - // Enforce the body limit while streaming so a - // chunked body (no Content-Length) is bounded too, - // and bound the read by the shared deadline. - let read_fut = read_body_capped(body_stream, max_body_size); - let body_read = match overall_deadline { - Some(dl) => match tokio::time::timeout_at(dl, read_fut).await { - Ok(r) => r, - Err(_) => { - log::warn!( - "web server request from {} did not finish its body in time; shedding 408", - remote_addr - .map(|a| a.ip().to_string()) - .unwrap_or_else(|| "unknown".to_string()), - ); - return Ok(request_timeout_response()); - } - }, - None => read_fut.await, - }; - let body_bytes = match body_read { - Ok(bytes) => bytes, - Err(BodyReadError::TooLarge) => { - return Ok(payload_too_large_response()); - } - Err(BodyReadError::Io) => { - return Err(warp::reject::custom(ServerError( - "Failed to read request body".to_string(), - ))); - } - }; + // Wait for request with or without timeout. Loop so a request + // whose client already gave up (its oneshot receiver dropped + // on 408/504/disconnect, closing the sender) is skipped rather + // than handled — otherwise the interpreter would run a handler + // for a dead request and register a dead pending-response + // entry, letting repeated timeouts accumulate zombie work. + loop { + let req = if let Some(duration) = timeout_duration { + match tokio::time::timeout(duration, receiver.recv()).await { + Ok(Some(req)) => req, + Ok(None) => { + return Err(RuntimeError::new( + "Request channel closed".to_string(), + *line, + *column, + )); + } + Err(_) => { + // Finite wait expiry is an expected idle-server + // outcome, not a structural fault. Classify as + // `Timeout` so the concurrent loop's consecutive- + // failure breaker does not tear a healthy server + // down after enough empty poll intervals. + return Err(RuntimeError::with_kind( + format!( + "{REQUEST_WAIT_TIMEOUT_PREFIX} ({} ms)", + duration.as_millis() + ), + *line, + *column, + ErrorKind::Timeout, + )); + } + } + } else { + // No timeout - wait indefinitely + match receiver.recv().await { + Some(req) => req, + None => { + return Err(RuntimeError::new( + "Request channel closed".to_string(), + *line, + *column, + )); + } + } + }; - // Generate unique request ID - let request_id = uuid::Uuid::new_v4().to_string(); + let abandoned = { + let sender_opt = req.response_sender.lock().await; + sender_opt.as_ref().is_none_or(|s| s.is_closed()) + }; + if abandoned { + log::debug!( + "skipping abandoned request {} ({} {}) from {}", + req.id, + req.method, + req.path, + req.client_ip + ); + continue; + } + break req; + } + }; - // Extract client IP - let client_ip = remote_addr - .map(|addr| addr.ip().to_string()) - .unwrap_or_else(|| "unknown".to_string()); + // Define individual variables for request properties (more natural for WFL) + let mut env_mut = env.borrow_mut(); - // Convert headers to HashMap - let mut header_map = HashMap::new(); - for (name, value) in headers.iter() { - if let Ok(value_str) = value.to_str() { - header_map.insert(name.to_string(), value_str.to_string()); - } - } + // Convert headers to a WFL object (shared by the request object and + // the standalone headers variable defined below) + let mut headers_map = HashMap::new(); + for (key, value) in request.headers.iter() { + headers_map.insert(key.clone(), Value::Text(Arc::from(value.clone()))); + } + let headers_object = Value::Object(Rc::new(RefCell::new(headers_map))); - // Create response channel - let (response_sender, response_receiver) = - oneshot::channel::(); + // Define the main request variable (for use in respond statements and + // as request context for `execute file ... with `) + let mut request_properties = HashMap::new(); + request_properties.insert( + "_response_sender".to_string(), + Value::Text(Arc::from(request.id.clone())), + ); + request_properties.insert( + "method".to_string(), + Value::Text(Arc::from(request.method.clone())), + ); + request_properties.insert( + "path".to_string(), + Value::Text(Arc::from(request.path.clone())), + ); + request_properties.insert( + "query".to_string(), + Value::Text(Arc::from(request.query.clone())), + ); + request_properties.insert( + "client_ip".to_string(), + Value::Text(Arc::from(request.client_ip.clone())), + ); + // `body` is a lossy-UTF-8 text view (backward compatible); + // `body_bytes` is the lossless binary view for binary uploads. + let body_text = String::from_utf8_lossy(&request.body).into_owned(); + let body_binary = Value::Binary(Arc::from(request.body.as_slice())); + request_properties.insert( + "body".to_string(), + Value::Text(Arc::from(body_text.as_str())), + ); + request_properties.insert("body_bytes".to_string(), body_binary.clone()); + request_properties.insert("headers".to_string(), headers_object.clone()); + let request_object = Value::Object(Rc::new(RefCell::new(request_properties))); - // Create the WFL request. The admission guard is - // NOT moved in — it stays bound to this transport - // future (see `_admission_guard` above), which - // outlives the enqueue and awaits the response, so - // the slot is held through handling and released - // when this future ends (respond/timeout/disconnect). - let wfl_request = WflHttpRequest { - id: request_id, - method: method.to_string(), - path: path.as_str().to_string(), - query, - client_ip, - body: body_bytes, - headers: header_map, - response_sender: Arc::new(tokio::sync::Mutex::new(Some( - response_sender, - ))), - }; + // These bindings are refreshed on every wait, so overwrite any + // previous request's values instead of failing on redefinition. + env_mut.define_or_replace(request_name, request_object); - // Send request to WFL interpreter. The queue is - // bounded: a full queue means the interpreter is - // saturated, so shed with 503 rather than - // buffering unbounded work. `try_send` never - // blocks the transport task. - match sender.try_send(wfl_request) { - Ok(()) => {} - Err(mpsc::error::TrySendError::Full(shed)) => { - log::warn!( - "web server request queue full (capacity {}); shedding {} {} from {} with 503", - sender.max_capacity(), - shed.method, - shed.path, - shed.client_ip - ); - return Ok(overloaded_response()); - } - Err(mpsc::error::TrySendError::Closed(_)) => { - return Err(warp::reject::custom(ServerError( - "Request channel closed".to_string(), - ))); - } - } + // Define individual request property variables + env_mut.define_or_replace("method", Value::Text(Arc::from(request.method.clone()))); - // Wait for the handler's response, bounded by the - // *same* deadline as the body read so a handler - // that never answers frees its in-flight slot with - // a 504. Dropping `response_receiver` here closes - // its oneshot sender, which the interpreter - // observes (`is_closed`) to skip/prune the - // abandoned request rather than run zombie work. - let received = match overall_deadline { - Some(dl) => { - match tokio::time::timeout_at(dl, response_receiver).await { - Ok(r) => r, - Err(_) => { - log::warn!( - "web server request from {} timed out awaiting handler; shedding 504", - remote_addr - .map(|a| a.ip().to_string()) - .unwrap_or_else(|| "unknown".to_string()), - ); - return Ok(gateway_timeout_response()); - } - } - } - None => response_receiver.await, - }; + env_mut.define_or_replace("path", Value::Text(Arc::from(request.path.clone()))); - match received { - Ok(response) => { - let status_code = - warp::http::StatusCode::from_u16(response.status) - .unwrap_or(warp::http::StatusCode::OK); + env_mut.define_or_replace("query", Value::Text(Arc::from(request.query.clone()))); - // Content is already raw bytes (text responses - // stored their UTF-8 encoding, binary responses - // their verbatim bytes), so Content-Length is the - // exact byte count of the body. - let content_bytes = response.content; - let content_length = content_bytes.len(); + env_mut.define_or_replace( + "client_ip", + Value::Text(Arc::from(request.client_ip.clone())), + ); - let mut reply_builder = warp::http::Response::builder() - .status(status_code) - .header("Content-Type", response.content_type) - .header("Content-Length", content_length); + env_mut.define_or_replace("body", Value::Text(Arc::from(body_text.as_str()))); + env_mut.define_or_replace("body_bytes", body_binary); - // Add additional headers - for (name, value) in response.headers { - reply_builder = reply_builder.header(name, value); - } + env_mut.define_or_replace("headers", headers_object); - match reply_builder.body(content_bytes) { - Ok(response) => Ok(response), - Err(_) => Err(warp::reject::custom(ServerError( - "Failed to build response".to_string(), - ))), - } - } - Err(_) => Err(warp::reject::custom(ServerError( - "Response channel closed".to_string(), - ))), - } - } + drop(env_mut); // Release the borrow + + // Store the request in a global map for RespondStatement to access. + // Done only after every define above succeeded: registering earlier + // would park the oneshot sender on an error path and leave the HTTP + // client hanging instead of failing fast. + { + let mut pending_responses = self.pending_responses.borrow_mut(); + // Prune entries whose client already disconnected/timed out + // (oneshot sender closed) before inserting the new one, so a + // handler that never `respond`s to a since-abandoned request + // cannot let the map grow without bound across many timeouts. + // (The admission slot itself is released by the transport task, + // not this prune — see `PendingResponse`.) + pending_responses.retain(|_, pending| match pending.sender.try_lock() { + Ok(guard) => guard.as_ref().is_some_and(|s| !s.is_closed()), + // Locked right now (being responded to) — keep it. + Err(_) => true, + }); + pending_responses.insert( + request.id.clone(), + PendingResponse { + sender: request.response_sender, }, + ); + } + // Track it against the current handler so it is answered 500 if + // the handler ends without responding (rather than the client + // waiting out the request timeout). + self.open_pending_requests + .borrow_mut() + .push(request.id.clone()); + // Sticky: this handler accepted work. Request-local failures from + // here on must not trip the concurrent structural-failure breaker. + self.accepted_request.set(true); + + Ok((Value::Null, ControlFlow::None)) + } + Statement::RespondStatement { + request, + content, + status, + content_type, + headers, + line, + column, + } => { + // The request operand can call actions or wait asynchronously. + // Establish the response-attempt baseline before evaluating it + // so a client disconnect cancels that work too. + let (request_val, response_snapshot) = self + .evaluate_response_request( + *line, + *column, + "Client disconnected before the response request was resolved", + self.evaluate_expression(request, Rc::clone(&env)), ) - .recover(handle_overloaded); + .await?; + let request_id = match &request_val { + Value::Object(obj) => { + let obj_ref = obj.borrow(); + match obj_ref.get("_response_sender") { + Some(Value::Text(id)) => id.as_ref().to_string(), + _ => { + return Err(RuntimeError::new( + "Request object missing response sender ID".to_string(), + *line, + *column, + )); + } + } + } + _ => { + return Err(RuntimeError::new( + "Expected request object".to_string(), + *line, + *column, + )); + } + }; - // Parse the bind address from config - let bind_addr: IpAddr = match self.config.web_server_bind_address.parse() { - Ok(addr) => addr, - Err(_) => { + // Keep the pending request parked until content/status/header + // expressions are evaluated so a browser disconnect still cancels + // any upstream open/read performed during that evaluation + // (`any_pending_request_disconnected` watches the parked sender). + // Only after evaluation do we take the sender into the completion + // guard. Early eval errors leave the id in open_pending so the + // handler-exit 500 path still resolves the client. + let (response, response_snapshot) = self + .evaluate_response_precommit( + &request_id, + *line, + *column, + "Client disconnected before the response was sent", + response_snapshot, + async { + // Evaluate response content. Binary values are carried through + // as raw bytes so fonts/images/etc. serve losslessly; text and + // scalar values keep their existing UTF-8 rendering. + let content_val = + self.evaluate_expression(content, Rc::clone(&env)).await?; + let is_binary = matches!(content_val, Value::Binary(_)); + + // Enforce the response-body ceiling on the *borrowed* length + // first, so an oversized Text/Binary body is refused before it is + // duplicated into `content_bytes` (bounding peak allocation). + if let Value::Text(text) = &content_val + && let Err(exceeded) = self.budget.check_response_bytes(text.len()) + { + return Err(self.budget_error(exceeded, *line, *column)); + } + if let Value::Binary(bytes) = &content_val + && let Err(exceeded) = self.budget.check_response_bytes(bytes.len()) + { + return Err(self.budget_error(exceeded, *line, *column)); + } + + let content_bytes: Vec = match &content_val { + Value::Text(text) => text.as_bytes().to_vec(), + Value::Number(n) => n.to_string().into_bytes(), + Value::Bool(b) => b.to_string().into_bytes(), + Value::Binary(bytes) => bytes.to_vec(), + Value::Null => Vec::new(), + // Composite/opaque values (lists, objects, functions, …) have + // no meaningful HTTP body rendering, and their `{:?}` form is + // unbounded — materializing it would allocate past the + // response cap before it could be checked. Reject them with a + // clear error instead. + other => { return Err(RuntimeError::new( format!( - "Invalid web_server_bind_address in config: '{}'. Expected a valid IP address (e.g., '127.0.0.1' or '0.0.0.0')", - self.config.web_server_bind_address + "Cannot use {} as a response body; respond with text, a number, a boolean, binary data, or nothing", + other.type_name() ), *line, *column, @@ -7153,753 +10253,811 @@ impl Interpreter { } }; - if let Some(target_port_expr) = redirect_to_port { - // Redirect server: answers every request natively with a - // 301 to the HTTPS port. Requests never reach the WFL - // request loop, so `wait for request` on this server - // never fires. - let target_val = self - .evaluate_expression(target_port_expr, Rc::clone(&env)) + // Re-check the materialized length to cover the small formatted + // variants (Number/Bool), which have no cheap borrowed length. + if let Err(exceeded) = self.budget.check_response_bytes(content_bytes.len()) { + return Err(self.budget_error(exceeded, *line, *column)); + } + + // Evaluate status code (optional) + let status_code = if let Some(status_expr) = status { + let status_val = self + .evaluate_expression(status_expr, Rc::clone(&env)) .await?; - // Reject non-integer and out-of-range values instead of - // letting the float->u16 cast saturate to a wrong port - let target_port = match &target_val { - Value::Number(n) if n.fract() == 0.0 && *n >= 1.0 && *n <= 65535.0 => { - *n as u16 + match &status_val { + Value::Number(n) => *n as u16, + _ => { + return Err(RuntimeError::new( + "Status code must be a number".to_string(), + *line, + *column, + )); } + } + } else { + 200 // Default to 200 OK + }; + + // Evaluate content type (optional) + let content_type_str = if let Some(ct_expr) = content_type { + let ct_val = self.evaluate_expression(ct_expr, Rc::clone(&env)).await?; + match &ct_val { + Value::Text(text) => text.as_ref().to_string(), _ => { return Err(RuntimeError::new( - format!( - "Expected a whole number between 1 and 65535 for redirect target port, got {target_val:?}" - ), + "Content type must be text".to_string(), *line, *column, )); } - }; + } + } else if is_binary { + // Binary responses default to a generic binary media type + // rather than text/plain so browsers don't misinterpret them. + "application/octet-stream".to_string() + } else { + "text/plain".to_string() // Default content type + }; - let fallback_host = match bind_addr { - IpAddr::V6(v6) => format!("[{v6}]"), - IpAddr::V4(v4) => v4.to_string(), - }; - let redirect_routes = warp::any() - .and(warp::header::optional::("host")) - .and(warp::path::full()) - .and(warp::query::raw().or(warp::any().map(String::new)).unify()) - .map( - move |host: Option, - path: warp::path::FullPath, - query: String| { - let host_value = host.unwrap_or_else(|| fallback_host.clone()); - let mut location = - format!("https://{}", strip_host_port(&host_value)); - if target_port != 443 { - location.push_str(&format!(":{target_port}")); - } - location.push_str(path.as_str()); - if !query.is_empty() { - location.push('?'); - location.push_str(&query); + // Evaluate custom response headers (optional). Mirrors the + // outbound client's headers map: a WFL Object of name -> value. + // Enables RFC 10008 (HTTP QUERY) servers to advertise + // `Accept-Query` and point at results with `Content-Location` + // or `Location`. + let mut custom_headers: HashMap = HashMap::new(); + if let Some(headers_expr) = headers { + let headers_val = self + .evaluate_expression(headers_expr, Rc::clone(&env)) + .await?; + match &headers_val { + Value::Object(obj) => { + for (name, value) in obj.borrow().iter() { + let value_str = match value { + Value::Text(s) => s.to_string(), + Value::Number(_) | Value::Bool(_) => value.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Response header '{name}' must be text, a number, or a boolean, got {}", + value.type_name() + ), + *line, + *column, + )); + } + }; + // Content-Type, Content-Length, and + // Transfer-Encoding are computed by the response + // pipeline (the `content_type` clause and warp's + // builder set them explicitly). Warp *appends* + // custom headers, so letting the map override + // these would emit duplicate/conflicting headers + // (RFC 7230 §3.3.2) and risk response splitting. + // Drop them so the pipeline stays authoritative. + if name.eq_ignore_ascii_case("content-type") + || name.eq_ignore_ascii_case("content-length") + || name.eq_ignore_ascii_case("transfer-encoding") + { + continue; } - warp::http::Response::builder() - .status(warp::http::StatusCode::MOVED_PERMANENTLY) - .header("Location", location) - .header("Content-Length", 0) - .body(Vec::new()) - .unwrap_or_else(|_| { - let mut resp = warp::http::Response::new(Vec::new()); - *resp.status_mut() = - warp::http::StatusCode::INTERNAL_SERVER_ERROR; - resp - }) - }, - ); - - match warp::serve(redirect_routes).try_bind_ephemeral((bind_addr, port_num)) { - Ok((addr, server)) => { - let server_handle = tokio::spawn(server); - - // Registered like any other server so `close server` - // works; its request channels are never fed. - let wfl_server = WflWebServer { - request_receiver: request_receiver.clone(), - request_sender: request_sender.clone(), - server_handle: Some(server_handle), - }; - self.web_servers - .borrow_mut() - .insert(server_name.clone(), wfl_server); + custom_headers.insert(name.clone(), value_str); + } + } + _ => { + return Err(RuntimeError::new( + format!( + "Expected a map for response headers, got {}", + headers_val.type_name() + ), + *line, + *column, + )); + } + } + } - let server_value = Value::Text(Arc::from(format!( - "WebServer::{}:{}", - addr.ip(), - addr.port() - ))); + Ok(WflHttpResponse { + content: content_bytes, + status: status_code, + content_type: content_type_str, + headers: custom_headers, + }) + }, + ) + .await?; - println!( - "Redirect server is listening on port {} (redirecting to HTTPS port {})", - addr.port(), - target_port - ); + // Now commit: take the sender (disconnect signal ends) and deliver. + let mut completion = match self + .take_pending_response_completion(&request_id, *line, *column) + .await + { + Ok(completion) => completion, + Err(error) => { + if error.kind == ErrorKind::Cancelled { + self.cancel_response_precommit(&request_id, response_snapshot); + } + return Err(error); + } + }; + match completion.take_sender() { + Some(sender) => { + if sender.send(HandlerReply::Buffered(response)).is_err() { + // Receiver dropped => the client hung up before the + // buffered reply landed. That is a cooperative + // cancellation, not a handler fault — mark it `Cancelled` + // so the concurrent loop's structural-failure breaker + // skips it (a burst of post-dequeue disconnects must not + // tear the server down). + self.cancel_response_precommit(&request_id, response_snapshot); + return Err(RuntimeError::with_kind( + "Client disconnected before the response was sent".to_string(), + *line, + *column, + ErrorKind::Cancelled, + )); + } + } + None => { + return Err(RuntimeError::new( + "Response already sent for this request".to_string(), + *line, + *column, + )); + } + } - match env.borrow_mut().define(server_name, server_value) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } + Ok((Value::Null, ControlFlow::None)) + } + Statement::StartStreamingResponseStatement { + request, + status, + content_type, + headers, + variable_name, + line, + column, + } => { + // Resolve the request id under the same cancellation baseline as + // the streaming-head fields and final transport commit. + let (request_val, response_snapshot) = self + .evaluate_response_request( + *line, + *column, + "Client disconnected before the streaming response request was resolved", + self.evaluate_expression(request, Rc::clone(&env)), + ) + .await?; + let request_id = match &request_val { + Value::Object(obj) => match obj.borrow().get("_response_sender") { + Some(Value::Text(id)) => id.as_ref().to_string(), + _ => { + return Err(RuntimeError::new( + "Request object missing response sender ID".to_string(), + *line, + *column, + )); } - Err(e) => Err(RuntimeError::new( - format!("Failed to start web server on port {}: {}", port_num, e), + }, + _ => { + return Err(RuntimeError::new( + "Expected request object".to_string(), *line, *column, - )), + )); } - } else if let Some(tls_config) = tls { - // HTTPS server. Certificate/key paths come from the listen - // statement itself, falling back to .wflcfg for the bare - // `secured` form. - let cert_path = match &tls_config.cert_path { - Some(expr) => { - let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; - match &v { - Value::Text(t) => t.to_string(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected text for TLS certificate path, got {v:?}" - ), - *line, - *column, - )); - } + }; + + // Keep pending parked through status/content-type/header + // evaluation so disconnect still cancels any upstream work those + // expressions perform. Handler-exit 500 covers early eval errors. + let ((status_code, content_type_str, custom_headers), response_snapshot) = self + .evaluate_response_precommit( + &request_id, + *line, + *column, + "Client disconnected before the streaming response started", + response_snapshot, + async { + let status_code = match status { + Some(expr) => { + let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; + match &v { + // Require a whole number in the HTTP status range + // rather than silently wrapping a fractional or + // out-of-range value through `as u16`. + Value::Number(n) if n.fract() == 0.0 && *n >= 100.0 && *n <= 599.0 => { + *n as u16 } - } - None => match &self.config.web_server_tls_cert_file { - Some(path) => path.clone(), - None => { + Value::Number(n) => { return Err(RuntimeError::new( - "This listen statement is marked 'secured' but no certificate is configured. Either write 'secured with certificate \"cert.pem\" and key \"key.pem\"' or set web_server_tls_cert_file and web_server_tls_key_file in .wflcfg".to_string(), + format!( + "Expected a whole HTTP status code between 100 and 599, got {n}" + ), *line, *column, )); } - }, - }; - let key_path = match &tls_config.key_path { - Some(expr) => { - let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; - match &v { - Value::Text(t) => t.to_string(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected text for TLS private key path, got {v:?}" - ), - *line, - *column, - )); - } + _ => { + return Err(RuntimeError::new( + format!("Expected number for status, got {}", v.type_name()), + *line, + *column, + )); } } - None => match &self.config.web_server_tls_key_file { - Some(path) => path.clone(), - None => { + } + None => 200, + }; + + let content_type_str = match content_type { + Some(expr) => { + let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; + match &v { + Value::Text(s) => s.to_string(), + _ => { return Err(RuntimeError::new( - "This listen statement is marked 'secured' but no private key is configured. Either write 'secured with certificate \"cert.pem\" and key \"key.pem\"' or set web_server_tls_cert_file and web_server_tls_key_file in .wflcfg".to_string(), + format!( + "Expected text for content type, got {}", + v.type_name() + ), *line, *column, )); } - }, - }; - - // Validate up front for actionable errors; warp would - // otherwise surface a bad certificate as a generic - // bind-time failure. - if let Err(msg) = validate_tls_pem_files(&cert_path, &key_path) { - return Err(RuntimeError::new(msg, *line, *column)); + } } + None => "application/octet-stream".to_string(), + }; - // try_bind_with_graceful_shutdown is the only TlsServer - // constructor that returns bind/TLS errors instead of - // panicking inside the spawned task; the never-completing - // signal keeps the server running until `close server` - // aborts its task. - match warp::serve(routes) - .tls() - .cert_path(&cert_path) - .key_path(&key_path) - .try_bind_with_graceful_shutdown( - (bind_addr, port_num), - std::future::pending::<()>(), - ) { - Ok((addr, server)) => { - let server_handle = tokio::spawn(server); - - let wfl_server = WflWebServer { - request_receiver: request_receiver.clone(), - request_sender: request_sender.clone(), - server_handle: Some(server_handle), - }; - self.web_servers - .borrow_mut() - .insert(server_name.clone(), wfl_server); - - let server_value = Value::Text(Arc::from(format!( - "WebServer::{}:{}", - addr.ip(), - addr.port() - ))); - - println!("Secure server is listening on port {}", addr.port()); - - match env.borrow_mut().define(server_name, server_value) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + let mut custom_headers: HashMap = HashMap::new(); + if let Some(expr) = headers { + let v = self.evaluate_expression(expr, Rc::clone(&env)).await?; + match &v { + Value::Object(obj) => { + for (name, value) in obj.borrow().iter() { + let value_str = match value { + Value::Text(s) => s.to_string(), + Value::Number(_) | Value::Bool(_) => value.to_string(), + _ => { + return Err(RuntimeError::new( + format!( + "Header '{name}' must be text, got {}", + value.type_name() + ), + *line, + *column, + )); + } + }; + // The pipeline owns framing headers. + if name.eq_ignore_ascii_case("content-type") + || name.eq_ignore_ascii_case("content-length") + || name.eq_ignore_ascii_case("transfer-encoding") + { + continue; + } + custom_headers.insert(name.clone(), value_str); } } - Err(e) => Err(RuntimeError::new( - format!( - "Failed to start secure web server on port {}: {}", - port_num, e - ), - *line, - *column, - )), + _ => { + return Err(RuntimeError::new( + format!( + "Expected a map for response headers, got {}", + v.type_name() + ), + *line, + *column, + )); + } } - } else { - // Plain HTTP server (unchanged behavior) - let server_task = warp::serve(routes).try_bind_ephemeral((bind_addr, port_num)); - - match server_task { - Ok((addr, server)) => { - // Spawn the server in the background - let server_handle = tokio::spawn(server); - - // Create WFL web server object - let wfl_server = WflWebServer { - request_receiver: request_receiver.clone(), - request_sender: request_sender.clone(), - server_handle: Some(server_handle), - }; - - // Store the server in the interpreter - self.web_servers - .borrow_mut() - .insert(server_name.clone(), wfl_server); - - // Create a server value with the actual address - let server_value = Value::Text(Arc::from(format!( - "WebServer::{}:{}", - addr.ip(), - addr.port() - ))); + } - println!("Server is listening on port {}", addr.port()); + Ok((status_code, content_type_str, custom_headers)) + }, + ) + .await?; - match env.borrow_mut().define(server_name, server_value) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } + // Commit: take the sender and hand the streaming head to the transport. + let (tx, rx) = mpsc::channel::>(RESPONSE_STREAM_BUFFER); + let mut completion = match self + .take_pending_response_completion(&request_id, *line, *column) + .await + { + Ok(completion) => completion, + Err(error) => { + if error.kind == ErrorKind::Cancelled { + self.cancel_response_precommit(&request_id, response_snapshot); } - Err(e) => Err(RuntimeError::new( - format!("Failed to start web server on port {}: {}", port_num, e), + return Err(error); + } + }; + match completion.take_sender() { + Some(sender) => { + if sender + .send(HandlerReply::Streaming { + status: status_code, + content_type: content_type_str, + headers: custom_headers, + body: rx, + }) + .is_err() + { + // Client hung up before the streaming head committed — + // cooperative cancellation, not a fault (see the buffered + // `respond` path). `Cancelled` keeps the concurrent + // breaker from counting a disconnect as a failure. + self.cancel_response_precommit(&request_id, response_snapshot); + return Err(RuntimeError::with_kind( + "Client disconnected before the streaming response started" + .to_string(), + *line, + *column, + ErrorKind::Cancelled, + )); + } + } + None => { + return Err(RuntimeError::new( + "Response already sent for this request".to_string(), *line, *column, - )), + )); } } + + let handle_id = { + let n = self.next_response_stream_id.get(); + self.next_response_stream_id.set(n + 1); + format!("respstream{n}") + }; + self.server_response_streams + .borrow_mut() + .insert(handle_id.clone(), (tx, 0)); + // Track it against the current handler so it is auto-closed if + // the handler ends without an explicit `close out`. + self.open_response_streams + .borrow_mut() + .push(handle_id.clone()); + + let mut stream_map = HashMap::new(); + stream_map.insert("_server_stream".to_string(), Value::Text(handle_id.into())); + stream_map.insert("status".to_string(), Value::Number(status_code as f64)); + let value = Value::Object(Rc::new(RefCell::new(stream_map))); + // define_or_replace so a `main loop` handler that rebinds `out` + // each request works — and so binding never fails after the + // stream head was already committed (which would otherwise leak + // the sender and hang the client). + env.borrow_mut().define_or_replace(variable_name, value); + Ok((Value::Null, ControlFlow::None)) } - Statement::WaitForRequestStatement { - server, - request_name, - timeout, + Statement::StreamWriteStatement { + value, + target, + is_line, + fallback_content, line, column, } => { - // Look up the server by name - let server_name = match self.evaluate_expression(server, Rc::clone(&env)).await? { - Value::Text(name) => { - // Extract server name from "WebServer::host:port" format - let name_str = name.as_ref(); - if name_str.starts_with("WebServer::") { - // Find the server by matching the exact server value - let web_servers = self.web_servers.borrow(); - - // Search through all servers to find which one has this exact value - let mut found_server = None; - for server_name in web_servers.keys() { - // Get the stored value for this server name - if let Some(Value::Text(stored_text)) = - env.borrow().get(server_name) - && stored_text.as_ref() == name_str - { - // Found the matching server - found_server = Some(server_name.clone()); - break; + // The target decides the interpretation. Evaluate it once; if it + // is a server response stream, this is a stream write. If it is + // not and this parsed from the ambiguous merged form + // (`write line to `), fall back to the classic + // file write `write to ` so a + // pre-existing file write is never reinterpreted. + let target_val = self.evaluate_expression(target, Rc::clone(&env)).await?; + let handle_id = match &target_val { + Value::Object(obj) => match obj.borrow().get("_server_stream") { + Some(Value::Text(id)) => Some(id.to_string()), + _ => None, + }, + _ => None, + }; + let handle_id = match handle_id { + Some(id) => id, + None => { + if let Some(fallback) = fallback_content { + // Classic file write: `write to `. + let content_value = + self.evaluate_expression(fallback, Rc::clone(&env)).await?; + let file_str = match &target_val { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected a server response stream or a file handle, got {}", + target_val.type_name() + ), + *line, + *column, + )); } - } - - // Return the found server or use first server as fallback - if let Some(server_name) = found_server { - server_name - } else if let Some((found_name, _)) = web_servers.iter().next() { - found_name.clone() - } else { - return Err(RuntimeError::new( - "No web servers found".to_string(), - *line, - *column, - )); - } - } else { - name_str.to_string() + }; + let content_str = format!("{content_value}"); + return match self.io_client.write_file(&file_str, &content_str).await { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + }; } - } - _ => { return Err(RuntimeError::new( - "Expected server name as text".to_string(), + format!( + "Expected a server response stream (from `start streaming response as ...`), got {}", + target_val.type_name() + ), *line, *column, )); } }; - - // Get the server's request receiver - let request_receiver = { - let web_servers = self.web_servers.borrow(); - if let Some(server) = web_servers.get(&server_name) { - server.request_receiver.clone() - } else { + let val = self.evaluate_expression(value, Rc::clone(&env)).await?; + let newline = usize::from(*is_line); + // Compute the outgoing byte length WITHOUT cloning a large + // text/binary value, so an over-budget write is rejected *before* + // any big allocation/copy (a huge write must not be materialized + // just to be refused). + let incoming_len = match &val { + Value::Text(s) => s.len() + newline, + Value::Binary(b) => b.len() + newline, + // Numbers/booleans render to a short string; the tiny + // allocation to measure them is not a DoS concern. + Value::Number(_) | Value::Bool(_) => val.to_string().len() + newline, + _ => { return Err(RuntimeError::new( - format!("Web server '{}' not found", server_name), + format!( + "Can only write text or binary to a response stream, got {}", + val.type_name() + ), *line, *column, )); } }; - // Wait for a request to come in (with optional timeout) - let request = { - let mut receiver = request_receiver.lock().await; - - // Evaluate timeout if provided - let timeout_duration = if let Some(timeout_expr) = timeout { - let timeout_val = self - .evaluate_expression(timeout_expr, Rc::clone(&env)) - .await?; - match timeout_val { - Value::Number(ms) if ms > 0.0 => { - Some(std::time::Duration::from_millis(ms as u64)) - } - _ => { - return Err(RuntimeError::new( - "Timeout must be a positive number (milliseconds)".to_string(), + // Reserve the response-byte budget on the running total FIRST (so a + // stream cannot bypass `web_server_max_response_size` via one huge + // chunk or many chunks), then clone the sender out so the map borrow + // is not held across the (possibly backpressured) send await. + let max_response_bytes = self.budget.limits().max_response_bytes; + let sender = { + let mut map = self.server_response_streams.borrow_mut(); + match map.get_mut(&handle_id) { + Some((tx, bytes_written)) => { + let new_total = bytes_written.saturating_add(incoming_len); + if new_total > max_response_bytes { + let actual = new_total; + // Drop the stream so the body ends rather than + // silently truncating, and untrack it so a handler + // that catches this error keeps no stale id. + map.remove(&handle_id); + self.open_response_streams + .borrow_mut() + .retain(|s| s != &handle_id); + return Err(self.budget_error( + BudgetExceeded::ResponseBytes { + limit: max_response_bytes, + actual, + }, *line, *column, )); } + *bytes_written = new_total; + Some(tx.clone()) } - } else { - None - }; - - // Wait for request with or without timeout. Loop so a request - // whose client already gave up (its oneshot receiver dropped - // on 408/504/disconnect, closing the sender) is skipped rather - // than handled — otherwise the interpreter would run a handler - // for a dead request and register a dead pending-response - // entry, letting repeated timeouts accumulate zombie work. - loop { - let req = if let Some(duration) = timeout_duration { - match tokio::time::timeout(duration, receiver.recv()).await { - Ok(Some(req)) => req, - Ok(None) => { - return Err(RuntimeError::new( - "Request channel closed".to_string(), - *line, - *column, - )); - } - Err(_) => { - return Err(RuntimeError::new( - format!( - "Timeout waiting for request ({} ms)", - duration.as_millis() - ), + None => None, + } + }; + match sender { + Some(tx) => { + // Within budget and the stream is open: materialize the + // bytes now (after the ceiling check) and send them. + let mut bytes = match &val { + Value::Text(s) => s.as_bytes().to_vec(), + Value::Binary(b) => b.to_vec(), + _ => val.to_string().into_bytes(), + }; + if *is_line { + bytes.push(b'\n'); + } + // Bound the (possibly backpressured) send. Once the 64-slot + // channel fills, `tx.send(..).await` parks until the client + // reads — a client that stays connected but stops reading + // would otherwise pin this handler forever (`main loop` is + // deadline-exempt). Cap the wait at + // `web_server_response_timeout_seconds` (0 = disabled, the + // documented sentinel — keep the unbounded behavior). A + // dropped receiver (disconnect) still returns immediately. + let write_timeout = self.config.web_server_response_timeout_seconds; + // `Ok(())` sent; `Err(false)` receiver dropped (disconnect); + // `Err(true)` timed out with the client still connected but + // not reading (a stall). + let outcome: Result<(), bool> = if write_timeout == 0 { + tx.send(bytes).await.map_err(|_| false) + } else { + match tokio::time::timeout( + std::time::Duration::from_secs(write_timeout), + tx.send(bytes), + ) + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err(false), + Err(_) => Err(true), + } + }; + match outcome { + Ok(()) => Ok((Value::Null, ControlFlow::None)), + Err(stalled) => { + // Disconnect → Cancelled (cooperative). Stall + // (client still connected, not reading) → Timeout + // (not Cancelled). Post-accept either way so the + // concurrent breaker does not tear the server down. + self.server_response_streams.borrow_mut().remove(&handle_id); + self.open_response_streams + .borrow_mut() + .retain(|s| s != &handle_id); + if stalled { + Err(RuntimeError::with_kind( + "Cannot write to response stream: the client stopped reading \ + (write timed out)" + .to_string(), *line, *column, - )); - } - } - } else { - // No timeout - wait indefinitely - match receiver.recv().await { - Some(req) => req, - None => { - return Err(RuntimeError::new( - "Request channel closed".to_string(), + ErrorKind::Timeout, + )) + } else { + Err(RuntimeError::with_kind( + "Cannot write to response stream: the client has disconnected" + .to_string(), *line, *column, - )); + ErrorKind::Cancelled, + )) } } - }; - - let abandoned = { - let sender_opt = req.response_sender.lock().await; - sender_opt.as_ref().is_none_or(|s| s.is_closed()) - }; - if abandoned { - log::debug!( - "skipping abandoned request {} ({} {}) from {}", - req.id, - req.method, - req.path, - req.client_ip - ); - continue; } - break req; } - }; - - // Define individual variables for request properties (more natural for WFL) - let mut env_mut = env.borrow_mut(); - - // Convert headers to a WFL object (shared by the request object and - // the standalone headers variable defined below) - let mut headers_map = HashMap::new(); - for (key, value) in request.headers.iter() { - headers_map.insert(key.clone(), Value::Text(Arc::from(value.clone()))); + None => Err(RuntimeError::new( + "Cannot write to a closed response stream".to_string(), + *line, + *column, + )), } - let headers_object = Value::Object(Rc::new(RefCell::new(headers_map))); - - // Define the main request variable (for use in respond statements and - // as request context for `execute file ... with `) - let mut request_properties = HashMap::new(); - request_properties.insert( - "_response_sender".to_string(), - Value::Text(Arc::from(request.id.clone())), - ); - request_properties.insert( - "method".to_string(), - Value::Text(Arc::from(request.method.clone())), - ); - request_properties.insert( - "path".to_string(), - Value::Text(Arc::from(request.path.clone())), - ); - request_properties.insert( - "query".to_string(), - Value::Text(Arc::from(request.query.clone())), - ); - request_properties.insert( - "client_ip".to_string(), - Value::Text(Arc::from(request.client_ip.clone())), - ); - // `body` is a lossy-UTF-8 text view (backward compatible); - // `body_bytes` is the lossless binary view for binary uploads. - let body_text = String::from_utf8_lossy(&request.body).into_owned(); - let body_binary = Value::Binary(Arc::from(request.body.as_slice())); - request_properties.insert( - "body".to_string(), - Value::Text(Arc::from(body_text.as_str())), - ); - request_properties.insert("body_bytes".to_string(), body_binary.clone()); - request_properties.insert("headers".to_string(), headers_object.clone()); - let request_object = Value::Object(Rc::new(RefCell::new(request_properties))); - - // These bindings are refreshed on every wait, so overwrite any - // previous request's values instead of failing on redefinition. - env_mut.define_or_replace(request_name, request_object); - - // Define individual request property variables - env_mut.define_or_replace("method", Value::Text(Arc::from(request.method.clone()))); - - env_mut.define_or_replace("path", Value::Text(Arc::from(request.path.clone()))); - - env_mut.define_or_replace("query", Value::Text(Arc::from(request.query.clone()))); - - env_mut.define_or_replace( - "client_ip", - Value::Text(Arc::from(request.client_ip.clone())), - ); - - env_mut.define_or_replace("body", Value::Text(Arc::from(body_text.as_str()))); - env_mut.define_or_replace("body_bytes", body_binary); - - env_mut.define_or_replace("headers", headers_object); + } + Statement::FlushStreamStatement { + target, + legacy_binding, + action_fallback, + line, + column, + } => { + // Backward compatibility: before `flush` was a streaming command, + // the full merged form (including postfix) was an expression + // statement. When the root binding of the legacy AST exists, + // evaluate it with the same ExpressionStatement semantics + // (zero-arg auto-call; parameterized bare call → arity error). + if let Some(fallback_expr) = action_fallback { + let root_bound = legacy_binding + .as_deref() + .is_some_and(|name| env.borrow().get(name).is_some()); + if root_bound { + // Reuse ExpressionStatement semantics by dispatching a + // synthetic statement. + return self + .execute_statement( + &Statement::ExpressionStatement { + expression: fallback_expr.clone(), + line: *line, + column: *column, + }, + Rc::clone(&env), + ) + .await; + } + } + let handle_id = self + .resolve_server_stream_handle(target, &env, *line, *column) + .await?; + let exists = self + .server_response_streams + .borrow() + .contains_key(&handle_id); + if !exists { + return Err(RuntimeError::new( + "Cannot flush a closed response stream".to_string(), + *line, + *column, + )); + } + // Advisory: chunks are already handed to the transport as they + // are written; yield so the transport task is scheduled to push + // them to the socket. + tokio::task::yield_now().await; + Ok((Value::Null, ControlFlow::None)) + } + // Graceful shutdown and signal handling statements + Statement::RegisterSignalHandlerStatement { + signal_type, + handler_name, + line, + column, + } => { + // For now, just store the signal handler registration + // In a full implementation, this would set up actual signal handlers + let signal_handler_key = format!("signal_handler_{}", signal_type); - drop(env_mut); // Release the borrow + env.borrow_mut() + .define( + &signal_handler_key, + Value::Text(Arc::from(handler_name.clone())), + ) + .map_err(|e| RuntimeError::new(e, *line, *column))?; - // Store the request in a global map for RespondStatement to access. - // Done only after every define above succeeded: registering earlier - // would park the oneshot sender on an error path and leave the HTTP - // client hanging instead of failing fast. - { - let mut pending_responses = self.pending_responses.borrow_mut(); - // Prune entries whose client already disconnected/timed out - // (oneshot sender closed) before inserting the new one, so a - // handler that never `respond`s to a since-abandoned request - // cannot let the map grow without bound across many timeouts. - // (The admission slot itself is released by the transport task, - // not this prune — see `PendingResponse`.) - pending_responses.retain(|_, pending| match pending.sender.try_lock() { - Ok(guard) => guard.as_ref().is_some_and(|s| !s.is_closed()), - // Locked right now (being responded to) — keep it. - Err(_) => true, - }); - pending_responses.insert( - request.id.clone(), - PendingResponse { - sender: request.response_sender, - }, - ); - } + // TODO: Implement actual signal handling with tokio::signal + // For now, we'll simulate this in the graceful shutdown test Ok((Value::Null, ControlFlow::None)) } - Statement::RespondStatement { - request, - content, - status, - content_type, - headers, + Statement::StopAcceptingConnectionsStatement { + server, line, column, } => { - // Get the request object - let request_val = self.evaluate_expression(request, Rc::clone(&env)).await?; - let request_id = match &request_val { - Value::Object(obj) => { - let obj_ref = obj.borrow(); - match obj_ref.get("_response_sender") { - Some(Value::Text(id)) => id.as_ref().to_string(), - _ => { + let server_val = self.evaluate_expression(server, Rc::clone(&env)).await?; + let server_name = match &server_val { + Value::Text(name) => { + let name_str = name.as_ref(); + if name_str.starts_with("WebServer::") { + // Find the original server name in our web_servers map + let web_servers = self.web_servers.borrow(); + if let Some((found_name, _)) = web_servers.iter().next() { + found_name.clone() + } else { return Err(RuntimeError::new( - "Request object missing response sender ID".to_string(), + "No web servers found".to_string(), *line, *column, )); } + } else { + name_str.to_string() } } _ => { return Err(RuntimeError::new( - "Expected request object".to_string(), + "Expected server name as text".to_string(), *line, *column, )); } }; - // Take the response sender out of the pending map (and out of its - // mutex) up front, into an RAII completion guard, *before* any - // fallible response construction below (content/status/type/header - // evaluation, byte-cap checks). On an early error the guard's Drop - // answers 500, so the request is always resolved instead of - // hanging until its timeout; a successful respond disarms it via - // `take_sender`. - let pending_entry = { - let mut pending = self.pending_responses.borrow_mut(); - pending.remove(&request_id) - }; - let mut completion = match pending_entry { - // The admission slot is released by the transport task when it - // finishes delivering this response (or on its timeout), so the - // completion guard carries only the response channel. - Some(entry) => match entry.sender.lock().await.take() { - Some(sender) => ResponseCompletion { - sender: Some(sender), - }, - None => { - return Err(RuntimeError::new( - "Response already sent for this request".to_string(), - *line, - *column, - )); - } - }, - None => { - return Err(RuntimeError::new( - "Request ID not found - response may have already been sent" - .to_string(), - *line, - *column, - )); - } - }; + // Mark server as no longer accepting connections + // In a full implementation, this would stop the warp server from accepting new connections + // For now, we'll just set a flag + env.borrow_mut() + .define( + &format!("{}_accepting_connections", server_name), + Value::Bool(false), + ) + .map_err(|e| RuntimeError::new(e, *line, *column))?; - // Evaluate response content. Binary values are carried through - // as raw bytes so fonts/images/etc. serve losslessly; text and - // scalar values keep their existing UTF-8 rendering. - let content_val = self.evaluate_expression(content, Rc::clone(&env)).await?; - let is_binary = matches!(content_val, Value::Binary(_)); + Ok((Value::Null, ControlFlow::None)) + } + Statement::CloseServerStatement { + server, + line, + column, + } => { + let server_val = self.evaluate_expression(server, Rc::clone(&env)).await?; - // Enforce the response-body ceiling on the *borrowed* length - // first, so an oversized Text/Binary body is refused before it is - // duplicated into `content_bytes` (bounding peak allocation). - if let Value::Text(text) = &content_val - && let Err(exceeded) = self.budget.check_response_bytes(text.len()) - { - return Err(self.budget_error(exceeded, *line, *column)); - } - if let Value::Binary(bytes) = &content_val - && let Err(exceeded) = self.budget.check_response_bytes(bytes.len()) + // A WebSocket server closes by asking each connection's writer to + // send a close frame, then aborting the accept task. + if let Value::Text(name) = &server_val + && name.starts_with("WebSocketServer::") { - return Err(self.budget_error(exceeded, *line, *column)); - } - - let content_bytes: Vec = match &content_val { - Value::Text(text) => text.as_bytes().to_vec(), - Value::Number(n) => n.to_string().into_bytes(), - Value::Bool(b) => b.to_string().into_bytes(), - Value::Binary(bytes) => bytes.to_vec(), - Value::Null => Vec::new(), - // Composite/opaque values (lists, objects, functions, …) have - // no meaningful HTTP body rendering, and their `{:?}` form is - // unbounded — materializing it would allocate past the - // response cap before it could be checked. Reject them with a - // clear error instead. - other => { - return Err(RuntimeError::new( - format!( - "Cannot use {} as a response body; respond with text, a number, a boolean, binary data, or nothing", - other.type_name() - ), - *line, - *column, - )); - } - }; - - // Re-check the materialized length to cover the small formatted - // variants (Number/Bool), which have no cheap borrowed length. - if let Err(exceeded) = self.budget.check_response_bytes(content_bytes.len()) { - return Err(self.budget_error(exceeded, *line, *column)); - } - - // Evaluate status code (optional) - let status_code = if let Some(status_expr) = status { - let status_val = self - .evaluate_expression(status_expr, Rc::clone(&env)) - .await?; - match &status_val { - Value::Number(n) => *n as u16, - _ => { - return Err(RuntimeError::new( - "Status code must be a number".to_string(), - *line, - *column, - )); + let key = name.to_string(); + // Remove and drop the borrow before any `.await` below. + let removed_ws = self.web_socket_servers.borrow_mut().remove(&key); + if let Some(mut ws_server) = removed_ws { + // Wake every live connection's reader so it stops waiting + // on the peer and tears down (releasing its slot), even if + // the peer never answers the close handshake. + let _ = ws_server.close_tx.send(true); + let ids = ws_server + .connection_ids + .lock() + .map(|g| g.clone()) + .unwrap_or_default(); + if let Ok(mut map) = self.ws_connections.lock() { + for id in ids { + if let Some(tx) = map.remove(&id) { + let _ = tx.try_send(WsOutbound::Close); + } + } } - } - } else { - 200 // Default to 200 OK - }; - - // Evaluate content type (optional) - let content_type_str = if let Some(ct_expr) = content_type { - let ct_val = self.evaluate_expression(ct_expr, Rc::clone(&env)).await?; - match &ct_val { - Value::Text(text) => text.as_ref().to_string(), - _ => { - return Err(RuntimeError::new( - "Content type must be text".to_string(), - *line, - *column, - )); + if let Some(handle) = ws_server.server_handle.take() { + // Give queued close frames a moment to flush before + // the accept task is torn down. + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + handle.abort(); } + return Ok((Value::Null, ControlFlow::None)); } - } else if is_binary { - // Binary responses default to a generic binary media type - // rather than text/plain so browsers don't misinterpret them. - "application/octet-stream".to_string() - } else { - "text/plain".to_string() // Default content type - }; + return Err(RuntimeError::new( + format!("WebSocket server '{key}' not found"), + *line, + *column, + )); + } - // Evaluate custom response headers (optional). Mirrors the - // outbound client's headers map: a WFL Object of name -> value. - // Enables RFC 10008 (HTTP QUERY) servers to advertise - // `Accept-Query` and point at results with `Content-Location` - // or `Location`. - let mut custom_headers: HashMap = HashMap::new(); - if let Some(headers_expr) = headers { - let headers_val = self - .evaluate_expression(headers_expr, Rc::clone(&env)) - .await?; - match &headers_val { - Value::Object(obj) => { - for (name, value) in obj.borrow().iter() { - let value_str = match value { - Value::Text(s) => s.to_string(), - Value::Number(_) | Value::Bool(_) => value.to_string(), - _ => { - return Err(RuntimeError::new( - format!( - "Response header '{name}' must be text, a number, or a boolean, got {}", - value.type_name() - ), - *line, - *column, - )); - } - }; - // Content-Type, Content-Length, and - // Transfer-Encoding are computed by the response - // pipeline (the `content_type` clause and warp's - // builder set them explicitly). Warp *appends* - // custom headers, so letting the map override - // these would emit duplicate/conflicting headers - // (RFC 7230 §3.3.2) and risk response splitting. - // Drop them so the pipeline stays authoritative. - if name.eq_ignore_ascii_case("content-type") - || name.eq_ignore_ascii_case("content-length") - || name.eq_ignore_ascii_case("transfer-encoding") + let server_name = match &server_val { + Value::Text(name) => { + let name_str = name.as_ref(); + if name_str.starts_with("WebServer::") { + // Find the server name that corresponds to this WebServer value + let web_servers = self.web_servers.borrow(); + + // Search through all servers to find which one has this exact value + let mut found_server = None; + for server_name in web_servers.keys() { + // Check if this server name's variable has the matching value + if let Some(Value::Text(stored_text)) = + env.borrow().get(server_name) + && stored_text.as_ref() == name_str { - continue; + found_server = Some(server_name.clone()); + break; } - custom_headers.insert(name.clone(), value_str); } + + // Return the found server or use first server as fallback + if let Some(server_name) = found_server { + server_name + } else if let Some((found_name, _)) = web_servers.iter().next() { + found_name.clone() + } else { + return Err(RuntimeError::new( + "No web servers found".to_string(), + *line, + *column, + )); + } + } else { + name_str.to_string() } - _ => { - return Err(RuntimeError::new( - format!( - "Expected a map for response headers, got {}", - headers_val.type_name() - ), - *line, - *column, - )); - } } - } - - // Create response - let response = WflHttpResponse { - content: content_bytes, - status: status_code, - content_type: content_type_str, - headers: custom_headers, + _ => { + return Err(RuntimeError::new( + "Expected server name as text".to_string(), + *line, + *column, + )); + } }; - // Deliver the response and disarm the guard's 500 fallback. The - // sender was taken up front, so this is the sole delivery path. - match completion.take_sender() { - Some(sender) => { - if sender.send(response).is_err() { - return Err(RuntimeError::new( - "Failed to send response - client may have disconnected" - .to_string(), - *line, - *column, - )); + // Close the server. Remove it from the map and DROP the borrow + // before the graceful-shutdown await — holding `web_servers` + // borrowed across `.await` would panic a concurrent sibling that + // touches the map during the yield (the reason this module can + // drop its `await_holding_refcell_ref` allow — see lib.rs). + let removed = self.web_servers.borrow_mut().remove(&server_name); + match removed { + Some(mut wfl_server) => { + // Graceful shutdown: give in-flight responses time to + // complete transmission before forcefully aborting the + // server task. The map borrow is already released, so + // this await cannot conflict with a sibling handler. + if let Some(handle) = wfl_server.server_handle.take() { + // Allow 50ms for pending HTTP responses to reach the + // client before abort() closes the TCP connection + // (otherwise IncompleteMessage on the client). + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + handle.abort(); } } None => { return Err(RuntimeError::new( - "Response already sent for this request".to_string(), + format!("Server '{}' not found", server_name), *line, *column, )); @@ -7908,261 +11066,401 @@ impl Interpreter { Ok((Value::Null, ControlFlow::None)) } - // Graceful shutdown and signal handling statements - Statement::RegisterSignalHandlerStatement { - signal_type, - handler_name, + Statement::ListenWebSocketStatement { + port, + server_name, line, column, } => { - // For now, just store the signal handler registration - // In a full implementation, this would set up actual signal handlers - let signal_handler_key = format!("signal_handler_{}", signal_type); + let port_val = self.evaluate_expression(port, Rc::clone(&env)).await?; + let port_num = match &port_val { + Value::Number(n) if n.fract() == 0.0 && *n >= 0.0 && *n <= 65535.0 => *n as u16, + _ => { + return Err(RuntimeError::new( + format!( + "Expected a whole number between 0 and 65535 for the websocket port, got {port_val:?}" + ), + *line, + *column, + )); + } + }; - env.borrow_mut() - .define( - &signal_handler_key, - Value::Text(Arc::from(handler_name.clone())), - ) - .map_err(|e| RuntimeError::new(e, *line, *column))?; + // Bounded lifecycle-event channel (sized from the shared budget): + // a flood of connect/message/disconnect events sheds on `Full` + // rather than growing memory without bound. + let (event_sender, event_receiver) = + mpsc::channel::(self.budget.ws_queue_bound()); + let event_receiver = Arc::new(tokio::sync::Mutex::new(event_receiver)); + let connection_ids = Arc::new(std::sync::Mutex::new(Vec::new())); + // Per-server cancellation channel. Each connection clones the + // receiver; `close server` flips/drops the sender to wake them. + let (close_tx, close_rx) = tokio::sync::watch::channel(false); - // TODO: Implement actual signal handling with tokio::signal - // For now, we'll simulate this in the graceful shutdown test + // Clones handed to warp's per-connection tasks. + let ws_connections = Arc::clone(&self.ws_connections); + let connection_ids_task = Arc::clone(&connection_ids); + let event_sender_task = event_sender.clone(); + let budget_task = Arc::clone(&self.budget); + let close_rx_task = close_rx.clone(); - Ok((Value::Null, ControlFlow::None)) - } - Statement::StopAcceptingConnectionsStatement { - server, - line, - column, - } => { - let server_val = self.evaluate_expression(server, Rc::clone(&env)).await?; - let server_name = match &server_val { - Value::Text(name) => { - let name_str = name.as_ref(); - if name_str.starts_with("WebServer::") { - // Find the original server name in our web_servers map - let web_servers = self.web_servers.borrow(); - if let Some((found_name, _)) = web_servers.iter().next() { - found_name.clone() - } else { - return Err(RuntimeError::new( - "No web servers found".to_string(), - *line, - *column, - )); - } - } else { - name_str.to_string() - } - } - _ => { + // Cap the transport's own message/frame assembly at the budget's + // per-message limit *before* upgrade, so a fragmented text frame + // or an ignored binary frame cannot allocate up to Tungstenite's + // independent defaults on the receive side. The queued-byte permit + // in the reader loop is then the second (global) layer. + let max_ws_message = self.budget.max_ws_message_bytes(); + let route = warp::ws().and(warp::addr::remote()).map( + move |ws: warp::ws::Ws, remote: Option| { + let events = event_sender_task.clone(); + let connections = Arc::clone(&ws_connections); + let ids = Arc::clone(&connection_ids_task); + let budget = Arc::clone(&budget_task); + let cancel = close_rx_task.clone(); + ws.max_message_size(max_ws_message) + .max_frame_size(max_ws_message) + .on_upgrade(move |socket| { + handle_ws_connection( + socket, + remote, + events, + connections, + ids, + budget, + cancel, + ) + }) + }, + ); + + let bind_addr: IpAddr = match self.config.web_server_bind_address.parse() { + Ok(addr) => addr, + Err(_) => { return Err(RuntimeError::new( - "Expected server name as text".to_string(), + format!( + "Invalid web_server_bind_address in config: '{}'. Expected a valid IP address (e.g., '127.0.0.1' or '0.0.0.0')", + self.config.web_server_bind_address + ), *line, *column, )); } }; - // Mark server as no longer accepting connections - // In a full implementation, this would stop the warp server from accepting new connections - // For now, we'll just set a flag - env.borrow_mut() - .define( - &format!("{}_accepting_connections", server_name), - Value::Bool(false), + match warp::serve(route).try_bind_ephemeral((bind_addr, port_num)) { + Ok((addr, server)) => { + let server_handle = tokio::spawn(server); + let key = format!("WebSocketServer::{}:{}", addr.ip(), addr.port()); + + let wfl_ws = WflWebSocketServer { + event_receiver, + connection_ids, + handlers: RefCell::new(WsHandlerSet::default()), + server_handle: Some(server_handle), + close_tx, + }; + self.web_socket_servers + .borrow_mut() + .insert(key.clone(), wfl_ws); + + let server_value = Value::Text(Arc::from(key)); + println!("WebSocket server is listening on port {}", addr.port()); + + match env.borrow_mut().define(server_name, server_value) { + Ok(_) => Ok((Value::Null, ControlFlow::None)), + Err(msg) => Err(RuntimeError::new(msg, *line, *column)), + } + } + Err(e) => Err(RuntimeError::new( + format!("Failed to start websocket server on port {port_num}: {e}"), + *line, + *column, + )), + } + } + Statement::WebSocketHandlerStatement { + event, + server, + binding, + body, + line, + column, + } => { + let server_key = self + .resolve_ws_server_key(server, Rc::clone(&env), *line, *column) + .await?; + + let handler = WsRegisteredHandler { + binding: binding.clone(), + body: body.clone(), + env: Rc::clone(&env), + }; + + let servers = self.web_socket_servers.borrow(); + let ws_server = servers.get(&server_key).ok_or_else(|| { + RuntimeError::new( + format!( + "WebSocket server '{server_key}' is not running. Start it with 'listen for websockets ...' first." + ), + *line, + *column, ) - .map_err(|e| RuntimeError::new(e, *line, *column))?; + })?; + + let mut handlers = ws_server.handlers.borrow_mut(); + match event { + WsHandlerEvent::Connect => handlers.connect = Some(handler), + WsHandlerEvent::Message => handlers.message = Some(handler), + WsHandlerEvent::Disconnect => handlers.disconnect = Some(handler), + } Ok((Value::Null, ControlFlow::None)) } - Statement::CloseServerStatement { + Statement::SendWebSocketMessageStatement { + message, + target, + line, + column, + } => { + let message_val = self.evaluate_expression(message, Rc::clone(&env)).await?; + // Measure the payload from the borrowed value first (no clone). + let msg_len = Self::ws_message_byte_len(&message_val, *line, *column)?; + + let target_val = self.evaluate_expression(target, Rc::clone(&env)).await?; + let conn_id = Self::ws_connection_id(&target_val, *line, *column)?; + + let sender = self + .ws_connections + .lock() + .ok() + .and_then(|map| map.get(&conn_id).cloned()); + + match sender { + Some(tx) => { + // Reserve the frame's bytes against the per-message and + // global queued-byte budget *before* materializing the + // payload, so an oversized value is never cloned first. + match self.budget.try_reserve_ws_bytes(msg_len) { + Some(permit) => { + let text = Self::ws_message_text(&message_val, *line, *column)?; + // A closed writer task is indistinguishable from a + // live one here; a dropped frame simply means the + // peer left (or the bounded queue is saturated). + if let Err(err) = tx.try_send(WsOutbound::Text { + text, + _permit: permit, + }) { + log::warn!( + "WebSocket outbound queue full/closed for {conn_id}; dropping frame: {err}" + ); + } + } + None => { + log::warn!( + "WebSocket outbound frame for {conn_id} ({msg_len} bytes) exceeds the per-message or global queued-byte limit; dropping frame" + ); + } + } + Ok((Value::Null, ControlFlow::None)) + } + None => Err(RuntimeError::new( + "That websocket connection is closed or unknown".to_string(), + *line, + *column, + )), + } + } + Statement::BroadcastWebSocketMessageStatement { + message, server, line, column, } => { - let server_val = self.evaluate_expression(server, Rc::clone(&env)).await?; + let message_val = self.evaluate_expression(message, Rc::clone(&env)).await?; + // Measure first; reject an oversized broadcast payload before it + // is materialized (and then cloned per recipient). + let msg_len = Self::ws_message_byte_len(&message_val, *line, *column)?; + if msg_len > self.budget.max_ws_message_bytes() { + log::warn!( + "WebSocket broadcast payload ({msg_len} bytes) exceeds the per-message limit; dropping broadcast" + ); + return Ok((Value::Null, ControlFlow::None)); + } + let text = Self::ws_message_text(&message_val, *line, *column)?; - // A WebSocket server closes by asking each connection's writer to - // send a close frame, then aborting the accept task. - if let Value::Text(name) = &server_val - && name.starts_with("WebSocketServer::") - { - let key = name.to_string(); - if let Some(mut ws_server) = self.web_socket_servers.borrow_mut().remove(&key) { - // Wake every live connection's reader so it stops waiting - // on the peer and tears down (releasing its slot), even if - // the peer never answers the close handshake. - let _ = ws_server.close_tx.send(true); - let ids = ws_server + let server_key = self + .resolve_ws_server_key(server, Rc::clone(&env), *line, *column) + .await?; + + let ids: Vec = { + let servers = self.web_socket_servers.borrow(); + match servers.get(&server_key) { + Some(ws_server) => ws_server .connection_ids .lock() .map(|g| g.clone()) - .unwrap_or_default(); - if let Ok(mut map) = self.ws_connections.lock() { - for id in ids { - if let Some(tx) = map.remove(&id) { - let _ = tx.try_send(WsOutbound::Close); - } - } - } - if let Some(handle) = ws_server.server_handle.take() { - // Give queued close frames a moment to flush before - // the accept task is torn down. - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - handle.abort(); - } - return Ok((Value::Null, ControlFlow::None)); + .unwrap_or_default(), + None => Vec::new(), } - return Err(RuntimeError::new( - format!("WebSocket server '{key}' not found"), - *line, - *column, - )); - } - - let server_name = match &server_val { - Value::Text(name) => { - let name_str = name.as_ref(); - if name_str.starts_with("WebServer::") { - // Find the server name that corresponds to this WebServer value - let web_servers = self.web_servers.borrow(); + }; - // Search through all servers to find which one has this exact value - let mut found_server = None; - for server_name in web_servers.keys() { - // Check if this server name's variable has the matching value - if let Some(Value::Text(stored_text)) = - env.borrow().get(server_name) - && stored_text.as_ref() == name_str - { - found_server = Some(server_name.clone()); - break; + if let Ok(map) = self.ws_connections.lock() { + for id in ids { + if let Some(tx) = map.get(&id) { + // Reserve each recipient's copy against the global + // queued-byte budget; shed over-budget frames rather + // than buffering them without bound. + match self.budget.try_reserve_ws_bytes(msg_len) { + Some(permit) => { + if let Err(err) = tx.try_send(WsOutbound::Text { + text: text.clone(), + _permit: permit, + }) { + log::warn!( + "WebSocket broadcast: outbound queue full/closed for {id}; dropping frame: {err}" + ); + } + } + None => { + log::warn!( + "WebSocket broadcast frame for {id} ({msg_len} bytes) exceeds the global queued-byte limit; dropping frame" + ); } } - - // Return the found server or use first server as fallback - if let Some(server_name) = found_server { - server_name - } else if let Some((found_name, _)) = web_servers.iter().next() { - found_name.clone() - } else { - return Err(RuntimeError::new( - "No web servers found".to_string(), - *line, - *column, - )); - } - } else { - name_str.to_string() } } - _ => { - return Err(RuntimeError::new( - "Expected server name as text".to_string(), - *line, - *column, - )); - } - }; - - // Close the server - let mut web_servers = self.web_servers.borrow_mut(); - if let Some(mut wfl_server) = web_servers.remove(&server_name) { - // Graceful shutdown: Give in-flight responses time to complete transmission - // before forcefully aborting the server task - if let Some(handle) = wfl_server.server_handle.take() { - // Allow 50ms for pending HTTP responses to be transmitted - // This prevents race condition where abort() closes the TCP connection - // before response bytes reach the client, causing IncompleteMessage errors - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - handle.abort(); - } - } else { - return Err(RuntimeError::new( - format!("Server '{}' not found", server_name), - *line, - *column, - )); } Ok((Value::Null, ControlFlow::None)) } - Statement::ListenWebSocketStatement { - port, - server_name, + // Subprocess statements + Statement::ExecuteCommandStatement { + command, + arguments, + variable_name, + use_shell, line, column, } => { - let port_val = self.evaluate_expression(port, Rc::clone(&env)).await?; - let port_num = match &port_val { - Value::Number(n) if n.fract() == 0.0 && *n >= 0.0 && *n <= 65535.0 => *n as u16, + // Evaluate command expression + let cmd_val = self.evaluate_expression(command, Rc::clone(&env)).await?; + let cmd_str = match &cmd_val { + Value::Text(text) => text.as_ref(), _ => { return Err(RuntimeError::new( - format!( - "Expected a whole number between 0 and 65535 for the websocket port, got {port_val:?}" - ), + format!("Command must be text, got {}", cmd_val.type_name()), *line, *column, )); } }; - // Bounded lifecycle-event channel (sized from the shared budget): - // a flood of connect/message/disconnect events sheds on `Full` - // rather than growing memory without bound. - let (event_sender, event_receiver) = - mpsc::channel::(self.budget.ws_queue_bound()); - let event_receiver = Arc::new(tokio::sync::Mutex::new(event_receiver)); - let connection_ids = Arc::new(std::sync::Mutex::new(Vec::new())); - // Per-server cancellation channel. Each connection clones the - // receiver; `close server` flips/drops the sender to wake them. - let (close_tx, close_rx) = tokio::sync::watch::channel(false); + // Evaluate arguments if provided + let args_vec: Vec = if let Some(args_expr) = arguments { + let args_val = self.evaluate_expression(args_expr, Rc::clone(&env)).await?; + match &args_val { + Value::List(list) => { + let list_ref = list.borrow(); + list_ref + .iter() + .map(|v| match v { + Value::Text(t) => Ok(t.as_ref().to_string()), + _ => Ok(v.to_string()), + }) + .collect::, RuntimeError>>()? + } + Value::Text(text) => vec![text.as_ref().to_string()], + _ => { + return Err(RuntimeError::new( + format!( + "Arguments must be a list or text, got {}", + args_val.type_name() + ), + *line, + *column, + )); + } + } + } else { + Vec::new() + }; - // Clones handed to warp's per-connection tasks. - let ws_connections = Arc::clone(&self.ws_connections); - let connection_ids_task = Arc::clone(&connection_ids); - let event_sender_task = event_sender.clone(); - let budget_task = Arc::clone(&self.budget); - let close_rx_task = close_rx.clone(); + // Execute command + let args_refs: Vec<&str> = args_vec.iter().map(|s| s.as_str()).collect(); + let (stdout, stderr, exit_code) = self + .io_client + .execute_command(cmd_str, &args_refs, *use_shell, *line, *column) + .await + .map_err(|e| match e { + ExecuteCommandError::Budget(exceeded) => { + self.budget_error(exceeded, *line, *column) + } + ExecuteCommandError::Timeout { seconds } => RuntimeError::with_kind( + format!("Subprocess execution exceeded timeout ({seconds}s)"), + *line, + *column, + ErrorKind::Timeout, + ), + ExecuteCommandError::Other(message) => { + // Preserve the existing subprocess error + // classification for non-budget failures. + let kind = if message.contains("program not found") + || message.contains("cannot find") + || message.contains("not recognized") + { + ErrorKind::CommandNotFound + } else if message.contains("spawn") { + ErrorKind::ProcessSpawnFailed + } else { + ErrorKind::General + }; + RuntimeError::with_kind(message, *line, *column, kind) + } + })?; - // Cap the transport's own message/frame assembly at the budget's - // per-message limit *before* upgrade, so a fragmented text frame - // or an ignored binary frame cannot allocate up to Tungstenite's - // independent defaults on the receive side. The queued-byte permit - // in the reader loop is then the second (global) layer. - let max_ws_message = self.budget.max_ws_message_bytes(); - let route = warp::ws().and(warp::addr::remote()).map( - move |ws: warp::ws::Ws, remote: Option| { - let events = event_sender_task.clone(); - let connections = Arc::clone(&ws_connections); - let ids = Arc::clone(&connection_ids_task); - let budget = Arc::clone(&budget_task); - let cancel = close_rx_task.clone(); - ws.max_message_size(max_ws_message) - .max_frame_size(max_ws_message) - .on_upgrade(move |socket| { - handle_ws_connection( - socket, - remote, - events, - connections, - ids, - budget, - cancel, - ) - }) - }, + // Build result object + let mut result_map = HashMap::new(); + result_map.insert( + "output".to_string(), + Value::Text(Arc::from(stdout.as_str())), ); + result_map.insert("error".to_string(), Value::Text(Arc::from(stderr.as_str()))); + result_map.insert("exit_code".to_string(), Value::Number(exit_code as f64)); + result_map.insert("success".to_string(), Value::Bool(exit_code == 0)); - let bind_addr: IpAddr = match self.config.web_server_bind_address.parse() { - Ok(addr) => addr, - Err(_) => { + let result_obj = Value::Object(Rc::new(RefCell::new(result_map))); + + // Store result if variable name provided + if let Some(var_name) = variable_name { + env.borrow_mut() + .define(var_name, result_obj) + .map_err(|e| RuntimeError::new(e, *line, *column))?; + } + + Ok((Value::Null, ControlFlow::None)) + } + Statement::ExecuteFileStatement { + path, + request, + variable_name, + line, + column, + } => { + // Guard against a file that (directly or indirectly) executes + // itself, using the shared budget's execute-file depth ceiling. + if let Err(exceeded) = self.budget.check_execute_file_depth(self.execute_depth) { + return Err(self.budget_error(exceeded, *line, *column)); + } + + // Evaluate path expression to string + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str: String = match &path_value { + Value::Text(s) => s.to_string(), + _ => { return Err(RuntimeError::new( format!( - "Invalid web_server_bind_address in config: '{}'. Expected a valid IP address (e.g., '127.0.0.1' or '0.0.0.0')", - self.config.web_server_bind_address + "Execute file path must be text, got {}", + path_value.type_name() ), *line, *column, @@ -8170,194 +11468,232 @@ impl Interpreter { } }; - match warp::serve(route).try_bind_ephemeral((bind_addr, port_num)) { - Ok((addr, server)) => { - let server_handle = tokio::spawn(server); - let key = format!("WebSocketServer::{}:{}", addr.ip(), addr.port()); - - let wfl_ws = WflWebSocketServer { - event_receiver, - connection_ids, - handlers: RefCell::new(WsHandlerSet::default()), - server_handle: Some(server_handle), - close_tx, - }; - self.web_socket_servers - .borrow_mut() - .insert(key.clone(), wfl_ws); - - let server_value = Value::Text(Arc::from(key)); - println!("WebSocket server is listening on port {}", addr.port()); - - match env.borrow_mut().define(server_name, server_value) { - Ok(_) => Ok((Value::Null, ControlFlow::None)), - Err(msg) => Err(RuntimeError::new(msg, *line, *column)), - } - } - Err(e) => Err(RuntimeError::new( - format!("Failed to start websocket server on port {port_num}: {e}"), - *line, - *column, - )), - } - } - Statement::WebSocketHandlerStatement { - event, - server, - binding, - body, - line, - column, - } => { - let server_key = self - .resolve_ws_server_key(server, Rc::clone(&env), *line, *column) - .await?; - - let handler = WsRegisteredHandler { - binding: binding.clone(), - body: body.clone(), - env: Rc::clone(&env), + // Resolve relative to the current script's directory (like load module), + // mapping a missing file to FileNotFound so `when file not found` works + let opt_source = self.current_source_file.borrow().as_ref().cloned(); + let joined = if let Some(source_path) = opt_source { + source_path + .parent() + .map(|dir| dir.join(&path_str)) + .unwrap_or_else(|| PathBuf::from(&path_str)) + } else { + let cwd = std::env::current_dir().map_err(|e| { + RuntimeError::new( + format!("Cannot determine current directory: {e}"), + *line, + *column, + ) + })?; + cwd.join(&path_str) }; - - let servers = self.web_socket_servers.borrow(); - let ws_server = servers.get(&server_key).ok_or_else(|| { - RuntimeError::new( - format!( - "WebSocket server '{server_key}' is not running. Start it with 'listen for websockets ...' first." - ), + let map_io_error = |e: std::io::Error| { + let kind = match e.kind() { + std::io::ErrorKind::NotFound => ErrorKind::FileNotFound, + std::io::ErrorKind::PermissionDenied => ErrorKind::PermissionDenied, + _ => ErrorKind::General, + }; + RuntimeError::with_kind( + format!("Cannot execute wfl file '{path_str}': {e}"), *line, *column, + kind, ) - })?; - - let mut handlers = ws_server.handlers.borrow_mut(); - match event { - WsHandlerEvent::Connect => handlers.connect = Some(handler), - WsHandlerEvent::Message => handlers.message = Some(handler), - WsHandlerEvent::Disconnect => handlers.disconnect = Some(handler), - } - - Ok((Value::Null, ControlFlow::None)) - } - Statement::SendWebSocketMessageStatement { - message, - target, - line, - column, - } => { - let message_val = self.evaluate_expression(message, Rc::clone(&env)).await?; - // Measure the payload from the borrowed value first (no clone). - let msg_len = Self::ws_message_byte_len(&message_val, *line, *column)?; - - let target_val = self.evaluate_expression(target, Rc::clone(&env)).await?; - let conn_id = Self::ws_connection_id(&target_val, *line, *column)?; - - let sender = self - .ws_connections - .lock() - .ok() - .and_then(|map| map.get(&conn_id).cloned()); + }; + let resolved_path = tokio::fs::canonicalize(&joined) + .await + .map_err(map_io_error)?; + // Read under the shared source-size ceiling (bounded read). + let content = self + .read_source_bounded(&resolved_path, *line, *column) + .await?; - match sender { - Some(tx) => { - // Reserve the frame's bytes against the per-message and - // global queued-byte budget *before* materializing the - // payload, so an oversized value is never cloned first. - match self.budget.try_reserve_ws_bytes(msg_len) { - Some(permit) => { - let text = Self::ws_message_text(&message_val, *line, *column)?; - // A closed writer task is indistinguishable from a - // live one here; a dropped frame simply means the - // peer left (or the bounded queue is saturated). - if let Err(err) = tx.try_send(WsOutbound::Text { - text, - _permit: permit, - }) { - log::warn!( - "WebSocket outbound queue full/closed for {conn_id}; dropping frame: {err}" - ); + // Evaluate the optional request context and extract the variables + // that `wait for request` defines, so the executed file sees the + // same names. Validate the shape upfront so a wrong object fails + // here with a clear message instead of as confusing undefined + // variable errors inside the executed file. + let request_vars: Vec<(String, Value)> = if let Some(request_expr) = request { + let request_value = self + .evaluate_expression(request_expr, Rc::clone(&env)) + .await?; + match &request_value { + Value::Object(props) => { + let props = props.borrow(); + let mut vars = Vec::new(); + for key in ["method", "path", "query", "client_ip", "body", "headers"] { + let value = props.get(key).ok_or_else(|| { + RuntimeError::new( + format!( + "Execute file request context is missing '{key}' - pass the request object from 'wait for request'" + ), + *line, + *column, + ) + })?; + let type_ok = match key { + "headers" => matches!(value, Value::Object(_)), + _ => matches!(value, Value::Text(_)), + }; + if !type_ok { + return Err(RuntimeError::new( + format!( + "Execute file request context field '{key}' must be {}, got {}", + if key == "headers" { + "an object" + } else { + "text" + }, + value.type_name() + ), + *line, + *column, + )); } + // Deep clone so the executed file cannot mutate the + // parent's request data (e.g. the headers object) + vars.push((key.to_string(), value.deep_clone())); } - None => { - log::warn!( - "WebSocket outbound frame for {conn_id} ({msg_len} bytes) exceeds the per-message or global queued-byte limit; dropping frame" - ); - } + vars + } + _ => { + return Err(RuntimeError::new( + format!( + "Execute file request context must be a request object, got {}", + request_value.type_name() + ), + *line, + *column, + )); } - Ok((Value::Null, ControlFlow::None)) } - None => Err(RuntimeError::new( - "That websocket connection is closed or unknown".to_string(), + } else { + Vec::new() + }; + + // Parse the file; errors are catchable in the parent + use crate::lexer::lex_wfl_with_positions_checked; + use crate::parser::Parser; + + // Lex under the shared run budget: a deadline / cancellation / + // operation breach during nested source loading surfaces as a + // typed, catchable runtime error instead of a truncated token + // stream that could execute as if it were the whole file. + let tokens = lex_wfl_with_positions_checked(&content) + .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; + let mut parser = Parser::new(&tokens); + let program = parser.parse().map_err(|errors| { + let first_error = errors.first(); + RuntimeError::new( + format!( + "Parse error in executed file '{}' (line {}, column {}): {}", + resolved_path.display(), + first_error.map(|e| e.line).unwrap_or(1), + first_error.map(|e| e.column).unwrap_or(1), + first_error.map(|e| e.message.as_str()).unwrap_or("unknown") + ), *line, *column, - )), + ) + })?; + + // Analyze semantics, seeding the injected request variable names. + // The type checker is intentionally skipped: main.rs treats type + // errors as warnings only, so a hard gate here would reject files + // that run fine standalone. + use crate::analyzer::Analyzer; + + let mut seeded_vars: HashMap = + HashMap::new(); + for (name, value) in &request_vars { + seeded_vars.insert(name.clone(), (Self::infer_type_from_value(value), true)); } - } - Statement::BroadcastWebSocketMessageStatement { - message, - server, - line, - column, - } => { - let message_val = self.evaluate_expression(message, Rc::clone(&env)).await?; - // Measure first; reject an oversized broadcast payload before it - // is materialized (and then cloned per recipient). - let msg_len = Self::ws_message_byte_len(&message_val, *line, *column)?; - if msg_len > self.budget.max_ws_message_bytes() { - log::warn!( - "WebSocket broadcast payload ({msg_len} bytes) exceeds the per-message limit; dropping broadcast" - ); - return Ok((Value::Null, ControlFlow::None)); + let mut analyzer = Analyzer::with_parent_variables(seeded_vars); + if let Err(errors) = analyzer.analyze(&program) { + let first_error = errors.first(); + return Err(RuntimeError::new( + format!( + "Semantic error in executed file '{}': {}", + resolved_path.display(), + first_error.map(|e| e.to_string()).unwrap_or_default() + ), + *line, + *column, + )); } - let text = Self::ws_message_text(&message_val, *line, *column)?; - let server_key = self - .resolve_ws_server_key(server, Rc::clone(&env), *line, *column) - .await?; + // Run the file in a fresh nested interpreter (own global env and + // stdlib, inherits the parent's config) with request context injected + let mut child = Interpreter::with_config(Arc::clone(&self.config)); + child.set_source_file(resolved_path.clone()); + child.execute_depth = self.execute_depth + 1; + // Share the parent's budget so the deadline, operation ceiling, + // and cancellation span the whole run — otherwise splitting work + // across `execute file` calls would reset them and evade the cap. + child.budget = Arc::clone(&self.budget); + // Seed the child's recursion accounting with the parent's live + // depth so the combined WFL call depth across nested `execute + // file` runs is bounded by `max_call_depth` (not multiplied per + // level), preventing native-stack overflow before the guard fires. + child.base_call_depth = self.call_depth.get(); - let ids: Vec = { - let servers = self.web_socket_servers.borrow(); - match servers.get(&server_key) { - Some(ws_server) => ws_server - .connection_ids - .lock() - .map(|g| g.clone()) - .unwrap_or_default(), - None => Vec::new(), + { + let mut child_env = child.global_env().borrow_mut(); + for (name, value) in request_vars { + if let Err(msg) = child_env.define(&name, value) { + return Err(RuntimeError::new(msg, *line, *column)); + } } + } + + // With an output clause, capture the child's display/print output; + // without one, child output flows to the current sink (stdout, or + // the parent's own capture buffer if the parent is being captured) + let capture_buffer = variable_name + .as_ref() + .map(|_| Rc::new(RefCell::new(String::new()))); + // The child shares this budget, so the parent's active main-loop + // exemption (a depth counter, not a flag) naturally covers the + // child and the nested front end — `execute file` from inside a + // server's `main loop` handler inherits the exemption instead of + // spuriously timing out, and the RAII guard needs no save/restore. + let run_result = { + let _guard = capture_buffer + .as_ref() + .map(|buffer| io_capture::push_capture(Rc::clone(buffer))); + // Box::pin breaks the recursive future (this statement awaits a + // full nested interpret), keeping the future finitely sized + Box::pin(child.interpret(&program)).await }; - if let Ok(map) = self.ws_connections.lock() { - for id in ids { - if let Some(tx) = map.get(&id) { - // Reserve each recipient's copy against the global - // queued-byte budget; shed over-budget frames rather - // than buffering them without bound. - match self.budget.try_reserve_ws_bytes(msg_len) { - Some(permit) => { - if let Err(err) = tx.try_send(WsOutbound::Text { - text: text.clone(), - _permit: permit, - }) { - log::warn!( - "WebSocket broadcast: outbound queue full/closed for {id}; dropping frame: {err}" - ); - } - } - None => { - log::warn!( - "WebSocket broadcast frame for {id} ({msg_len} bytes) exceeds the global queued-byte limit; dropping frame" - ); - } - } - } - } + if let Err(errors) = run_result { + let first = errors.into_iter().next().unwrap_or_else(|| { + RuntimeError::new("unknown error".to_string(), *line, *column) + }); + // Position the error at the parent's execute statement, keep the + // child's error kind so typed `when` clauses still match + return Err(RuntimeError::with_kind( + format!( + "Error in executed file '{}' (line {}): {}", + resolved_path.display(), + first.line, + first.message + ), + *line, + *column, + first.kind, + )); + } + + if let (Some(var_name), Some(buffer)) = (variable_name, capture_buffer) { + let output = buffer.borrow(); + env.borrow_mut() + .define(var_name, Value::Text(Arc::from(output.as_str()))) + .map_err(|e| RuntimeError::new(e, *line, *column))?; } Ok((Value::Null, ControlFlow::None)) } - // Subprocess statements - Statement::ExecuteCommandStatement { + Statement::SpawnProcessStatement { command, arguments, variable_name, @@ -8404,1184 +11740,1337 @@ impl Interpreter { )); } } - } else { - Vec::new() + } else { + Vec::new() + }; + + // Spawn process + let args_refs: Vec<&str> = args_vec.iter().map(|s| s.as_str()).collect(); + let process_id = self + .io_client + .spawn_process(cmd_str, &args_refs, *use_shell, *line, *column) + .await + .map_err(|e| { + let kind = if e.contains("program not found") + || e.contains("cannot find") + || e.contains("not recognized") + { + ErrorKind::CommandNotFound + } else { + ErrorKind::ProcessSpawnFailed + }; + RuntimeError::with_kind(e, *line, *column, kind) + })?; + + // Store process ID in variable + env.borrow_mut() + .define(variable_name, Value::Text(Arc::from(process_id.as_str()))) + .map_err(|e| RuntimeError::new(e, *line, *column))?; + + Ok((Value::Null, ControlFlow::None)) + } + Statement::ReadProcessOutputStatement { + process_id, + variable_name, + line, + column, + } => { + // Evaluate process ID expression + let proc_val = self + .evaluate_expression(process_id, Rc::clone(&env)) + .await?; + let proc_id = match &proc_val { + Value::Text(text) => text.as_ref(), + _ => { + return Err(RuntimeError::new( + format!("Process ID must be text, got {}", proc_val.type_name()), + *line, + *column, + )); + } + }; + + // Read process output + let output = self + .io_client + .read_process_output(proc_id) + .await + .map_err(|e| { + let kind = if e.contains("Invalid process ID") { + ErrorKind::ProcessNotFound + } else { + ErrorKind::General + }; + RuntimeError::with_kind(e, *line, *column, kind) + })?; + + // Store output in variable + env.borrow_mut() + .define(variable_name, Value::Text(Arc::from(output.as_str()))) + .map_err(|e| RuntimeError::new(e, *line, *column))?; + + Ok((Value::Null, ControlFlow::None)) + } + Statement::KillProcessStatement { + process_id, + line, + column, + } => { + // Evaluate process ID expression + let proc_val = self + .evaluate_expression(process_id, Rc::clone(&env)) + .await?; + let proc_id = match &proc_val { + Value::Text(text) => text.as_ref(), + _ => { + return Err(RuntimeError::new( + format!("Process ID must be text, got {}", proc_val.type_name()), + *line, + *column, + )); + } + }; + + // Kill process + self.io_client.kill_process(proc_id).await.map_err(|e| { + let kind = if e.contains("Invalid process ID") { + ErrorKind::ProcessNotFound + } else { + ErrorKind::ProcessKillFailed + }; + RuntimeError::with_kind(e, *line, *column, kind) + })?; + + Ok((Value::Null, ControlFlow::None)) + } + Statement::WaitForProcessStatement { + process_id, + variable_name, + line, + column, + } => { + // Evaluate process ID expression + let proc_val = self + .evaluate_expression(process_id, Rc::clone(&env)) + .await?; + let proc_id = match &proc_val { + Value::Text(text) => text.as_ref(), + _ => { + return Err(RuntimeError::new( + format!("Process ID must be text, got {}", proc_val.type_name()), + *line, + *column, + )); + } }; - // Execute command - let args_refs: Vec<&str> = args_vec.iter().map(|s| s.as_str()).collect(); - let (stdout, stderr, exit_code) = self + // Wait for process to complete + let exit_code = self .io_client - .execute_command(cmd_str, &args_refs, *use_shell, *line, *column) + .wait_for_process(proc_id) .await - .map_err(|e| match e { - ExecuteCommandError::Budget(exceeded) => { - self.budget_error(exceeded, *line, *column) - } - ExecuteCommandError::Timeout { seconds } => RuntimeError::with_kind( - format!("Subprocess execution exceeded timeout ({seconds}s)"), - *line, - *column, - ErrorKind::Timeout, - ), - ExecuteCommandError::Other(message) => { - // Preserve the existing subprocess error - // classification for non-budget failures. - let kind = if message.contains("program not found") - || message.contains("cannot find") - || message.contains("not recognized") - { - ErrorKind::CommandNotFound - } else if message.contains("spawn") { - ErrorKind::ProcessSpawnFailed - } else { - ErrorKind::General - }; - RuntimeError::with_kind(message, *line, *column, kind) - } + .map_err(|e| { + let kind = if e.contains("Invalid process ID") { + ErrorKind::ProcessNotFound + } else { + ErrorKind::General + }; + RuntimeError::with_kind(e, *line, *column, kind) })?; - // Build result object - let mut result_map = HashMap::new(); - result_map.insert( - "output".to_string(), - Value::Text(Arc::from(stdout.as_str())), - ); - result_map.insert("error".to_string(), Value::Text(Arc::from(stderr.as_str()))); - result_map.insert("exit_code".to_string(), Value::Number(exit_code as f64)); - result_map.insert("success".to_string(), Value::Bool(exit_code == 0)); - - let result_obj = Value::Object(Rc::new(RefCell::new(result_map))); - - // Store result if variable name provided + // Store exit code in variable if provided if let Some(var_name) = variable_name { env.borrow_mut() - .define(var_name, result_obj) + .define(var_name, Value::Number(exit_code as f64)) .map_err(|e| RuntimeError::new(e, *line, *column))?; } Ok((Value::Null, ControlFlow::None)) } - Statement::ExecuteFileStatement { - path, - request, - variable_name, + // Test framework statements + Statement::DescribeBlock { + description, + setup, + teardown, + tests, line, column, } => { - // Guard against a file that (directly or indirectly) executes - // itself, using the shared budget's execute-file depth ceiling. - if let Err(exceeded) = self.budget.check_execute_file_depth(self.execute_depth) { - return Err(self.budget_error(exceeded, *line, *column)); + if !*self.test_mode.borrow() { + return Err(RuntimeError::new( + "describe blocks can only be used in test mode (run with --test flag)" + .to_string(), + *line, + *column, + )); } - // Evaluate path expression to string - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str: String = match &path_value { - Value::Text(s) => s.to_string(), - _ => { - return Err(RuntimeError::new( - format!( - "Execute file path must be text, got {}", - path_value.type_name() - ), - *line, - *column, - )); + // Push describe context + self.current_describe_stack + .borrow_mut() + .push(description.clone()); + + // Create describe-level environment for setup/teardown sharing + // This allows tests to access setup variables while remaining isolated from each other + let describe_env = Environment::new_child_env(&env); + + // Run setup if present (runs in describe environment). + // Setup/tests/teardown are three separate statement blocks, + // each with its own overload-duplicate scope. + if let Some(setup_stmts) = setup { + let (_overload_dups, _armed_members) = + self.enter_block_overloads(setup_stmts, &describe_env); + for stmt in setup_stmts { + Box::pin(self._execute_statement(stmt, describe_env.clone())).await?; } - }; + } - // Resolve relative to the current script's directory (like load module), - // mapping a missing file to FileNotFound so `when file not found` works - let opt_source = self.current_source_file.borrow().as_ref().cloned(); - let joined = if let Some(source_path) = opt_source { - source_path - .parent() - .map(|dir| dir.join(&path_str)) - .unwrap_or_else(|| PathBuf::from(&path_str)) - } else { - let cwd = std::env::current_dir().map_err(|e| { - RuntimeError::new( - format!("Cannot determine current directory: {e}"), - *line, - *column, - ) - })?; - cwd.join(&path_str) - }; - let map_io_error = |e: std::io::Error| { - let kind = match e.kind() { - std::io::ErrorKind::NotFound => ErrorKind::FileNotFound, - std::io::ErrorKind::PermissionDenied => ErrorKind::PermissionDenied, - _ => ErrorKind::General, - }; - RuntimeError::with_kind( - format!("Cannot execute wfl file '{path_str}': {e}"), + // Execute all tests (each gets a child of describe_env for isolation) + { + let (_overload_dups, _armed_members) = + self.enter_block_overloads(tests, &describe_env); + for test in tests { + Box::pin(self._execute_statement(test, describe_env.clone())).await?; + } + } + + // Run teardown if present (runs in describe environment) + if let Some(teardown_stmts) = teardown { + let (_overload_dups, _armed_members) = + self.enter_block_overloads(teardown_stmts, &describe_env); + for stmt in teardown_stmts { + Box::pin(self._execute_statement(stmt, describe_env.clone())).await?; + } + } + + // Pop describe context + self.current_describe_stack.borrow_mut().pop(); + + Ok((Value::Null, ControlFlow::None)) + } + Statement::TestBlock { + description, + body, + line, + column, + } => { + if !*self.test_mode.borrow() { + return Err(RuntimeError::new( + "test blocks can only be used in test mode (run with --test flag)" + .to_string(), *line, *column, - kind, - ) - }; - let resolved_path = tokio::fs::canonicalize(&joined) - .await - .map_err(map_io_error)?; - // Read under the shared source-size ceiling (bounded read). - let content = self - .read_source_bounded(&resolved_path, *line, *column) - .await?; + )); + } - // Evaluate the optional request context and extract the variables - // that `wait for request` defines, so the executed file sees the - // same names. Validate the shape upfront so a wrong object fails - // here with a clear message instead of as confusing undefined - // variable errors inside the executed file. - let request_vars: Vec<(String, Value)> = if let Some(request_expr) = request { - let request_value = self - .evaluate_expression(request_expr, Rc::clone(&env)) - .await?; - match &request_value { - Value::Object(props) => { - let props = props.borrow(); - let mut vars = Vec::new(); - for key in ["method", "path", "query", "client_ip", "body", "headers"] { - let value = props.get(key).ok_or_else(|| { - RuntimeError::new( - format!( - "Execute file request context is missing '{key}' - pass the request object from 'wait for request'" - ), - *line, - *column, - ) - })?; - let type_ok = match key { - "headers" => matches!(value, Value::Object(_)), - _ => matches!(value, Value::Text(_)), + // Set current test name for failure tracking + *self.current_test_name.borrow_mut() = Some(description.clone()); + + // Increment test count + self.test_results.borrow_mut().total_tests += 1; + + // Create isolated environment for test (child of describe env) + // Using isolated mode prevents tests from mutating setup variables, + // ensuring each test gets a fresh copy for true isolation + let test_env = Environment::new_isolated_child_env(&env); + + // Execute test body and catch assertion failures. The body + // is its own statement block for overload enforcement. + let (_overload_dups, _armed_members) = self.enter_block_overloads(body, &test_env); + let mut test_passed = true; + + for stmt in body { + match Box::pin(self._execute_statement(stmt, test_env.clone())).await { + Ok(_) => {} + Err(e) => { + test_passed = false; + + // Assertion failures are already recorded (failure entry + + // failed_tests increment) by the ExpectStatement handler; its + // raw message begins with "Assertion failed:". We must inspect + // the raw `message` field here rather than the Display string, + // which is prefixed with "Runtime error at line ...:" and would + // otherwise never match the guard, causing the assertion to be + // recorded twice. Any error that is NOT an assertion failure is a + // runtime error in the test body that we record and count here so + // it is reflected in the failure count and the process exit code. + if !e.message.starts_with("Assertion failed:") { + let context = self.current_describe_stack.borrow().clone(); + let failure = TestFailure { + describe_context: context, + test_name: description.clone(), + assertion_message: e.to_string(), + line: *line, + column: *column, }; - if !type_ok { - return Err(RuntimeError::new( - format!( - "Execute file request context field '{key}' must be {}, got {}", - if key == "headers" { - "an object" - } else { - "text" - }, - value.type_name() - ), - *line, - *column, - )); - } - // Deep clone so the executed file cannot mutate the - // parent's request data (e.g. the headers object) - vars.push((key.to_string(), value.deep_clone())); + let mut results = self.test_results.borrow_mut(); + results.failures.push(failure); + results.failed_tests += 1; } - vars - } - _ => { - return Err(RuntimeError::new( - format!( - "Execute file request context must be a request object, got {}", - request_value.type_name() - ), - *line, - *column, - )); + + // Don't propagate the error - continue running other tests + break; } } - } else { - Vec::new() - }; + } - // Parse the file; errors are catchable in the parent - use crate::lexer::lex_wfl_with_positions_checked; - use crate::parser::Parser; + if test_passed { + self.test_results.borrow_mut().passed_tests += 1; + } - // Lex under the shared run budget: a deadline / cancellation / - // operation breach during nested source loading surfaces as a - // typed, catchable runtime error instead of a truncated token - // stream that could execute as if it were the whole file. - let tokens = lex_wfl_with_positions_checked(&content) - .map_err(|exceeded| self.budget_error(exceeded, *line, *column))?; - let mut parser = Parser::new(&tokens); - let program = parser.parse().map_err(|errors| { - let first_error = errors.first(); - RuntimeError::new( - format!( - "Parse error in executed file '{}' (line {}, column {}): {}", - resolved_path.display(), - first_error.map(|e| e.line).unwrap_or(1), - first_error.map(|e| e.column).unwrap_or(1), - first_error.map(|e| e.message.as_str()).unwrap_or("unknown") - ), + // Clear current test name + *self.current_test_name.borrow_mut() = None; + + Ok((Value::Null, ControlFlow::None)) + } + Statement::ExpectStatement { + subject, + assertion, + line, + column, + } => { + if !*self.test_mode.borrow() { + return Err(RuntimeError::new( + "expect statements can only be used in test mode (run with --test flag)" + .to_string(), *line, *column, - ) - })?; + )); + } - // Analyze semantics, seeding the injected request variable names. - // The type checker is intentionally skipped: main.rs treats type - // errors as warnings only, so a hard gate here would reject files - // that run fine standalone. - use crate::analyzer::Analyzer; + // Evaluate subject expression + let subject_value = self.evaluate_expression(subject, env.clone()).await?; + + // Check assertion + let (passed, expected_value) = self + .check_assertion(&subject_value, assertion, env.clone()) + .await?; + + if !passed { + // Record failure with proper test name tracking + let message = self.create_assertion_message_with_values( + assertion, + &subject_value, + expected_value.as_ref(), + ); + let context = self.current_describe_stack.borrow().clone(); + let test_name = self + .current_test_name + .borrow() + .clone() + .unwrap_or_else(|| "unknown test".to_string()); + + let failure = TestFailure { + describe_context: context, + test_name, + assertion_message: message.clone(), + line: *line, + column: *column, + }; + + self.test_results.borrow_mut().failures.push(failure); + self.test_results.borrow_mut().failed_tests += 1; - let mut seeded_vars: HashMap = - HashMap::new(); - for (name, value) in &request_vars { - seeded_vars.insert(name.clone(), (Self::infer_type_from_value(value), true)); - } - let mut analyzer = Analyzer::with_parent_variables(seeded_vars); - if let Err(errors) = analyzer.analyze(&program) { - let first_error = errors.first(); return Err(RuntimeError::new( - format!( - "Semantic error in executed file '{}': {}", - resolved_path.display(), - first_error.map(|e| e.to_string()).unwrap_or_default() - ), + format!("Assertion failed: {message}"), *line, *column, )); } - // Run the file in a fresh nested interpreter (own global env and - // stdlib, inherits the parent's config) with request context injected - let mut child = Interpreter::with_config(Arc::clone(&self.config)); - child.set_source_file(resolved_path.clone()); - child.execute_depth = self.execute_depth + 1; - // Share the parent's budget so the deadline, operation ceiling, - // and cancellation span the whole run — otherwise splitting work - // across `execute file` calls would reset them and evade the cap. - child.budget = Arc::clone(&self.budget); - // Seed the child's recursion accounting with the parent's live - // depth so the combined WFL call depth across nested `execute - // file` runs is bounded by `max_call_depth` (not multiplied per - // level), preventing native-stack overflow before the guard fires. - child.base_call_depth = self.call_depth.get(); + Ok((Value::Null, ControlFlow::None)) + } + }; + + if self.step_mode { + self.dump_state(stmt, line, column, &env_before); + if !self.prompt_continue() { + std::process::exit(0); + } + } + + result + } + + /// Resolves a server expression to the key of a running WebSocket server. + async fn resolve_ws_server_key( + &self, + server: &Expression, + env: Rc>, + line: usize, + column: usize, + ) -> Result { + let value = self.evaluate_expression(server, env).await?; + match &value { + Value::Text(t) => { + let key = t.to_string(); + if self.web_socket_servers.borrow().contains_key(&key) { + Ok(key) + } else { + Err(RuntimeError::new( + format!("'{key}' is not a running websocket server"), + line, + column, + )) + } + } + _ => Err(RuntimeError::new( + "Expected a websocket server (the name bound by 'listen for websockets')" + .to_string(), + line, + column, + )), + } + } + + /// Coerces a WFL value into the text payload of a websocket frame. + fn ws_message_text(value: &Value, line: usize, column: usize) -> Result { + match value { + Value::Text(t) => Ok(t.to_string()), + Value::Number(n) => Ok(format!("{n}")), + Value::Bool(b) => Ok(if *b { "yes" } else { "no" }.to_string()), + Value::Null | Value::Nothing => Err(RuntimeError::new( + "Cannot send an empty websocket message".to_string(), + line, + column, + )), + other => Err(RuntimeError::new( + format!( + "Cannot send a {} as a websocket message; expected text", + other.type_name() + ), + line, + column, + )), + } + } + + /// The byte length a `send`/`broadcast` value would serialize to, computed + /// from the *borrowed* value — for `Value::Text` (`Arc`) this is + /// `t.len()` with no allocation. Lets the queued-byte permit be reserved + /// (and an oversized message rejected) *before* the payload is cloned into a + /// `String`, so an oversized runtime value is never fully duplicated first. + fn ws_message_byte_len( + value: &Value, + line: usize, + column: usize, + ) -> Result { + match value { + Value::Text(t) => Ok(t.len()), + // Small, bounded scalars — measuring == materializing cost. + Value::Number(n) => Ok(format!("{n}").len()), + Value::Bool(b) => Ok(if *b { 3 } else { 2 }), + // Reuse `ws_message_text`'s errors for the unsupported cases. + _ => Self::ws_message_text(value, line, column).map(|s| s.len()), + } + } + + /// Extracts a connection id from a `send ... to ` target. Accepts the + /// connection object bound by a connect/message handler (reads its `id`). + fn ws_connection_id(value: &Value, line: usize, column: usize) -> Result { + if let Value::Object(map) = value + && let Some(Value::Text(id)) = map.borrow().get("id") + { + return Ok(id.to_string()); + } + Err(RuntimeError::new( + "Expected a websocket connection (the value bound by 'on websocket connect/message')" + .to_string(), + line, + column, + )) + } + + /// Drains and dispatches queued websocket events for up to `budget`, running + /// the matching handler block for each. With no websocket servers active it + /// is a plain sleep, preserving `wait for ` semantics. + async fn pump_websocket_events(&self, budget: Duration) -> Result<(), RuntimeError> { + let deadline = tokio::time::Instant::now() + budget; + loop { + let now = tokio::time::Instant::now(); + if now >= deadline { + break; + } + let remaining = deadline - now; + + // Snapshot the receivers with a short borrow; dispatch below must not + // hold a borrow of web_socket_servers across handler execution. + let receivers: Vec<(String, Arc>>)> = + self.web_socket_servers + .borrow() + .iter() + .map(|(key, srv)| (key.clone(), Arc::clone(&srv.event_receiver))) + .collect(); - { - let mut child_env = child.global_env().borrow_mut(); - for (name, value) in request_vars { - if let Err(msg) = child_env.define(&name, value) { - return Err(RuntimeError::new(msg, *line, *column)); - } + if receivers.is_empty() { + tokio::time::sleep(remaining).await; + break; + } + + let mut recv_futs = Vec::with_capacity(receivers.len()); + for (key, rx) in receivers { + recv_futs.push(Box::pin(async move { + let mut guard = rx.lock().await; + let event = guard.recv().await; + (key, event) + })); + } + + let sleep_fut = tokio::time::sleep(remaining); + tokio::pin!(sleep_fut); + + tokio::select! { + _ = &mut sleep_fut => break, + ((key, event), _idx, _rest) = futures_util::future::select_all(recv_futs) => { + if let Some(event) = event { + self.dispatch_ws_event(&key, event).await?; } + // A `None` means that server's channel closed; the next loop + // iteration rebuilds the receiver set. } + } + } + Ok(()) + } - // With an output clause, capture the child's display/print output; - // without one, child output flows to the current sink (stdout, or - // the parent's own capture buffer if the parent is being captured) - let capture_buffer = variable_name - .as_ref() - .map(|_| Rc::new(RefCell::new(String::new()))); - // The child shares this budget, so the parent's active main-loop - // exemption (a depth counter, not a flag) naturally covers the - // child and the nested front end — `execute file` from inside a - // server's `main loop` handler inherits the exemption instead of - // spuriously timing out, and the RAII guard needs no save/restore. - let run_result = { - let _guard = capture_buffer - .as_ref() - .map(|buffer| io_capture::push_capture(Rc::clone(buffer))); - // Box::pin breaks the recursive future (this statement awaits a - // full nested interpret), keeping the future finitely sized - Box::pin(child.interpret(&program)).await - }; + /// Runs the registered handler block for one websocket event. Handler errors + /// are reported but do not tear down the server, matching event-driven norms. + async fn dispatch_ws_event( + &self, + server_key: &str, + event: WflWsEvent, + ) -> Result<(), RuntimeError> { + // Clone the selected handler out before executing: the body may itself + // register handlers or send frames, which re-borrow web_socket_servers. + let handler = { + let servers = self.web_socket_servers.borrow(); + let Some(server) = servers.get(server_key) else { + return Ok(()); + }; + let handlers = server.handlers.borrow(); + let selected = match event.kind { + WsEventKind::Connect => handlers.connect.as_ref(), + WsEventKind::Message => handlers.message.as_ref(), + WsEventKind::Disconnect => handlers.disconnect.as_ref(), + }; + selected.map(|h| (h.binding.clone(), h.body.clone(), Rc::clone(&h.env))) + }; - if let Err(errors) = run_result { - let first = errors.into_iter().next().unwrap_or_else(|| { - RuntimeError::new("unknown error".to_string(), *line, *column) - }); - // Position the error at the parent's execute statement, keep the - // child's error kind so typed `when` clauses still match - return Err(RuntimeError::with_kind( - format!( - "Error in executed file '{}' (line {}): {}", - resolved_path.display(), - first.line, - first.message - ), - *line, - *column, - first.kind, - )); - } + let Some((binding, body, handler_env)) = handler else { + return Ok(()); + }; - if let (Some(var_name), Some(buffer)) = (variable_name, capture_buffer) { - let output = buffer.borrow(); - env.borrow_mut() - .define(var_name, Value::Text(Arc::from(output.as_str()))) - .map_err(|e| RuntimeError::new(e, *line, *column))?; - } + let event_obj = build_ws_event_object(&event); + let child_env = Environment::new_child_env(&handler_env); + // `define_direct` shadows in the fresh handler scope without consulting + // parents, so a same-named outer variable (`store conn as ...` before an + // `on websocket connect ... as conn`) does not abort the handler. + if let Err(msg) = child_env.borrow_mut().define_direct(&binding, event_obj) { + return Err(RuntimeError::new(msg, 0, 0)); + } - Ok((Value::Null, ControlFlow::None)) - } - Statement::SpawnProcessStatement { - command, - arguments, - variable_name, - use_shell, - line, - column, - } => { - // Evaluate command expression - let cmd_val = self.evaluate_expression(command, Rc::clone(&env)).await?; - let cmd_str = match &cmd_val { - Value::Text(text) => text.as_ref(), - _ => { - return Err(RuntimeError::new( - format!("Command must be text, got {}", cmd_val.type_name()), - *line, - *column, - )); - } - }; + if let Err(err) = self.execute_block(&body, child_env).await { + eprintln!( + "WebSocket {} handler error: {}", + event.kind.as_str(), + err.message + ); + } + Ok(()) + } - // Evaluate arguments if provided - let args_vec: Vec = if let Some(args_expr) = arguments { - let args_val = self.evaluate_expression(args_expr, Rc::clone(&env)).await?; - match &args_val { - Value::List(list) => { - let list_ref = list.borrow(); - list_ref - .iter() - .map(|v| match v { - Value::Text(t) => Ok(t.as_ref().to_string()), - _ => Ok(v.to_string()), - }) - .collect::, RuntimeError>>()? - } - Value::Text(text) => vec![text.as_ref().to_string()], - _ => { - return Err(RuntimeError::new( - format!( - "Arguments must be a list or text, got {}", - args_val.type_name() - ), - *line, - *column, - )); - } - } - } else { - Vec::new() - }; + async fn execute_block( + &self, + statements: &[Statement], + env: Rc>, + ) -> Result<(Value, ControlFlow), RuntimeError> { + Box::pin(self._execute_block(statements, env)).await + } - // Spawn process - let args_refs: Vec<&str> = args_vec.iter().map(|s| s.as_str()).collect(); - let process_id = self - .io_client - .spawn_process(cmd_str, &args_refs, *use_shell, *line, *column) - .await - .map_err(|e| { - let kind = if e.contains("program not found") - || e.contains("cannot find") - || e.contains("not recognized") - { - ErrorKind::CommandNotFound - } else { - ErrorKind::ProcessSpawnFailed - }; - RuntimeError::with_kind(e, *line, *column, kind) - })?; + async fn _execute_block( + &self, + statements: &[Statement], + env: Rc>, + ) -> Result<(Value, ControlFlow), RuntimeError> { + self.assert_invariants(); + // Each block carries its own overload-duplicate set for temporal + // enforcement (restored on drop, including `?` early returns), and + // arms same-scope members its definitions will merge with. + let (_overload_dups, _armed_members) = self.enter_block_overloads(statements, &env); + let mut last_value = Value::Null; - // Store process ID in variable - env.borrow_mut() - .define(variable_name, Value::Text(Arc::from(process_id.as_str()))) - .map_err(|e| RuntimeError::new(e, *line, *column))?; + #[cfg(debug_assertions)] + exec_trace!("Executing block of {} statements", statements.len()); - Ok((Value::Null, ControlFlow::None)) - } - Statement::ReadProcessOutputStatement { - process_id, - variable_name, - line, - column, - } => { - // Evaluate process ID expression - let proc_val = self - .evaluate_expression(process_id, Rc::clone(&env)) - .await?; - let proc_id = match &proc_val { - Value::Text(text) => text.as_ref(), - _ => { - return Err(RuntimeError::new( - format!("Process ID must be text, got {}", proc_val.type_name()), - *line, - *column, - )); - } - }; + #[cfg(debug_assertions)] + let _guard = IndentGuard::new(); - // Read process output - let output = self - .io_client - .read_process_output(proc_id) - .await - .map_err(|e| { - let kind = if e.contains("Invalid process ID") { - ErrorKind::ProcessNotFound - } else { - ErrorKind::General - }; - RuntimeError::with_kind(e, *line, *column, kind) - })?; + let mut control_flow = ControlFlow::None; - // Store output in variable - env.borrow_mut() - .define(variable_name, Value::Text(Arc::from(output.as_str()))) - .map_err(|e| RuntimeError::new(e, *line, *column))?; + for statement in statements { + let result = self.execute_statement(statement, Rc::clone(&env)).await?; + last_value = result.0; + control_flow = result.1; - Ok((Value::Null, ControlFlow::None)) + if !matches!(control_flow, ControlFlow::None) { + #[cfg(debug_assertions)] + exec_trace!( + "Block execution interrupted by control flow: {:?}", + control_flow + ); + break; } - Statement::KillProcessStatement { - process_id, - line, - column, - } => { - // Evaluate process ID expression - let proc_val = self - .evaluate_expression(process_id, Rc::clone(&env)) - .await?; - let proc_id = match &proc_val { - Value::Text(text) => text.as_ref(), - _ => { - return Err(RuntimeError::new( - format!("Process ID must be text, got {}", proc_val.type_name()), - *line, - *column, - )); - } - }; + } - // Kill process - self.io_client.kill_process(proc_id).await.map_err(|e| { - let kind = if e.contains("Invalid process ID") { - ErrorKind::ProcessNotFound - } else { - ErrorKind::ProcessKillFailed - }; - RuntimeError::with_kind(e, *line, *column, kind) - })?; + self.assert_invariants(); + Ok((last_value, control_flow)) + } - Ok((Value::Null, ControlFlow::None)) + /// Attempts to recycle a loop iteration environment to avoid heap allocation. + /// + /// Recycling is only safe when all three conditions are met: + /// 1. We are the sole owner (`strong_count == 1`) — no other code holds a reference. + /// 2. No weak references exist (`weak_count == 0`) — closures capture environments via + /// `Weak` refs, so any weak ref means a closure may still need the environment's state. + /// 3. The recycled environment's parent matches the expected `parent` — prevents scoping + /// bugs where an environment from a different scope is reused with the wrong parent chain. + /// + /// If any condition fails, a fresh child environment is allocated instead. + fn get_recycled_env( + &self, + reusable_env: Option>>, + parent: &Rc>, + ) -> Rc> { + if let Some(env) = reusable_env + && Rc::strong_count(&env) == 1 + && Rc::weak_count(&env) == 0 + { + // Validate parent matches to prevent scoping bugs + let parent_matches = env + .borrow() + .parent + .as_ref() + .is_some_and(|p| p.ptr_eq(&Rc::downgrade(parent))); + if parent_matches { + env.borrow_mut().clear(); + return env; } - Statement::WaitForProcessStatement { - process_id, - variable_name, - line, - column, - } => { - // Evaluate process ID expression - let proc_val = self - .evaluate_expression(process_id, Rc::clone(&env)) - .await?; - let proc_id = match &proc_val { - Value::Text(text) => text.as_ref(), - _ => { - return Err(RuntimeError::new( - format!("Process ID must be text, got {}", proc_val.type_name()), - *line, - *column, - )); - } - }; - - // Wait for process to complete - let exit_code = self - .io_client - .wait_for_process(proc_id) - .await - .map_err(|e| { - let kind = if e.contains("Invalid process ID") { - ErrorKind::ProcessNotFound - } else { - ErrorKind::General - }; - RuntimeError::with_kind(e, *line, *column, kind) - })?; + } + Environment::new_child_env(parent) + } - // Store exit code in variable if provided - if let Some(var_name) = variable_name { - env.borrow_mut() - .define(var_name, Value::Number(exit_code as f64)) - .map_err(|e| RuntimeError::new(e, *line, *column))?; + // Helper to evaluate literals directly without Box::pin allocation + fn evaluate_literal_direct( + &self, + literal: &Literal, + env: &Rc>, + line: usize, + column: usize, + ) -> Result, RuntimeError> { + match literal { + Literal::String(s) => Ok(Some(Value::Text(s.clone()))), + Literal::Integer(i) => Ok(Some(Value::Number(*i as f64))), + Literal::Float(f) => Ok(Some(Value::Number(*f))), + Literal::Boolean(b) => Ok(Some(Value::Bool(*b))), + Literal::Nothing => Ok(Some(Value::Null)), + // Pattern literals might error, so we can handle them here + Literal::Pattern(ir_string) => self + .compile_pattern_literal(ir_string, env, line, column) + .map(Some), + Literal::List(elements) => { + // First, pre-scan all elements to detect if any require async evaluation + // This prevents double execution of side effects + for element in elements { + if self.requires_async_evaluation(element, env) { + // At least one element requires async, abort sync optimization for the whole list + return Ok(None); + } } - Ok((Value::Null, ControlFlow::None)) + // All elements can be evaluated synchronously, proceed safely + let mut list_values = Vec::with_capacity(elements.len()); + for element in elements { + // Since we've already verified all elements are sync-compatible, + // this should never return None, but handle it gracefully just in case + if let Some(value) = self.try_evaluate_simple_expr_sync(element, env)? { + list_values.push(value); + } else { + // This shouldn't happen after our pre-scan, but fall back to async + return Ok(None); + } + } + Ok(Some(Value::List(Rc::new(RefCell::new(list_values))))) } - // Test framework statements - Statement::DescribeBlock { - description, - setup, - teardown, - tests, + } + } + + /// Compiles a pattern literal string into a Value::Pattern + fn compile_pattern_literal( + &self, + ir_string: &str, + env: &Rc>, + line: usize, + column: usize, + ) -> Result { + let pattern_expr = crate::parser::ast::PatternExpression::Literal(ir_string.to_string()); + let compiled_pattern = { + let env_borrow = env.borrow(); + CompiledPattern::compile_with_env(&pattern_expr, &env_borrow) + }; + match compiled_pattern { + Ok(compiled) => Ok(Value::Pattern(Rc::new(compiled))), + Err(e) => Err(RuntimeError::new( + format!("Failed to compile pattern literal: {}", e), line, column, - } => { - if !*self.test_mode.borrow() { - return Err(RuntimeError::new( - "describe blocks can only be used in test mode (run with --test flag)" - .to_string(), - *line, - *column, - )); - } - - // Push describe context - self.current_describe_stack - .borrow_mut() - .push(description.clone()); - - // Create describe-level environment for setup/teardown sharing - // This allows tests to access setup variables while remaining isolated from each other - let describe_env = Environment::new_child_env(&env); + )), + } + } - // Run setup if present (runs in describe environment). - // Setup/tests/teardown are three separate statement blocks, - // each with its own overload-duplicate scope. - if let Some(setup_stmts) = setup { - let (_overload_dups, _armed_members) = - self.enter_block_overloads(setup_stmts, &describe_env); - for stmt in setup_stmts { - Box::pin(self._execute_statement(stmt, describe_env.clone())).await?; + /// Probes whether an expression requires async evaluation without executing it. + /// + /// Returns `true` if the expression must be evaluated asynchronously (e.g., it contains + /// a zero-argument user-defined function that would trigger auto-call), or `false` if + /// it can be safely evaluated on the synchronous fast-path. Used to pre-scan list + /// elements so we can avoid double-execution of side effects. + fn requires_async_evaluation(&self, expr: &Expression, env: &Rc>) -> bool { + match expr { + Expression::Literal(literal, _line, _column) => { + match literal { + Literal::Pattern(_) => false, // Patterns don't require async + Literal::List(elements) => { + // Recursively check all elements + elements + .iter() + .any(|element| self.requires_async_evaluation(element, env)) + } + _ => false, // Other literals are synchronous + } + } + Expression::Variable(name, _line, _column) => { + // Check if variable exists and if it would require async auto-call + if let Ok(env_borrowed) = env.try_borrow() { + if let Some(value) = env_borrowed.get(name) { + match &value { + Value::Function(func) => func.params.is_empty(), // Zero-arg user functions auto-call (async) + Value::Overloaded(overloaded) => overloaded + .overloads + .iter() + .any(|func| func.params.is_empty()), + Value::NativeFunction(_, _) => false, // Native functions evaluate sync + _ => false, + } + } else { + false // Variable doesn't exist, will be sync error } + } else { + true // Can't borrow environment, conservatively assume async } + } + Expression::UnaryOperation { expression, .. } => { + self.requires_async_evaluation(expression, env) + } + Expression::BinaryOperation { left, right, .. } => { + self.requires_async_evaluation(left, env) + || self.requires_async_evaluation(right, env) + } + Expression::Concatenation { left, right, .. } => { + self.requires_async_evaluation(left, env) + || self.requires_async_evaluation(right, env) + } + _ => true, // All other expressions require async (function calls, etc.) + } + } - // Execute all tests (each gets a child of describe_env for isolation) - { - let (_overload_dups, _armed_members) = - self.enter_block_overloads(tests, &describe_env); - for test in tests { - Box::pin(self._execute_statement(test, describe_env.clone())).await?; - } + /// Handles the WFL auto-call convention for variables that resolve to functions. + /// + /// When a variable holds a zero-argument native function, it is invoked immediately + /// and the result is returned. For zero-argument user-defined functions (which require + /// async execution), returns `Ok(None)` to signal fallback to the async path. + /// Non-function values and functions with parameters are returned as-is. + fn handle_variable_auto_call( + &self, + value: Value, + line: usize, + column: usize, + ) -> Result, RuntimeError> { + match &value { + Value::NativeFunction(func_name, native_fn) => { + if get_function_arity(func_name) == 0 { + // Native functions are synchronous + native_fn(vec![]) + .map(Some) + .map_err(|e| RuntimeError::new(format!("{}", e), line, column)) + } else { + Ok(Some(value)) } - - // Run teardown if present (runs in describe environment) - if let Some(teardown_stmts) = teardown { - let (_overload_dups, _armed_members) = - self.enter_block_overloads(teardown_stmts, &describe_env); - for stmt in teardown_stmts { - Box::pin(self._execute_statement(stmt, describe_env.clone())).await?; - } + } + Value::Function(func) => { + if func.params.is_empty() { + // User functions are async -> return None to signal fallback to async + Ok(None) + } else { + Ok(Some(value)) } + } + Value::Overloaded(overloaded) => { + if overloaded + .overloads + .iter() + .any(|func| func.params.is_empty()) + { + // The zero-arg overload auto-calls (async path) + Ok(None) + } else { + Ok(Some(value)) + } + } + _ => Ok(Some(value)), + } + } - // Pop describe context - self.current_describe_stack.borrow_mut().pop(); - - Ok((Value::Null, ControlFlow::None)) + /// Attempts to evaluate an expression synchronously to avoid `Box::pin` allocation overhead. + /// + /// Handles literals, variables, and simple binary/unary operations recursively. + /// Returns `Ok(Some(value))` when the expression was fully evaluated on the sync path, + /// or `Ok(None)` when async evaluation is required (e.g., function calls, complex expressions). + fn try_evaluate_simple_expr_sync( + &self, + expr: &Expression, + env: &Rc>, + ) -> Result, RuntimeError> { + match expr { + Expression::Literal(literal, line, column) => { + self.evaluate_literal_direct(literal, env, *line, *column) } - Statement::TestBlock { - description, - body, + Expression::Variable(name, line, column) => { + self.try_evaluate_variable_sync(name, env, *line, *column) + } + Expression::UnaryOperation { + operator, + expression, line, column, } => { - if !*self.test_mode.borrow() { - return Err(RuntimeError::new( - "test blocks can only be used in test mode (run with --test flag)" - .to_string(), - *line, - *column, - )); - } - - // Set current test name for failure tracking - *self.current_test_name.borrow_mut() = Some(description.clone()); - - // Increment test count - self.test_results.borrow_mut().total_tests += 1; - - // Create isolated environment for test (child of describe env) - // Using isolated mode prevents tests from mutating setup variables, - // ensuring each test gets a fresh copy for true isolation - let test_env = Environment::new_isolated_child_env(&env); - - // Execute test body and catch assertion failures. The body - // is its own statement block for overload enforcement. - let (_overload_dups, _armed_members) = self.enter_block_overloads(body, &test_env); - let mut test_passed = true; - - for stmt in body { - match Box::pin(self._execute_statement(stmt, test_env.clone())).await { - Ok(_) => {} - Err(e) => { - test_passed = false; - - // Assertion failures are already recorded (failure entry + - // failed_tests increment) by the ExpectStatement handler; its - // raw message begins with "Assertion failed:". We must inspect - // the raw `message` field here rather than the Display string, - // which is prefixed with "Runtime error at line ...:" and would - // otherwise never match the guard, causing the assertion to be - // recorded twice. Any error that is NOT an assertion failure is a - // runtime error in the test body that we record and count here so - // it is reflected in the failure count and the process exit code. - if !e.message.starts_with("Assertion failed:") { - let context = self.current_describe_stack.borrow().clone(); - let failure = TestFailure { - describe_context: context, - test_name: description.clone(), - assertion_message: e.to_string(), - line: *line, - column: *column, - }; - let mut results = self.test_results.borrow_mut(); - results.failures.push(failure); - results.failed_tests += 1; - } - - // Don't propagate the error - continue running other tests - break; - } - } - } - - if test_passed { - self.test_results.borrow_mut().passed_tests += 1; + if let Some(val) = self.try_evaluate_simple_expr_sync(expression, env)? { + self.perform_unary_op(operator, val, *line, *column) + .map(Some) + } else { + Ok(None) } - - // Clear current test name - *self.current_test_name.borrow_mut() = None; - - Ok((Value::Null, ControlFlow::None)) } - Statement::ExpectStatement { - subject, - assertion, + Expression::BinaryOperation { + left, + operator, + right, line, column, } => { - if !*self.test_mode.borrow() { - return Err(RuntimeError::new( - "expect statements can only be used in test mode (run with --test flag)" - .to_string(), - *line, - *column, - )); - } - - // Evaluate subject expression - let subject_value = self.evaluate_expression(subject, env.clone()).await?; - - // Check assertion - let (passed, expected_value) = self - .check_assertion(&subject_value, assertion, env.clone()) - .await?; - - if !passed { - // Record failure with proper test name tracking - let message = self.create_assertion_message_with_values( - assertion, - &subject_value, - expected_value.as_ref(), - ); - let context = self.current_describe_stack.borrow().clone(); - let test_name = self - .current_test_name - .borrow() - .clone() - .unwrap_or_else(|| "unknown test".to_string()); - - let failure = TestFailure { - describe_context: context, - test_name, - assertion_message: message.clone(), - line: *line, - column: *column, - }; + // Evaluate left side + let left_val = match self.try_evaluate_simple_expr_sync(left, env)? { + Some(v) => v, + None => return Ok(None), + }; - self.test_results.borrow_mut().failures.push(failure); - self.test_results.borrow_mut().failed_tests += 1; + // Evaluate right side (WFL evaluates both eagerly currently) + let right_val = match self.try_evaluate_simple_expr_sync(right, env)? { + Some(v) => v, + None => return Ok(None), + }; - return Err(RuntimeError::new( - format!("Assertion failed: {message}"), - *line, - *column, - )); - } + // Perform binary op + self.perform_binary_op(operator, left_val, right_val, *line, *column) + .map(Some) + } + Expression::Concatenation { + left, + right, + line: _line, + column: _column, + } => { + // Evaluate left side + let left_val = match self.try_evaluate_simple_expr_sync(left, env)? { + Some(v) => v, + None => return Ok(None), + }; - Ok((Value::Null, ControlFlow::None)) - } - }; + // Evaluate right side + let right_val = match self.try_evaluate_simple_expr_sync(right, env)? { + Some(v) => v, + None => return Ok(None), + }; - if self.step_mode { - self.dump_state(stmt, line, column, &env_before); - if !self.prompt_continue() { - std::process::exit(0); + // Concatenate + Ok(Some(self.perform_concatenation(left_val, right_val))) } + _ => Ok(None), } - - result } - /// Resolves a server expression to the key of a running WebSocket server. - async fn resolve_ws_server_key( + /// Synchronously looks up a variable and handles auto-call for native functions. + /// + /// Returns `Ok(Some(value))` for regular values or zero-arg native function auto-calls, + /// `Ok(None)` when a zero-arg user-defined function requires async execution, or + /// `Err(...)` for runtime errors (e.g., undefined variable). + fn try_evaluate_variable_sync( &self, - server: &Expression, - env: Rc>, + name: &str, + env: &Rc>, line: usize, column: usize, - ) -> Result { - let value = self.evaluate_expression(server, env).await?; - match &value { - Value::Text(t) => { - let key = t.to_string(); - if self.web_socket_servers.borrow().contains_key(&key) { - Ok(key) - } else { - Err(RuntimeError::new( - format!("'{key}' is not a running websocket server"), - line, - column, - )) - } + ) -> Result, RuntimeError> { + // Handle special count variable inside count loops + if name == "count" && *self.in_count_loop.borrow() { + if let Some(count_value) = *self.current_count.borrow() { + return Ok(Some(Value::Number(count_value))); } - _ => Err(RuntimeError::new( - "Expected a websocket server (the name bound by 'listen for websockets')" + return Err(RuntimeError::new( + "Internal error: count variable accessed in count loop but no current count set" .to_string(), line, column, - )), + )); } - } - /// Coerces a WFL value into the text payload of a websocket frame. - fn ws_message_text(value: &Value, line: usize, column: usize) -> Result { - match value { - Value::Text(t) => Ok(t.to_string()), - Value::Number(n) => Ok(format!("{n}")), - Value::Bool(b) => Ok(if *b { "yes" } else { "no" }.to_string()), - Value::Null | Value::Nothing => Err(RuntimeError::new( - "Cannot send an empty websocket message".to_string(), + // Try normal variable lookup first + if let Some(value) = env.borrow().get(name) { + self.handle_variable_auto_call(value, line, column) + } else if name == "count" { + Err(RuntimeError::new( + "Variable 'count' can only be used inside count loops. Use 'count from X to Y:' to create a count loop.".to_string(), line, column, - )), - other => Err(RuntimeError::new( - format!( - "Cannot send a {} as a websocket message; expected text", - other.type_name() - ), + )) + } else { + Err(RuntimeError::new( + format!("Undefined variable '{name}'"), line, column, - )), + )) } } - /// The byte length a `send`/`broadcast` value would serialize to, computed - /// from the *borrowed* value — for `Value::Text` (`Arc`) this is - /// `t.len()` with no allocation. Lets the queued-byte permit be reserved - /// (and an oversized message rejected) *before* the payload is cloned into a - /// `String`, so an oversized runtime value is never fully duplicated first. - fn ws_message_byte_len( - value: &Value, + /// Run a database query/execute and return the result value. Shared by + /// `DatabaseQueryStatement` and the expression form (`return query ...`). + #[allow(clippy::too_many_arguments)] + async fn evaluate_database_query( + &self, + db: &Expression, + sql: &Expression, + parameters: Option<&Expression>, + kind: crate::parser::ast::DatabaseQueryKind, line: usize, column: usize, - ) -> Result { - match value { - Value::Text(t) => Ok(t.len()), - // Small, bounded scalars — measuring == materializing cost. - Value::Number(n) => Ok(format!("{n}").len()), - Value::Bool(b) => Ok(if *b { 3 } else { 2 }), - // Reuse `ws_message_text`'s errors for the unsupported cases. - _ => Self::ws_message_text(value, line, column).map(|s| s.len()), - } - } - - /// Extracts a connection id from a `send ... to ` target. Accepts the - /// connection object bound by a connect/message handler (reads its `id`). - fn ws_connection_id(value: &Value, line: usize, column: usize) -> Result { - if let Value::Object(map) = value - && let Some(Value::Text(id)) = map.borrow().get("id") - { - return Ok(id.to_string()); - } - Err(RuntimeError::new( - "Expected a websocket connection (the value bound by 'on websocket connect/message')" - .to_string(), - line, - column, - )) - } - - /// Drains and dispatches queued websocket events for up to `budget`, running - /// the matching handler block for each. With no websocket servers active it - /// is a plain sleep, preserving `wait for ` semantics. - async fn pump_websocket_events(&self, budget: Duration) -> Result<(), RuntimeError> { - let deadline = tokio::time::Instant::now() + budget; - loop { - let now = tokio::time::Instant::now(); - if now >= deadline { - break; - } - let remaining = deadline - now; - - // Snapshot the receivers with a short borrow; dispatch below must not - // hold a borrow of web_socket_servers across handler execution. - let receivers: Vec<(String, Arc>>)> = - self.web_socket_servers - .borrow() - .iter() - .map(|(key, srv)| (key.clone(), Arc::clone(&srv.event_receiver))) - .collect(); - - if receivers.is_empty() { - tokio::time::sleep(remaining).await; - break; + env: Rc>, + ) -> Result { + let db_value = self.evaluate_expression(db, Rc::clone(&env)).await?; + let handle = match &db_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected a database handle, got {db_value:?}"), + line, + column, + )); } + }; - let mut recv_futs = Vec::with_capacity(receivers.len()); - for (key, rx) in receivers { - recv_futs.push(Box::pin(async move { - let mut guard = rx.lock().await; - let event = guard.recv().await; - (key, event) - })); + let sql_value = self.evaluate_expression(sql, Rc::clone(&env)).await?; + let sql_str = match &sql_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected text for SQL statement, got {sql_value:?}"), + line, + column, + )); } + }; - let sleep_fut = tokio::time::sleep(remaining); - tokio::pin!(sleep_fut); - - tokio::select! { - _ = &mut sleep_fut => break, - ((key, event), _idx, _rest) = futures_util::future::select_all(recv_futs) => { - if let Some(event) = event { - self.dispatch_ws_event(&key, event).await?; + let params = match parameters { + Some(params_expr) => { + let params_value = self + .evaluate_expression(params_expr, Rc::clone(&env)) + .await?; + match ¶ms_value { + Value::List(list) => { + let mut sql_params = Vec::new(); + for value in list.borrow().iter() { + sql_params.push( + database::value_to_sql_param(value) + .map_err(|e| RuntimeError::new(e, line, column))?, + ); + } + sql_params + } + _ => { + return Err(RuntimeError::new( + format!("Expected a list of query parameters, got {params_value:?}"), + line, + column, + )); } - // A `None` means that server's channel closed; the next loop - // iteration rebuilds the receiver set. } } - } - Ok(()) - } - - /// Runs the registered handler block for one websocket event. Handler errors - /// are reported but do not tear down the server, matching event-driven norms. - async fn dispatch_ws_event( - &self, - server_key: &str, - event: WflWsEvent, - ) -> Result<(), RuntimeError> { - // Clone the selected handler out before executing: the body may itself - // register handlers or send frames, which re-borrow web_socket_servers. - let handler = { - let servers = self.web_socket_servers.borrow(); - let Some(server) = servers.get(server_key) else { - return Ok(()); - }; - let handlers = server.handlers.borrow(); - let selected = match event.kind { - WsEventKind::Connect => handlers.connect.as_ref(), - WsEventKind::Message => handlers.message.as_ref(), - WsEventKind::Disconnect => handlers.disconnect.as_ref(), - }; - selected.map(|h| (h.binding.clone(), h.body.clone(), Rc::clone(&h.env))) - }; - - let Some((binding, body, handler_env)) = handler else { - return Ok(()); + None => Vec::new(), }; - let event_obj = build_ws_event_object(&event); - let child_env = Environment::new_child_env(&handler_env); - // `define_direct` shadows in the fresh handler scope without consulting - // parents, so a same-named outer variable (`store conn as ...` before an - // `on websocket connect ... as conn`) does not abort the handler. - if let Err(msg) = child_env.borrow_mut().define_direct(&binding, event_obj) { - return Err(RuntimeError::new(msg, 0, 0)); - } + let pool = self + .io_client + .get_database(&handle) + .await + .map_err(|e| RuntimeError::new(e, line, column))?; - if let Err(err) = self.execute_block(&body, child_env).await { - eprintln!( - "WebSocket {} handler error: {}", - event.kind.as_str(), - err.message - ); + match kind { + crate::parser::ast::DatabaseQueryKind::Query => { + database::run_query(&pool, &sql_str, ¶ms).await + } + crate::parser::ast::DatabaseQueryKind::Execute => { + database::run_execute(&pool, &sql_str, ¶ms).await + } } - Ok(()) + .map_err(|e| RuntimeError::new(e, line, column)) } - async fn execute_block( + async fn evaluate_expression( &self, - statements: &[Statement], + expr: &Expression, env: Rc>, - ) -> Result<(Value, ControlFlow), RuntimeError> { - Box::pin(self._execute_block(statements, env)).await + ) -> Result { + #[cfg(debug_assertions)] + exec_trace!("Evaluating expression: {}", expr_type(expr)); + + // OPTIMIZATION: Handle simple expressions synchronously to avoid Box::pin allocation + // This recursively handles literals, variables, and simple math operations. + // It significantly improves performance for tight loops with arithmetic. + if let Some(value) = self.try_evaluate_simple_expr_sync(expr, &env)? { + return Ok(value); + } + + Box::pin(self._evaluate_expression(expr, env)).await } - async fn _execute_block( + async fn _evaluate_expression( &self, - statements: &[Statement], + expr: &Expression, env: Rc>, - ) -> Result<(Value, ControlFlow), RuntimeError> { + ) -> Result { self.assert_invariants(); - // Each block carries its own overload-duplicate set for temporal - // enforcement (restored on drop, including `?` early returns), and - // arms same-scope members its definitions will merge with. - let (_overload_dups, _armed_members) = self.enter_block_overloads(statements, &env); - let mut last_value = Value::Null; + self.check_time()?; - #[cfg(debug_assertions)] - exec_trace!("Executing block of {} statements", statements.len()); + let result = match expr { + // Container-related expressions + &Expression::StaticMemberAccess { + ref container, + ref member, + line, + column, + } => { + // Look up the container definition + let container_def = match env.borrow().get(container) { + Some(Value::ContainerDefinition(def)) => def.clone(), + _ => { + return Err(RuntimeError::new( + format!("Container '{container}' not found"), + line, + column, + )); + } + }; - #[cfg(debug_assertions)] - let _guard = IndentGuard::new(); + // Look up the static member + if let Some(value) = container_def.static_properties.get(member) { + Ok(value.clone()) + } else if let Some(method) = container_def.static_methods.get(member) { + // Create a function value from the method + let function = FunctionValue { + name: Some(method.name.clone()), + params: method.params.clone(), + param_types: vec![None; method.params.len()], + body: method.body.clone(), + env: method.env.clone(), + line: method.line, + column: method.column, + enforce_param_types: std::cell::Cell::new(false), + }; - let mut control_flow = ControlFlow::None; + Ok(Value::Function(Rc::new(function))) + } else { + Err(RuntimeError::new( + format!("Static member '{member}' not found in container '{container}'"), + line, + column, + )) + } + } - for statement in statements { - let result = self.execute_statement(statement, Rc::clone(&env)).await?; - last_value = result.0; - control_flow = result.1; + &Expression::MethodCall { + ref object, + ref method, + ref arguments, + line, + column, + } => { + // Evaluate the object + let object_val = self.evaluate_expression(object, Rc::clone(&env)).await?; - if !matches!(control_flow, ControlFlow::None) { - #[cfg(debug_assertions)] - exec_trace!( - "Block execution interrupted by control flow: {:?}", - control_flow - ); - break; - } - } + // Clone the object value to avoid borrow issues + let object_val_clone = object_val.clone(); - self.assert_invariants(); - Ok((last_value, control_flow)) - } + // Check if the object is a container instance + if let Value::ContainerInstance(instance_rc) = &object_val_clone { + // Clone instance_rc for later property write-back + let instance_rc_for_writeback = instance_rc.clone(); - /// Attempts to recycle a loop iteration environment to avoid heap allocation. - /// - /// Recycling is only safe when all three conditions are met: - /// 1. We are the sole owner (`strong_count == 1`) — no other code holds a reference. - /// 2. No weak references exist (`weak_count == 0`) — closures capture environments via - /// `Weak` refs, so any weak ref means a closure may still need the environment's state. - /// 3. The recycled environment's parent matches the expected `parent` — prevents scoping - /// bugs where an environment from a different scope is reused with the wrong parent chain. - /// - /// If any condition fails, a fresh child environment is allocated instead. - fn get_recycled_env( - &self, - reusable_env: Option>>, - parent: &Rc>, - ) -> Rc> { - if let Some(env) = reusable_env - && Rc::strong_count(&env) == 1 - && Rc::weak_count(&env) == 0 - { - // Validate parent matches to prevent scoping bugs - let parent_matches = env - .borrow() - .parent - .as_ref() - .is_some_and(|p| p.ptr_eq(&Rc::downgrade(parent))); - if parent_matches { - env.borrow_mut().clear(); - return env; - } - } - Environment::new_child_env(parent) - } + let (container_type, property_names) = { + let instance = instance_rc.borrow(); + let container_type = instance.container_type.clone(); + let prop_names: Vec = instance.properties.keys().cloned().collect(); + (container_type, prop_names) + }; - // Helper to evaluate literals directly without Box::pin allocation - fn evaluate_literal_direct( - &self, - literal: &Literal, - env: &Rc>, - line: usize, - column: usize, - ) -> Result, RuntimeError> { - match literal { - Literal::String(s) => Ok(Some(Value::Text(s.clone()))), - Literal::Integer(i) => Ok(Some(Value::Number(*i as f64))), - Literal::Float(f) => Ok(Some(Value::Number(*f))), - Literal::Boolean(b) => Ok(Some(Value::Bool(*b))), - Literal::Nothing => Ok(Some(Value::Null)), - // Pattern literals might error, so we can handle them here - Literal::Pattern(ir_string) => self - .compile_pattern_literal(ir_string, env, line, column) - .map(Some), - Literal::List(elements) => { - // First, pre-scan all elements to detect if any require async evaluation - // This prevents double execution of side effects - for element in elements { - if self.requires_async_evaluation(element, env) { - // At least one element requires async, abort sync optimization for the whole list - return Ok(None); + // Look up the container definition + let container_def = match env.borrow().get(&container_type) { + Some(Value::ContainerDefinition(def)) => def.clone(), + _ => { + return Err(RuntimeError::new( + format!("Container '{container_type}' not found"), + line, + column, + )); + } + }; + + // Look up the method (with inheritance support) + let mut found_method = container_def.methods.get(method).cloned(); + let mut current_container_name = container_type.clone(); + + // If method not found, check parent containers + while found_method.is_none() { + if let Some(Value::ContainerDefinition(def)) = + env.borrow().get(¤t_container_name) + { + if let Some(parent_name) = &def.extends { + current_container_name = parent_name.clone(); + if let Some(Value::ContainerDefinition(parent_def)) = + env.borrow().get(parent_name) + { + found_method = parent_def.methods.get(method).cloned(); + } else { + break; + } + } else { + break; + } + } else { + break; + } } - } - // All elements can be evaluated synchronously, proceed safely - let mut list_values = Vec::with_capacity(elements.len()); - for element in elements { - // Since we've already verified all elements are sync-compatible, - // this should never return None, but handle it gracefully just in case - if let Some(value) = self.try_evaluate_simple_expr_sync(element, env)? { - list_values.push(value); - } else { - // This shouldn't happen after our pre-scan, but fall back to async - return Ok(None); - } - } - Ok(Some(Value::List(Rc::new(RefCell::new(list_values))))) - } - } - } + if let Some(method_val) = found_method { + // Create a function value from the method + let function = FunctionValue { + name: Some(method_val.name.clone()), + params: method_val.params.clone(), + param_types: vec![None; method_val.params.len()], + body: method_val.body.clone(), + env: method_val.env.clone(), + line: method_val.line, + column: method_val.column, + enforce_param_types: std::cell::Cell::new(false), + }; + + // Create a new environment for the method execution + let method_env = Environment::new_child_env(&env); + + // Add 'this' to the environment + let _ = method_env.borrow_mut().define("this", object_val.clone()); + + // Add container properties and events as accessible variables + { + let instance = instance_rc.borrow(); + + // Add properties + for (prop_name, prop_value) in &instance.properties { + let _ = method_env + .borrow_mut() + .define(prop_name, prop_value.clone()); + } + + // Add events from the container definition + if let Some(Value::ContainerDefinition(container_def_rc)) = + env.borrow().get(&instance.container_type) + { + let container_def = container_def_rc.clone(); + for (event_name, event_value) in &container_def.events { + let _ = method_env.borrow_mut().define( + event_name, + Value::ContainerEvent(Rc::new(event_value.clone())), + ); + } + } + } // Drop instance borrow here + + // Evaluate the arguments + let mut arg_values = Vec::with_capacity(arguments.len()); + for arg in arguments { + let arg_val = self + .evaluate_expression(&arg.value, Rc::clone(&env)) + .await?; + arg_values.push(arg_val); + } - /// Compiles a pattern literal string into a Value::Pattern - fn compile_pattern_literal( - &self, - ir_string: &str, - env: &Rc>, - line: usize, - column: usize, - ) -> Result { - let pattern_expr = crate::parser::ast::PatternExpression::Literal(ir_string.to_string()); - let compiled_pattern = { - let env_borrow = env.borrow(); - CompiledPattern::compile_with_env(&pattern_expr, &env_borrow) - }; - match compiled_pattern { - Ok(compiled) => Ok(Value::Pattern(Rc::new(compiled))), - Err(e) => Err(RuntimeError::new( - format!("Failed to compile pattern literal: {}", e), - line, - column, - )), - } - } + // Create a modified function with the method environment + let method_function = FunctionValue { + name: function.name.clone(), + params: function.params.clone(), + param_types: function.param_types.clone(), + body: function.body.clone(), + env: Rc::downgrade(&method_env), + line: function.line, + column: function.column, + enforce_param_types: function.enforce_param_types.clone(), + }; - /// Probes whether an expression requires async evaluation without executing it. - /// - /// Returns `true` if the expression must be evaluated asynchronously (e.g., it contains - /// a zero-argument user-defined function that would trigger auto-call), or `false` if - /// it can be safely evaluated on the synchronous fast-path. Used to pre-scan list - /// elements so we can avoid double-execution of side effects. - fn requires_async_evaluation(&self, expr: &Expression, env: &Rc>) -> bool { - match expr { - Expression::Literal(literal, _line, _column) => { - match literal { - Literal::Pattern(_) => false, // Patterns don't require async - Literal::List(elements) => { - // Recursively check all elements - elements - .iter() - .any(|element| self.requires_async_evaluation(element, env)) - } - _ => false, // Other literals are synchronous - } - } - Expression::Variable(name, _line, _column) => { - // Check if variable exists and if it would require async auto-call - if let Ok(env_borrowed) = env.try_borrow() { - if let Some(value) = env_borrowed.get(name) { - match &value { - Value::Function(func) => func.params.is_empty(), // Zero-arg user functions auto-call (async) - Value::Overloaded(overloaded) => overloaded - .overloads - .iter() - .any(|func| func.params.is_empty()), - Value::NativeFunction(_, _) => false, // Native functions evaluate sync - _ => false, + // Call the function with the method environment + let result = self + .call_function(&method_function, arg_values, line, column) + .await?; + + // WRITE BACK MODIFIED PROPERTIES TO CONTAINER + // This fixes the property mutation issue where properties modified + // in container actions weren't persisting + for prop_name in property_names { + if let Some(updated_value) = method_env.borrow().get(&prop_name) { + instance_rc_for_writeback + .borrow_mut() + .properties + .insert(prop_name, updated_value); + } } + + Ok(result) } else { - false // Variable doesn't exist, will be sync error + Err(RuntimeError::new( + format!("Method '{method}' not found in container '{container_type}'"), + line, + column, + )) } } else { - true // Can't borrow environment, conservatively assume async + Err(RuntimeError::new( + format!("Cannot call method '{method}' on non-container value"), + line, + column, + )) } } - Expression::UnaryOperation { expression, .. } => { - self.requires_async_evaluation(expression, env) - } - Expression::BinaryOperation { left, right, .. } => { - self.requires_async_evaluation(left, env) - || self.requires_async_evaluation(right, env) - } - Expression::Concatenation { left, right, .. } => { - self.requires_async_evaluation(left, env) - || self.requires_async_evaluation(right, env) + &Expression::AwaitExpression { + ref expression, + line: _line, + column: _column, + } => { + let value = self + .evaluate_expression(expression, Rc::clone(&env)) + .await?; + Ok(value) } - _ => true, // All other expressions require async (function calls, etc.) - } - } - - /// Handles the WFL auto-call convention for variables that resolve to functions. - /// - /// When a variable holds a zero-argument native function, it is invoked immediately - /// and the result is returned. For zero-argument user-defined functions (which require - /// async execution), returns `Ok(None)` to signal fallback to the async path. - /// Non-function values and functions with parameters are returned as-is. - fn handle_variable_auto_call( - &self, - value: Value, - line: usize, - column: usize, - ) -> Result, RuntimeError> { - match &value { - Value::NativeFunction(func_name, native_fn) => { - if get_function_arity(func_name) == 0 { - // Native functions are synchronous - native_fn(vec![]) - .map(Some) - .map_err(|e| RuntimeError::new(format!("{}", e), line, column)) - } else { - Ok(Some(value)) + Expression::Literal(literal, _line, _column) => match literal { + Literal::String(s) => Ok(Value::Text(s.clone())), + Literal::Integer(i) => Ok(Value::Number(*i as f64)), + Literal::Float(f) => Ok(Value::Number(*f)), + Literal::Boolean(b) => Ok(Value::Bool(*b)), + Literal::Nothing => Ok(Value::Null), + Literal::Pattern(ir_string) => { + self.compile_pattern_literal(ir_string, &env, *_line, *_column) } - } - Value::Function(func) => { - if func.params.is_empty() { - // User functions are async -> return None to signal fallback to async - Ok(None) - } else { - Ok(Some(value)) + Literal::List(elements) => { + let mut list_values = Vec::new(); + for element in elements { + // Use Box::pin to handle recursion in async fn + let future = Box::pin(self._evaluate_expression(element, Rc::clone(&env))); + let value = future.await?; + list_values.push(value); + } + Ok(Value::List(Rc::new(RefCell::new(list_values)))) } - } - Value::Overloaded(overloaded) => { - if overloaded - .overloads - .iter() - .any(|func| func.params.is_empty()) - { - // The zero-arg overload auto-calls (async path) - Ok(None) + }, + + Expression::Variable(name, line, column) => { + // Handle special count variable inside count loops + if name == "count" && *self.in_count_loop.borrow() { + if let Some(count_value) = *self.current_count.borrow() { + return Ok(Value::Number(count_value)); + } + // If we're in a count loop but don't have a current count, this is an error + return Err(RuntimeError::new( + "Internal error: count variable accessed in count loop but no current count set".to_string(), + *line, + *column, + )); + } + + // Try normal variable lookup first (allows user-defined 'count' variables outside loops) + // Extract lookup result so the Ref is dropped before call_function + let lookup = env.borrow().get(name); + if let Some(value) = lookup { + // Check if this is a zero-argument native function that should be auto-called + match &value { + Value::NativeFunction(func_name, native_fn) => { + if get_function_arity(func_name) == 0 { + // Auto-call zero-argument functions when referenced as variables + native_fn(vec![]).map_err(|e| { + RuntimeError::new( + format!("Error in native function '{}': {}", func_name, e), + *line, + *column, + ) + }) + } else { + // Return function object for functions with arguments + Ok(value) + } + } + Value::Function(func) => { + if func.params.is_empty() { + // Auto-call zero-argument user-defined functions + self.call_function(func, vec![], *line, *column).await + } else { + // Return function object for functions with arguments + Ok(value) + } + } + Value::Overloaded(overloaded) => { + if let Some(func) = overloaded + .overloads + .iter() + .find(|func| func.params.is_empty()) + { + // Auto-call the zero-argument overload + self.call_function(func, vec![], *line, *column).await + } else { + // Return the overload set for calls with arguments + Ok(value) + } + } + _ => Ok(value), + } + } else if name == "count" { + // If 'count' is not found and we're not in a count loop, provide helpful error + Err(RuntimeError::new( + "Variable 'count' can only be used inside count loops. Use 'count from X to Y:' to create a count loop.".to_string(), + *line, + *column, + )) } else { - Ok(Some(value)) + Err(RuntimeError::new( + format!("Undefined variable '{name}'"), + *line, + *column, + )) } } - _ => Ok(Some(value)), - } - } - /// Attempts to evaluate an expression synchronously to avoid `Box::pin` allocation overhead. - /// - /// Handles literals, variables, and simple binary/unary operations recursively. - /// Returns `Ok(Some(value))` when the expression was fully evaluated on the sync path, - /// or `Ok(None)` when async evaluation is required (e.g., function calls, complex expressions). - fn try_evaluate_simple_expr_sync( - &self, - expr: &Expression, - env: &Rc>, - ) -> Result, RuntimeError> { - match expr { - Expression::Literal(literal, line, column) => { - self.evaluate_literal_direct(literal, env, *line, *column) - } - Expression::Variable(name, line, column) => { - self.try_evaluate_variable_sync(name, env, *line, *column) - } - Expression::UnaryOperation { - operator, - expression, - line, - column, - } => { - if let Some(val) = self.try_evaluate_simple_expr_sync(expression, env)? { - self.perform_unary_op(operator, val, *line, *column) - .map(Some) - } else { - Ok(None) - } - } Expression::BinaryOperation { left, operator, @@ -9589,2237 +13078,3771 @@ impl Interpreter { line, column, } => { - // Evaluate left side - let left_val = match self.try_evaluate_simple_expr_sync(left, env)? { - Some(v) => v, - None => return Ok(None), - }; + // Use Box::pin to handle recursion in async fn + let left_future = Box::pin(self.evaluate_expression(left, Rc::clone(&env))); + let left_val = left_future.await?; - // Evaluate right side (WFL evaluates both eagerly currently) - let right_val = match self.try_evaluate_simple_expr_sync(right, env)? { - Some(v) => v, - None => return Ok(None), - }; + let right_val = self.evaluate_expression(right, Rc::clone(&env)).await?; - // Perform binary op self.perform_binary_op(operator, left_val, right_val, *line, *column) - .map(Some) - } - Expression::Concatenation { - left, - right, - line: _line, - column: _column, - } => { - // Evaluate left side - let left_val = match self.try_evaluate_simple_expr_sync(left, env)? { - Some(v) => v, - None => return Ok(None), - }; - - // Evaluate right side - let right_val = match self.try_evaluate_simple_expr_sync(right, env)? { - Some(v) => v, - None => return Ok(None), - }; - - // Concatenate - Ok(Some(self.perform_concatenation(left_val, right_val))) - } - _ => Ok(None), - } - } - - /// Synchronously looks up a variable and handles auto-call for native functions. - /// - /// Returns `Ok(Some(value))` for regular values or zero-arg native function auto-calls, - /// `Ok(None)` when a zero-arg user-defined function requires async execution, or - /// `Err(...)` for runtime errors (e.g., undefined variable). - fn try_evaluate_variable_sync( - &self, - name: &str, - env: &Rc>, - line: usize, - column: usize, - ) -> Result, RuntimeError> { - // Handle special count variable inside count loops - if name == "count" && *self.in_count_loop.borrow() { - if let Some(count_value) = *self.current_count.borrow() { - return Ok(Some(Value::Number(count_value))); } - return Err(RuntimeError::new( - "Internal error: count variable accessed in count loop but no current count set" - .to_string(), - line, - column, - )); - } - // Try normal variable lookup first - if let Some(value) = env.borrow().get(name) { - self.handle_variable_auto_call(value, line, column) - } else if name == "count" { - Err(RuntimeError::new( - "Variable 'count' can only be used inside count loops. Use 'count from X to Y:' to create a count loop.".to_string(), - line, - column, - )) - } else { - Err(RuntimeError::new( - format!("Undefined variable '{name}'"), + Expression::UnaryOperation { + operator, + expression, line, column, - )) - } - } - - /// Run a database query/execute and return the result value. Shared by - /// `DatabaseQueryStatement` and the expression form (`return query ...`). - #[allow(clippy::too_many_arguments)] - async fn evaluate_database_query( - &self, - db: &Expression, - sql: &Expression, - parameters: Option<&Expression>, - kind: crate::parser::ast::DatabaseQueryKind, - line: usize, - column: usize, - env: Rc>, - ) -> Result { - let db_value = self.evaluate_expression(db, Rc::clone(&env)).await?; - let handle = match &db_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected a database handle, got {db_value:?}"), - line, - column, - )); - } - }; - - let sql_value = self.evaluate_expression(sql, Rc::clone(&env)).await?; - let sql_str = match &sql_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected text for SQL statement, got {sql_value:?}"), - line, - column, - )); - } - }; - - let params = match parameters { - Some(params_expr) => { - let params_value = self - .evaluate_expression(params_expr, Rc::clone(&env)) + } => { + let value = self + .evaluate_expression(expression, Rc::clone(&env)) .await?; - match ¶ms_value { - Value::List(list) => { - let mut sql_params = Vec::new(); - for value in list.borrow().iter() { - sql_params.push( - database::value_to_sql_param(value) - .map_err(|e| RuntimeError::new(e, line, column))?, - ); - } - sql_params - } - _ => { - return Err(RuntimeError::new( - format!("Expected a list of query parameters, got {params_value:?}"), - line, - column, - )); - } - } - } - None => Vec::new(), - }; - let pool = self - .io_client - .get_database(&handle) - .await - .map_err(|e| RuntimeError::new(e, line, column))?; - - match kind { - crate::parser::ast::DatabaseQueryKind::Query => { - database::run_query(&pool, &sql_str, ¶ms).await - } - crate::parser::ast::DatabaseQueryKind::Execute => { - database::run_execute(&pool, &sql_str, ¶ms).await + self.perform_unary_op(operator, value, *line, *column) } - } - .map_err(|e| RuntimeError::new(e, line, column)) - } - async fn evaluate_expression( - &self, - expr: &Expression, - env: Rc>, - ) -> Result { - #[cfg(debug_assertions)] - exec_trace!("Evaluating expression: {}", expr_type(expr)); - - // OPTIMIZATION: Handle simple expressions synchronously to avoid Box::pin allocation - // This recursively handles literals, variables, and simple math operations. - // It significantly improves performance for tight loops with arithmetic. - if let Some(value) = self.try_evaluate_simple_expr_sync(expr, &env)? { - return Ok(value); - } - - Box::pin(self._evaluate_expression(expr, env)).await - } - - async fn _evaluate_expression( - &self, - expr: &Expression, - env: Rc>, - ) -> Result { - self.assert_invariants(); - self.check_time()?; - - let result = match expr { - // Container-related expressions - &Expression::StaticMemberAccess { - ref container, - ref member, + Expression::FunctionCall { + function, + arguments, line, column, } => { - // Look up the container definition - let container_def = match env.borrow().get(container) { - Some(Value::ContainerDefinition(def)) => def.clone(), - _ => { - return Err(RuntimeError::new( - format!("Container '{container}' not found"), - line, - column, - )); + let mut arg_values = Vec::new(); + for arg in arguments { + arg_values.push( + self.evaluate_expression(&arg.value, Rc::clone(&env)) + .await?, + ); + } + + // Natural-language member access: `property of object`. WFL parses + // `body of msg` as a call `body(msg)`; when the callee is a bare + // name that is not a real function and the single argument is an + // object carrying that key, resolve it as a property read. This is + // how websocket handlers read `body of msg` / `id of conn` and how + // HTTP handlers read `method of request` / `body of request`. + if let Expression::Variable(name, _, _) = function.as_ref() + && arg_values.len() == 1 + && let Value::Object(obj) = &arg_values[0] + { + let field = obj.borrow().get(name.as_str()).cloned(); + if let Some(value) = field { + let callee_is_function = matches!( + env.borrow().get(name), + Some(Value::Function(_)) + | Some(Value::Overloaded(_)) + | Some(Value::NativeFunction(_, _)) + ) || crate::builtins::is_builtin_function(name); + if !callee_is_function { + return Ok(value); + } + } + } + + // A bare-Variable callee that names an overload set must not + // be evaluated through the Variable arm: that would auto-call + // a zero-argument overload and then try to call its result. + // The set itself is the call target (PR #639 review). + let overloaded_callee = if let Expression::Variable(name, _, _) = function.as_ref() + { + match env.borrow().get(name) { + Some(value @ Value::Overloaded(_)) => Some(value), + _ => None, } + } else { + None + }; + let function_val = match overloaded_callee { + Some(value) => value, + None => self.evaluate_expression(function, Rc::clone(&env)).await?, }; - // Look up the static member - if let Some(value) = container_def.static_properties.get(member) { - Ok(value.clone()) - } else if let Some(method) = container_def.static_methods.get(member) { - // Create a function value from the method - let function = FunctionValue { - name: Some(method.name.clone()), - params: method.params.clone(), - param_types: vec![None; method.params.len()], - body: method.body.clone(), - env: method.env.clone(), - line: method.line, - column: method.column, - enforce_param_types: std::cell::Cell::new(false), - }; + #[cfg(debug_assertions)] + let func_name = match &function_val { + Value::Function(f) => { + f.name.clone().unwrap_or_else(|| "".to_string()) + } + _ => format!("{function_val:?}"), + }; - Ok(Value::Function(Rc::new(function))) - } else { - Err(RuntimeError::new( - format!("Static member '{member}' not found in container '{container}'"), - line, - column, - )) + #[cfg(debug_assertions)] + exec_function_call!(&func_name, &arg_values); + + let result = match function_val { + Value::Function(func) => { + self.call_function(&func, arg_values, *line, *column).await + } + Value::Overloaded(overloaded) => { + let func = Self::select_overload(&overloaded, &arg_values, *line, *column)?; + self.call_function(&func, arg_values, *line, *column).await + } + Value::NativeFunction(native_name, native_fn) => { + // CPU-heavy crypto builtins hop onto the blocking pool so + // they don't monopolize the interpreter thread (Phase 0, + // PR-0b). Everything else runs synchronously as before. + if let Some(fut) = + crate::stdlib::crypto_async::route(native_name, &arg_values) + { + fut.await.map_err(|e| { + RuntimeError::new( + format!("Error in native function: {e}"), + *line, + *column, + ) + }) + } else { + native_fn(arg_values.clone()).map_err(|e| { + RuntimeError::new( + format!("Error in native function: {e}"), + *line, + *column, + ) + }) + } + } + _ => Err(RuntimeError::new( + format!("Cannot call {}", function_val.type_name()), + *line, + *column, + )), + }; + + #[cfg(debug_assertions)] + if let Ok(ref val) = result { + exec_function_return!(&func_name, val); } + + result } - &Expression::MethodCall { - ref object, - ref method, - ref arguments, + Expression::ActionCall { + name, + arguments, line, column, } => { - // Evaluate the object - let object_val = self.evaluate_expression(object, Rc::clone(&env)).await?; + let function_val = env.borrow().get(name).ok_or_else(|| { + RuntimeError::new(format!("Undefined action '{name}'"), *line, *column) + })?; - // Clone the object value to avoid borrow issues - let object_val_clone = object_val.clone(); + match function_val { + Value::Overloaded(overloaded) => { + let mut arg_values = Vec::new(); + for arg in arguments.iter() { + arg_values.push( + self.evaluate_expression(&arg.value, Rc::clone(&env)) + .await?, + ); + } + let func = Self::select_overload(&overloaded, &arg_values, *line, *column)?; + self.call_function(&func, arg_values, *line, *column).await + } + Value::Function(func) => { + let mut arg_values = Vec::new(); + for arg in arguments.iter() { + arg_values.push( + self.evaluate_expression(&arg.value, Rc::clone(&env)) + .await?, + ); + } - // Check if the object is a container instance - if let Value::ContainerInstance(instance_rc) = &object_val_clone { - // Clone instance_rc for later property write-back - let instance_rc_for_writeback = instance_rc.clone(); + #[cfg(debug_assertions)] + let func_name = func + .name + .clone() + .unwrap_or_else(|| "".to_string()); - let (container_type, property_names) = { - let instance = instance_rc.borrow(); - let container_type = instance.container_type.clone(); - let prop_names: Vec = instance.properties.keys().cloned().collect(); - (container_type, prop_names) - }; + #[cfg(debug_assertions)] + exec_function_call!(&func_name, &arg_values); - // Look up the container definition - let container_def = match env.borrow().get(&container_type) { - Some(Value::ContainerDefinition(def)) => def.clone(), - _ => { - return Err(RuntimeError::new( - format!("Container '{container_type}' not found"), - line, - column, - )); + let result = self.call_function(&func, arg_values, *line, *column).await; + + #[cfg(debug_assertions)] + if let Ok(ref val) = result { + exec_function_return!(&func_name, val); } - }; - // Look up the method (with inheritance support) - let mut found_method = container_def.methods.get(method).cloned(); - let mut current_container_name = container_type.clone(); + result + } + Value::NativeFunction(_, native_fn) => { + let mut arg_values = Vec::new(); + for arg in arguments.iter() { + arg_values.push( + self.evaluate_expression(&arg.value, Rc::clone(&env)) + .await?, + ); + } - // If method not found, check parent containers - while found_method.is_none() { - if let Some(Value::ContainerDefinition(def)) = - env.borrow().get(¤t_container_name) - { - if let Some(parent_name) = &def.extends { - current_container_name = parent_name.clone(); - if let Some(Value::ContainerDefinition(parent_def)) = - env.borrow().get(parent_name) - { - found_method = parent_def.methods.get(method).cloned(); - } else { - break; - } - } else { - break; - } + // Preserve the native error's message and kind; only + // point the location at the call site (natives report + // their position as 0,0). CPU-heavy crypto builtins are + // routed onto the blocking pool (Phase 0, PR-0b). + if let Some(fut) = crate::stdlib::crypto_async::route(name, &arg_values) { + fut.await.map_err(|mut e| { + e.line = *line; + e.column = *column; + e + }) + } else { + native_fn(arg_values).map_err(|mut e| { + e.line = *line; + e.column = *column; + e + }) + } + } + _ => Err(RuntimeError::new( + format!("'{name}' is not callable"), + *line, + *column, + )), + } + } + + Expression::MemberAccess { + object, + property, + line, + column, + } => { + let object_val = self.evaluate_expression(object, Rc::clone(&env)).await?; + + match object_val { + Value::Object(obj_rc) => { + let obj = obj_rc.borrow(); + if let Some(value) = obj.get(property) { + Ok(value.clone()) } else { - break; + Err(RuntimeError::new( + format!("Object has no property '{property}'"), + *line, + *column, + )) } } + _ => Err(RuntimeError::new( + format!("Cannot access property of {}", object_val.type_name()), + *line, + *column, + )), + } + } - if let Some(method_val) = found_method { - // Create a function value from the method - let function = FunctionValue { - name: Some(method_val.name.clone()), - params: method_val.params.clone(), - param_types: vec![None; method_val.params.len()], - body: method_val.body.clone(), - env: method_val.env.clone(), - line: method_val.line, - column: method_val.column, - enforce_param_types: std::cell::Cell::new(false), - }; - - // Create a new environment for the method execution - let method_env = Environment::new_child_env(&env); + Expression::IndexAccess { + collection, + index, + line, + column, + } => { + let collection_val = self + .evaluate_expression(collection, Rc::clone(&env)) + .await?; + let index_val = self.evaluate_expression(index, Rc::clone(&env)).await?; - // Add 'this' to the environment - let _ = method_env.borrow_mut().define("this", object_val.clone()); + match (collection_val, index_val) { + (Value::List(list_rc), Value::Number(idx)) => { + let list = list_rc.borrow(); + let idx = idx as usize; - // Add container properties and events as accessible variables - { - let instance = instance_rc.borrow(); + if idx < list.len() { + Ok(list[idx].clone()) + } else { + Err(RuntimeError::new( + format!( + "Index {} out of bounds for list of length {}", + idx, + list.len() + ), + *line, + *column, + )) + } + } + (Value::Object(obj_rc), Value::Text(key)) => { + let obj = obj_rc.borrow(); + let key_str = key.to_string(); - // Add properties - for (prop_name, prop_value) in &instance.properties { - let _ = method_env - .borrow_mut() - .define(prop_name, prop_value.clone()); - } + if let Some(value) = obj.get(&key_str) { + Ok(value.clone()) + } else { + Err(RuntimeError::new( + format!("Object has no key '{key_str}'"), + *line, + *column, + )) + } + } + (collection, index) => Err(RuntimeError::new( + format!( + "Cannot index {} with {}", + collection.type_name(), + index.type_name() + ), + *line, + *column, + )), + } + } - // Add events from the container definition - if let Some(Value::ContainerDefinition(container_def_rc)) = - env.borrow().get(&instance.container_type) - { - let container_def = container_def_rc.clone(); - for (event_name, event_value) in &container_def.events { - let _ = method_env.borrow_mut().define( - event_name, - Value::ContainerEvent(Rc::new(event_value.clone())), - ); - } - } - } // Drop instance borrow here + Expression::Concatenation { + left, + right, + line: _line, + column: _column, + } => { + // Use Box::pin to handle recursion in async fn + let left_future = Box::pin(self.evaluate_expression(left, Rc::clone(&env))); + let left_val = left_future.await?; - // Evaluate the arguments - let mut arg_values = Vec::with_capacity(arguments.len()); - for arg in arguments { - let arg_val = self - .evaluate_expression(&arg.value, Rc::clone(&env)) - .await?; - arg_values.push(arg_val); - } + let right_val = self.evaluate_expression(right, Rc::clone(&env)).await?; - // Create a modified function with the method environment - let method_function = FunctionValue { - name: function.name.clone(), - params: function.params.clone(), - param_types: function.param_types.clone(), - body: function.body.clone(), - env: Rc::downgrade(&method_env), - line: function.line, - column: function.column, - enforce_param_types: function.enforce_param_types.clone(), - }; + Ok(self.perform_concatenation(left_val, right_val)) + } - // Call the function with the method environment - let result = self - .call_function(&method_function, arg_values, line, column) - .await?; + Expression::PatternMatch { + text, + pattern, + line: _line, + column: _column, + } => { + let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; + let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; - // WRITE BACK MODIFIED PROPERTIES TO CONTAINER - // This fixes the property mutation issue where properties modified - // in container actions weren't persisting - for prop_name in property_names { - if let Some(updated_value) = method_env.borrow().get(&prop_name) { - instance_rc_for_writeback - .borrow_mut() - .properties - .insert(prop_name, updated_value); - } - } + // Extract text string + let text_str = match &text_val { + Value::Text(s) => s.as_ref(), + _ => { + return Err(RuntimeError::new( + "Pattern match requires text as first argument".to_string(), + *_line, + *_column, + )); + } + }; - Ok(result) - } else { - Err(RuntimeError::new( - format!("Method '{method}' not found in container '{container_type}'"), - line, - column, - )) + // Extract compiled pattern + let compiled_pattern = match &pattern_val { + Value::Pattern(p) => p, + _ => { + return Err(RuntimeError::new( + "Pattern match requires pattern as second argument".to_string(), + *_line, + *_column, + )); } - } else { - Err(RuntimeError::new( - format!("Cannot call method '{method}' on non-container value"), - line, - column, - )) - } + }; + + // Perform the match under the shared budget so a pathological + // pattern is bounded by the run's step/state ceilings; a breach + // surfaces as a catchable error, not a silent non-match. + let matches = compiled_pattern + .matches_with_budget(text_str, &self.budget) + .map_err(|e| self.pattern_error(e, *_line, *_column))?; + Ok(Value::Bool(matches)) } - &Expression::AwaitExpression { - ref expression, + + Expression::PatternFind { + text, + pattern, line: _line, column: _column, } => { - let value = self - .evaluate_expression(expression, Rc::clone(&env)) - .await?; - Ok(value) - } - Expression::Literal(literal, _line, _column) => match literal { - Literal::String(s) => Ok(Value::Text(s.clone())), - Literal::Integer(i) => Ok(Value::Number(*i as f64)), - Literal::Float(f) => Ok(Value::Number(*f)), - Literal::Boolean(b) => Ok(Value::Bool(*b)), - Literal::Nothing => Ok(Value::Null), - Literal::Pattern(ir_string) => { - self.compile_pattern_literal(ir_string, &env, *_line, *_column) - } - Literal::List(elements) => { - let mut list_values = Vec::new(); - for element in elements { - // Use Box::pin to handle recursion in async fn - let future = Box::pin(self._evaluate_expression(element, Rc::clone(&env))); - let value = future.await?; - list_values.push(value); + let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; + let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; + + // Extract text string + let text_str = match &text_val { + Value::Text(s) => s.as_ref(), + _ => { + return Err(RuntimeError::new( + "Pattern find requires text as first argument".to_string(), + *_line, + *_column, + )); } - Ok(Value::List(Rc::new(RefCell::new(list_values)))) - } - }, + }; - Expression::Variable(name, line, column) => { - // Handle special count variable inside count loops - if name == "count" && *self.in_count_loop.borrow() { - if let Some(count_value) = *self.current_count.borrow() { - return Ok(Value::Number(count_value)); + // Extract compiled pattern + let compiled_pattern = match &pattern_val { + Value::Pattern(p) => p, + _ => { + return Err(RuntimeError::new( + "Pattern find requires pattern as second argument".to_string(), + *_line, + *_column, + )); } - // If we're in a count loop but don't have a current count, this is an error - return Err(RuntimeError::new( - "Internal error: count variable accessed in count loop but no current count set".to_string(), - *line, - *column, - )); - } + }; - // Try normal variable lookup first (allows user-defined 'count' variables outside loops) - // Extract lookup result so the Ref is dropped before call_function - let lookup = env.borrow().get(name); - if let Some(value) = lookup { - // Check if this is a zero-argument native function that should be auto-called - match &value { - Value::NativeFunction(func_name, native_fn) => { - if get_function_arity(func_name) == 0 { - // Auto-call zero-argument functions when referenced as variables - native_fn(vec![]).map_err(|e| { - RuntimeError::new( - format!("Error in native function '{}': {}", func_name, e), - *line, - *column, - ) - }) - } else { - // Return function object for functions with arguments - Ok(value) - } - } - Value::Function(func) => { - if func.params.is_empty() { - // Auto-call zero-argument user-defined functions - self.call_function(func, vec![], *line, *column).await - } else { - // Return function object for functions with arguments - Ok(value) - } - } - Value::Overloaded(overloaded) => { - if let Some(func) = overloaded - .overloads - .iter() - .find(|func| func.params.is_empty()) - { - // Auto-call the zero-argument overload - self.call_function(func, vec![], *line, *column).await - } else { - // Return the overload set for calls with arguments - Ok(value) + // Find the first match under the shared budget (a breach is a + // catchable error rather than a silent non-match). + let found = compiled_pattern + .find_with_budget(text_str, &self.budget) + .map_err(|e| self.pattern_error(e, *_line, *_column))?; + match found { + Some(match_result) => { + // Return an object with match information + let mut result_map = std::collections::HashMap::new(); + result_map.insert( + "matched_text".to_string(), + Value::Text(Arc::from(match_result.matched_text.as_str())), + ); + result_map.insert( + "start".to_string(), + Value::Number(match_result.start as f64), + ); + result_map + .insert("end".to_string(), Value::Number(match_result.end as f64)); + + // Add captures if any + if !match_result.captures.is_empty() { + let mut captures_map = std::collections::HashMap::new(); + for (name, value) in match_result.captures { + captures_map.insert(name, Value::Text(Arc::from(value.as_str()))); } + result_map.insert( + "captures".to_string(), + Value::Object(Rc::new(RefCell::new(captures_map))), + ); } - _ => Ok(value), + + Ok(Value::Object(Rc::new(RefCell::new(result_map)))) } - } else if name == "count" { - // If 'count' is not found and we're not in a count loop, provide helpful error - Err(RuntimeError::new( - "Variable 'count' can only be used inside count loops. Use 'count from X to Y:' to create a count loop.".to_string(), - *line, - *column, - )) - } else { - Err(RuntimeError::new( - format!("Undefined variable '{name}'"), - *line, - *column, - )) + None => Ok(Value::Null), } } - Expression::BinaryOperation { - left, - operator, - right, - line, - column, + Expression::PatternReplace { + text, + pattern, + replacement, + line: _line, + column: _column, } => { - // Use Box::pin to handle recursion in async fn - let left_future = Box::pin(self.evaluate_expression(left, Rc::clone(&env))); - let left_val = left_future.await?; - - let right_val = self.evaluate_expression(right, Rc::clone(&env)).await?; + let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; + let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; + let replacement_val = self + .evaluate_expression(replacement, Rc::clone(&env)) + .await?; - self.perform_binary_op(operator, left_val, right_val, *line, *column) + let args = vec![text_val, pattern_val, replacement_val]; // Note: text, pattern, then replacement + crate::stdlib::pattern::native_pattern_replace(args, *_line, *_column) } - Expression::UnaryOperation { - operator, - expression, - line, - column, + Expression::PatternSplit { + text, + pattern, + line: _line, + column: _column, } => { - let value = self - .evaluate_expression(expression, Rc::clone(&env)) - .await?; + let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; + let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; - self.perform_unary_op(operator, value, *line, *column) + let args = vec![text_val, pattern_val]; + crate::stdlib::pattern::native_pattern_split(args, *_line, *_column) } + Expression::StringSplit { + text, + delimiter, + line: _line, + column: _column, + } => { + let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; + let delimiter_val = self.evaluate_expression(delimiter, Rc::clone(&env)).await?; - Expression::FunctionCall { - function, - arguments, + // Validate types + if !matches!(text_val, Value::Text(_)) { + return Err(RuntimeError::new( + format!("Cannot split {} - expected text", text_val.type_name()), + *_line, + *_column, + )); + } + if !matches!(delimiter_val, Value::Text(_)) { + return Err(RuntimeError::new( + format!("Delimiter must be text - got {}", delimiter_val.type_name()), + *_line, + *_column, + )); + } + + let args = vec![text_val, delimiter_val]; + crate::stdlib::text::native_string_split(args) + } + Expression::PropertyAccess { + object, + property, line, column, } => { - let mut arg_values = Vec::new(); - for arg in arguments { - arg_values.push( - self.evaluate_expression(&arg.value, Rc::clone(&env)) - .await?, - ); - } - - // Natural-language member access: `property of object`. WFL parses - // `body of msg` as a call `body(msg)`; when the callee is a bare - // name that is not a real function and the single argument is an - // object carrying that key, resolve it as a property read. This is - // how websocket handlers read `body of msg` / `id of conn` and how - // HTTP handlers read `method of request` / `body of request`. - if let Expression::Variable(name, _, _) = function.as_ref() - && arg_values.len() == 1 - && let Value::Object(obj) = &arg_values[0] - { - let field = obj.borrow().get(name.as_str()).cloned(); - if let Some(value) = field { - let callee_is_function = matches!( - env.borrow().get(name), - Some(Value::Function(_)) - | Some(Value::Overloaded(_)) - | Some(Value::NativeFunction(_, _)) - ) || crate::builtins::is_builtin_function(name); - if !callee_is_function { - return Ok(value); + let obj_value = self.evaluate_expression(object, Rc::clone(&env)).await?; + match obj_value { + Value::ContainerInstance(instance) => { + let instance_ref = instance.borrow(); + if let Some(prop_value) = instance_ref.properties.get(property) { + Ok(prop_value.clone()) + } else { + Err(RuntimeError::new( + format!("Property '{property}' not found"), + *line, + *column, + )) + } + } + Value::Object(obj_rc) => { + let obj = obj_rc.borrow(); + if let Some(prop_value) = obj.get(property) { + Ok(prop_value.clone()) + } else { + Err(RuntimeError::new( + format!("Object has no property '{property}'"), + *line, + *column, + )) } } + _ => Err(RuntimeError::new( + format!("Cannot access property '{property}' on non-container value"), + *line, + *column, + )), } - - // A bare-Variable callee that names an overload set must not - // be evaluated through the Variable arm: that would auto-call - // a zero-argument overload and then try to call its result. - // The set itself is the call target (PR #639 review). - let overloaded_callee = if let Expression::Variable(name, _, _) = function.as_ref() - { - match env.borrow().get(name) { - Some(value @ Value::Overloaded(_)) => Some(value), - _ => None, + } + Expression::FileExists { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file path, got {path_value:?}"), + *line, + *column, + )); } - } else { - None - }; - let function_val = match overloaded_callee { - Some(value) => value, - None => self.evaluate_expression(function, Rc::clone(&env)).await?, }; - #[cfg(debug_assertions)] - let func_name = match &function_val { - Value::Function(f) => { - f.name.clone().unwrap_or_else(|| "".to_string()) + Ok(Value::Bool(tokio::fs::metadata(&*path_str).await.is_ok())) + } + Expression::DirectoryExists { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); } - _ => format!("{function_val:?}"), }; - #[cfg(debug_assertions)] - exec_function_call!(&func_name, &arg_values); - - let result = match function_val { - Value::Function(func) => { - self.call_function(&func, arg_values, *line, *column).await - } - Value::Overloaded(overloaded) => { - let func = Self::select_overload(&overloaded, &arg_values, *line, *column)?; - self.call_function(&func, arg_values, *line, *column).await + match tokio::fs::metadata(&*path_str).await { + Ok(metadata) => Ok(Value::Bool(metadata.is_dir())), + Err(_) => Ok(Value::Bool(false)), + } + } + Expression::ListFiles { path, line, column } => { + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); } - Value::NativeFunction(native_name, native_fn) => { - // CPU-heavy crypto builtins hop onto the blocking pool so - // they don't monopolize the interpreter thread (Phase 0, - // PR-0b). Everything else runs synchronously as before. - if let Some(fut) = - crate::stdlib::crypto_async::route(native_name, &arg_values) - { - fut.await.map_err(|e| { - RuntimeError::new( - format!("Error in native function: {e}"), - *line, - *column, - ) - }) - } else { - native_fn(arg_values.clone()).map_err(|e| { - RuntimeError::new( - format!("Error in native function: {e}"), - *line, - *column, - ) - }) + }; + + match tokio::fs::read_dir(&*path_str).await { + Ok(mut entries) => { + let mut files = Vec::new(); + while let Ok(Some(entry)) = entries.next_entry().await { + if let Ok(file_name) = entry.file_name().into_string() { + files.push(Value::Text(file_name.into())); + } } + Ok(Value::List(Rc::new(RefCell::new(files)))) } - _ => Err(RuntimeError::new( - format!("Cannot call {}", function_val.type_name()), + Err(e) => Err(RuntimeError::new( + format!("Failed to list files in directory: {e}"), *line, *column, )), + } + } + Expression::ReadContent { + file_handle, + line, + column, + } => { + let handle_value = self + .evaluate_expression(file_handle, Rc::clone(&env)) + .await?; + let handle_str = match &handle_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for file handle, got {handle_value:?}"), + *line, + *column, + )); + } }; - #[cfg(debug_assertions)] - if let Ok(ref val) = result { - exec_function_return!(&func_name, val); + match self.io_client.read_file(&handle_str, &self.budget).await { + Ok(content) => Ok(Value::Text(content.into())), + Err(e) => Err(self.file_read_error(e, *line, *column)), } - - result } - - Expression::ActionCall { - name, - arguments, + Expression::ReadBinaryContent { + file_handle, line, column, } => { - let function_val = env.borrow().get(name).ok_or_else(|| { - RuntimeError::new(format!("Undefined action '{name}'"), *line, *column) - })?; - - match function_val { - Value::Overloaded(overloaded) => { - let mut arg_values = Vec::new(); - for arg in arguments.iter() { - arg_values.push( - self.evaluate_expression(&arg.value, Rc::clone(&env)) - .await?, - ); - } - let func = Self::select_overload(&overloaded, &arg_values, *line, *column)?; - self.call_function(&func, arg_values, *line, *column).await + let handle_value = self + .evaluate_expression(file_handle, Rc::clone(&env)) + .await?; + let handle_str = match &handle_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file handle, got {}", + handle_value.type_name() + ), + *line, + *column, + )); } - Value::Function(func) => { - let mut arg_values = Vec::new(); - for arg in arguments.iter() { - arg_values.push( - self.evaluate_expression(&arg.value, Rc::clone(&env)) - .await?, - ); - } - - #[cfg(debug_assertions)] - let func_name = func - .name - .clone() - .unwrap_or_else(|| "".to_string()); - - #[cfg(debug_assertions)] - exec_function_call!(&func_name, &arg_values); - - let result = self.call_function(&func, arg_values, *line, *column).await; - - #[cfg(debug_assertions)] - if let Ok(ref val) = result { - exec_function_return!(&func_name, val); - } + }; - result + match self.io_client.read_binary(&handle_str, &self.budget).await { + Ok(bytes) => Ok(Value::Binary(Arc::from(bytes))), + Err(e) => Err(self.file_read_error(e, *line, *column)), + } + } + Expression::ReadBinaryN { + file_handle, + count, + line, + column, + } => { + let handle_value = self + .evaluate_expression(file_handle, Rc::clone(&env)) + .await?; + let handle_str = match &handle_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file handle, got {}", + handle_value.type_name() + ), + *line, + *column, + )); } - Value::NativeFunction(_, native_fn) => { - let mut arg_values = Vec::new(); - for arg in arguments.iter() { - arg_values.push( - self.evaluate_expression(&arg.value, Rc::clone(&env)) - .await?, - ); - } + }; - // Preserve the native error's message and kind; only - // point the location at the call site (natives report - // their position as 0,0). CPU-heavy crypto builtins are - // routed onto the blocking pool (Phase 0, PR-0b). - if let Some(fut) = crate::stdlib::crypto_async::route(name, &arg_values) { - fut.await.map_err(|mut e| { - e.line = *line; - e.column = *column; - e - }) - } else { - native_fn(arg_values).map_err(|mut e| { - e.line = *line; - e.column = *column; - e - }) + let count_value = self.evaluate_expression(count, Rc::clone(&env)).await?; + let n = match &count_value { + Value::Number(n) => { + if !n.is_finite() || n.fract() != 0.0 || *n < 0.0 || *n > usize::MAX as f64 + { + return Err(RuntimeError::new( + format!("Invalid byte count: {n} — must be a non-negative integer"), + *line, + *column, + )); } + *n as usize } - _ => Err(RuntimeError::new( - format!("'{name}' is not callable"), - *line, - *column, - )), + _ => { + return Err(RuntimeError::new( + format!( + "Expected number for byte count, got {}", + count_value.type_name() + ), + *line, + *column, + )); + } + }; + + match self + .io_client + .read_binary_n(&handle_str, n, &self.budget) + .await + { + Ok(bytes) => Ok(Value::Binary(Arc::from(bytes))), + Err(e) => Err(self.file_read_error(e, *line, *column)), } } + Expression::FileSizeOf { + file_handle, + line, + column, + } => { + let handle_value = self + .evaluate_expression(file_handle, Rc::clone(&env)) + .await?; + let handle_str = match &handle_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string for file handle, got {}", + handle_value.type_name() + ), + *line, + *column, + )); + } + }; - Expression::MemberAccess { - object, - property, + match self.io_client.file_size(&handle_str).await { + Ok(size) => Ok(Value::Number(size as f64)), + Err(e) => Err(RuntimeError::new(e, *line, *column)), + } + } + Expression::ListFilesRecursive { + path, + extensions, line, column, } => { - let object_val = self.evaluate_expression(object, Rc::clone(&env)).await?; + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); + } + }; - match object_val { - Value::Object(obj_rc) => { - let obj = obj_rc.borrow(); - if let Some(value) = obj.get(property) { - Ok(value.clone()) - } else { - Err(RuntimeError::new( - format!("Object has no property '{property}'"), - *line, - *column, - )) + // Evaluate extensions if provided + let ext_filters = if let Some(ext_exprs) = extensions { + let mut filters = Vec::new(); + for ext_expr in ext_exprs { + let ext_value = self.evaluate_expression(ext_expr, Rc::clone(&env)).await?; + match &ext_value { + Value::Text(s) => filters.push(s.to_string()), + Value::List(list) => { + // If we get a list, extract all string values from it + let list_ref = list.borrow(); + for item in list_ref.iter() { + match item { + Value::Text(s) => filters.push(s.to_string()), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string in extension list, got {item:?}" + ), + *line, + *column, + )); + } + } + } + } + _ => { + return Err(RuntimeError::new( + format!( + "Expected string or list for extension, got {ext_value:?}" + ), + *line, + *column, + )); + } } } - _ => Err(RuntimeError::new( - format!("Cannot access property of {}", object_val.type_name()), + Some(filters) + } else { + None + }; + + // Perform recursive directory listing + match self.list_files_recursive(&path_str, ext_filters).await { + Ok(files) => Ok(Value::List(Rc::new(RefCell::new(files)))), + Err(e) => Err(RuntimeError::new( + format!("Failed to list files recursively: {e}"), *line, *column, )), } } - - Expression::IndexAccess { - collection, - index, + Expression::ListFilesFiltered { + path, + extensions, line, column, } => { - let collection_val = self - .evaluate_expression(collection, Rc::clone(&env)) - .await?; - let index_val = self.evaluate_expression(index, Rc::clone(&env)).await?; - - match (collection_val, index_val) { - (Value::List(list_rc), Value::Number(idx)) => { - let list = list_rc.borrow(); - let idx = idx as usize; - - if idx < list.len() { - Ok(list[idx].clone()) - } else { - Err(RuntimeError::new( - format!( - "Index {} out of bounds for list of length {}", - idx, - list.len() - ), - *line, - *column, - )) - } + let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; + let path_str = match &path_value { + Value::Text(s) => s.clone(), + _ => { + return Err(RuntimeError::new( + format!("Expected string for directory path, got {path_value:?}"), + *line, + *column, + )); } - (Value::Object(obj_rc), Value::Text(key)) => { - let obj = obj_rc.borrow(); - let key_str = key.to_string(); + }; - if let Some(value) = obj.get(&key_str) { - Ok(value.clone()) - } else { - Err(RuntimeError::new( - format!("Object has no key '{key_str}'"), + // Evaluate extensions + let mut ext_filters = Vec::new(); + for ext_expr in extensions { + let ext_value = self.evaluate_expression(ext_expr, Rc::clone(&env)).await?; + match &ext_value { + Value::Text(s) => ext_filters.push(s.to_string()), + Value::List(list) => { + // If we get a list, extract all string values from it + let list_ref = list.borrow(); + for item in list_ref.iter() { + match item { + Value::Text(s) => ext_filters.push(s.to_string()), + _ => { + return Err(RuntimeError::new( + format!( + "Expected string in extension list, got {item:?}" + ), + *line, + *column, + )); + } + } + } + } + _ => { + return Err(RuntimeError::new( + format!("Expected string or list for extension, got {ext_value:?}"), *line, *column, - )) + )); } } - (collection, index) => Err(RuntimeError::new( - format!( - "Cannot index {} with {}", - collection.type_name(), - index.type_name() - ), + } + + // List files with filtering + match self.list_files_filtered(&path_str, ext_filters).await { + Ok(files) => Ok(Value::List(Rc::new(RefCell::new(files)))), + Err(e) => Err(RuntimeError::new( + format!("Failed to list files with filter: {e}"), *line, *column, )), } } - - Expression::Concatenation { - left, - right, - line: _line, - column: _column, - } => { - // Use Box::pin to handle recursion in async fn - let left_future = Box::pin(self.evaluate_expression(left, Rc::clone(&env))); - let left_val = left_future.await?; - - let right_val = self.evaluate_expression(right, Rc::clone(&env)).await?; - - Ok(self.perform_concatenation(left_val, right_val)) - } - - Expression::PatternMatch { - text, - pattern, - line: _line, - column: _column, + Expression::HeaderAccess { + header_name, + request, + line, + column, } => { - let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; - let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; - - // Extract text string - let text_str = match &text_val { - Value::Text(s) => s.as_ref(), - _ => { - return Err(RuntimeError::new( - "Pattern match requires text as first argument".to_string(), - *_line, - *_column, - )); + // Resolve headers from the request object first so + // `header "X" of req` works inside actions that receive + // `req` as a parameter (issue #597). Fall back to the + // loop-scoped `headers` binding for backward compatibility + // when the request expression is not a request object. + let headers_val = { + let request_val = self.evaluate_expression(request, Rc::clone(&env)).await?; + match &request_val { + Value::Object(obj) => { + let map = obj.borrow(); + if let Some(headers) = map.get("headers") { + headers.clone() + } else { + // Not a request object — try env fallback + match env.borrow().get("headers") { + Some(val) => val.clone(), + None => { + return Err(RuntimeError::new( + "Cannot access headers: no request in scope. Use 'wait for request comes in' first, or pass a request object to the action.".to_string(), + *line, + *column, + )); + } + } + } + } + _ => match env.borrow().get("headers") { + Some(val) => val.clone(), + None => { + return Err(RuntimeError::new( + "Cannot access headers: no request in scope. Use 'wait for request comes in' first, or pass a request object to the action.".to_string(), + *line, + *column, + )); + } + }, } }; - // Extract compiled pattern - let compiled_pattern = match &pattern_val { - Value::Pattern(p) => p, + // Get the specific header from the headers object. + match &headers_val { + Value::Object(headers_map) => { + let map = headers_map.borrow(); + match lookup_header_case_insensitive(&map, header_name) { + Some(header_value) => Ok(header_value), + // Value::Null is the runtime value of WFL's + // `nothing` literal + None => Ok(Value::Null), + } + } _ => { return Err(RuntimeError::new( - "Pattern match requires pattern as second argument".to_string(), - *_line, - *_column, + format!( + "Expected headers to be an object, got {}", + headers_val.type_name() + ), + *line, + *column, )); } - }; - - // Perform the match under the shared budget so a pathological - // pattern is bounded by the run's step/state ceilings; a breach - // surfaces as a catchable error, not a silent non-match. - let matches = compiled_pattern - .matches_with_budget(text_str, &self.budget) - .map_err(|e| self.pattern_error(e, *_line, *_column))?; - Ok(Value::Bool(matches)) + } } - - Expression::PatternFind { - text, - pattern, - line: _line, - column: _column, + Expression::CurrentTimeMilliseconds { line: _, column: _ } => { + use std::time::{SystemTime, UNIX_EPOCH}; + let now = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| { + RuntimeError::new(format!("Failed to get current time: {}", e), 0, 0) + })?; + Ok(Value::Number(now.as_millis() as f64)) + } + Expression::CurrentTimeFormatted { + format, + line: _, + column: _, } => { - let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; - let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; + use chrono::{DateTime, Local}; + let now: DateTime = Local::now(); - // Extract text string - let text_str = match &text_val { - Value::Text(s) => s.as_ref(), - _ => { - return Err(RuntimeError::new( - "Pattern find requires text as first argument".to_string(), - *_line, - *_column, - )); - } - }; + // Convert WFL format to chrono format + // For now, support basic formats + let chrono_format = format + .replace("yyyy", "%Y") + .replace("MM", "%m") + .replace("dd", "%d") + .replace("HH", "%H") + .replace("mm", "%M") + .replace("ss", "%S"); - // Extract compiled pattern - let compiled_pattern = match &pattern_val { - Value::Pattern(p) => p, + let formatted = now.format(&chrono_format).to_string(); + Ok(Value::Text(Arc::from(formatted))) + } + Expression::ProcessRunning { + process_id, + line, + column, + } => { + // Evaluate process ID expression + let proc_val = self + .evaluate_expression(process_id, Rc::clone(&env)) + .await?; + let proc_id = match &proc_val { + Value::Text(text) => text.as_ref(), _ => { return Err(RuntimeError::new( - "Pattern find requires pattern as second argument".to_string(), - *_line, - *_column, + format!("Process ID must be text, got {}", proc_val.type_name()), + *line, + *column, )); } }; - // Find the first match under the shared budget (a breach is a - // catchable error rather than a silent non-match). - let found = compiled_pattern - .find_with_budget(text_str, &self.budget) - .map_err(|e| self.pattern_error(e, *_line, *_column))?; - match found { - Some(match_result) => { - // Return an object with match information - let mut result_map = std::collections::HashMap::new(); - result_map.insert( - "matched_text".to_string(), - Value::Text(Arc::from(match_result.matched_text.as_str())), - ); - result_map.insert( - "start".to_string(), - Value::Number(match_result.start as f64), - ); - result_map - .insert("end".to_string(), Value::Number(match_result.end as f64)); + // Check if process is running + let is_running = self.io_client.is_process_running(proc_id).await; + Ok(Value::Bool(is_running)) + } + Expression::DatabaseQuery { + db, + sql, + parameters, + kind, + line, + column, + } => { + self.evaluate_database_query( + db, + sql, + parameters.as_deref(), + *kind, + *line, + *column, + Rc::clone(&env), + ) + .await + } + }; + self.assert_invariants(); + result + } - // Add captures if any - if !match_result.captures.is_empty() { - let mut captures_map = std::collections::HashMap::new(); - for (name, value) in match_result.captures { - captures_map.insert(name, Value::Text(Arc::from(value.as_str()))); - } - result_map.insert( - "captures".to_string(), - Value::Object(Rc::new(RefCell::new(captures_map))), - ); + /// Picks the overload whose parameter count and declared parameter types + /// match the actual argument values: filter by count, drop candidates + /// whose concrete annotations reject an argument, then prefer the + /// candidate with the most concretely-matched parameters (ties resolve to + /// definition order). + fn select_overload( + overloaded: &OverloadedFunction, + args: &[Value], + line: usize, + column: usize, + ) -> Result, RuntimeError> { + let arity_matches: Vec<&Rc> = overloaded + .overloads + .iter() + .filter(|func| func.params.len() == args.len()) + .collect(); + + if arity_matches.is_empty() { + let mut arities: Vec = overloaded + .overloads + .iter() + .map(|func| func.params.len()) + .collect(); + arities.sort_unstable(); + arities.dedup(); + let arities_str = arities + .iter() + .map(usize::to_string) + .collect::>() + .join(" or "); + return Err(RuntimeError::new( + format!( + "No version of '{}' takes {} argument(s). It is defined with {} parameter(s).", + overloaded.name, + args.len(), + arities_str + ), + line, + column, + )); + } + + let mut best: Option<(&Rc, usize)> = None; + for func in &arity_matches { + let mut concrete_matches = 0usize; + let mut accepts = true; + for (param_type, arg) in func.param_types.iter().zip(args) { + if let Some(expected) = param_type { + // `any`/`Unknown` annotations accept everything and earn + // no specificity credit, matching untyped parameters. + if matches!(expected, Type::Any | Type::Unknown) { + continue; + } + if Self::value_matches_type(arg, expected) { + // A `nothing` argument is accepted by every parameter + // type but earns specificity credit only for an + // explicit `as nothing` parameter (its exact match) — + // otherwise versions accepting `nothing` stay tied + // and definition order decides, as documented. + if !matches!(arg, Value::Null | Value::Nothing) + || matches!(expected, Type::Nothing) + { + concrete_matches += 1; } + } else { + accepts = false; + break; + } + } + } + if accepts && best.is_none_or(|(_, count)| concrete_matches > count) { + best = Some((func, concrete_matches)); + } + } + + match best { + Some((func, _)) => Ok(Rc::clone(func)), + None => { + let provided: Vec<&str> = args.iter().map(|arg| arg.type_name()).collect(); + let mut message = format!( + "No version of '{}' matches this call.\nYou provided ({}), but '{}' accepts:", + overloaded.name, + provided.join(", "), + overloaded.name + ); + for func in &arity_matches { + message.push_str(&format!("\n {}", Self::format_overload_signature(func))); + } + Err(RuntimeError::new(message, line, column)) + } + } + } - Ok(Value::Object(Rc::new(RefCell::new(result_map)))) + /// Whether a runtime value satisfies a declared parameter type. Untyped + /// and unknown annotations accept everything; `Custom` types match a + /// container instance of that type or of a descendant (via the parent + /// instance chain). + fn value_matches_type(value: &Value, expected: &Type) -> bool { + // `nothing` is compatible with every parameter type, mirroring the + // static checkers' `(_, Type::Nothing) => true` rule; ties among + // overloads resolve by specificity then definition order. + if matches!(value, Value::Null | Value::Nothing) { + return true; + } + match expected { + Type::Number => matches!(value, Value::Number(_)), + Type::Text => matches!(value, Value::Text(_)), + Type::Boolean => matches!(value, Value::Bool(_)), + Type::Nothing => matches!(value, Value::Null | Value::Nothing), + Type::Pattern => matches!(value, Value::Pattern(_)), + Type::List(_) => matches!(value, Value::List(_)), + Type::Map(_, _) => matches!(value, Value::Object(_)), + Type::Custom(name) => { + if name.eq_ignore_ascii_case("any") { + return true; + } + if let Value::ContainerInstance(instance) = value { + let mut current = Some(Rc::clone(instance)); + while let Some(inst) = current { + let inst_ref = inst.borrow(); + if inst_ref.container_type == *name { + return true; + } + current = inst_ref.parent.clone(); } - None => Ok(Value::Null), + false + } else { + false } } + _ => true, + } + } - Expression::PatternReplace { - text, - pattern, - replacement, - line: _line, - column: _column, - } => { - let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; - let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; - let replacement_val = self - .evaluate_expression(replacement, Rc::clone(&env)) - .await?; + /// Renders an overload's signature in WFL surface syntax for error + /// messages, e.g. `depict with value as number`. + fn format_overload_signature(func: &FunctionValue) -> String { + let name = func.name.as_deref().unwrap_or("anonymous"); + if func.params.is_empty() { + return format!("{name} (no parameters)"); + } + let params = func + .params + .iter() + .zip(&func.param_types) + .map(|(param, param_type)| match param_type { + Some(t) => format!("{param} as {}", crate::analyzer::format_param_type(t)), + None => param.clone(), + }) + .collect::>() + .join(" and "); + format!("{name} with {params}") + } - let args = vec![text_val, pattern_val, replacement_val]; // Note: text, pattern, then replacement - crate::stdlib::pattern::native_pattern_replace(args, *_line, *_column) - } + async fn call_function( + &self, + func: &FunctionValue, + args: Vec, + line: usize, + column: usize, + ) -> Result { + #[cfg(feature = "dhat-ad-hoc")] + dhat::ad_hoc_event(1); - Expression::PatternSplit { - text, - pattern, - line: _line, - column: _column, - } => { - let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; - let pattern_val = self.evaluate_expression(pattern, Rc::clone(&env)).await?; + #[cfg(debug_assertions)] + let func_name = func + .name + .clone() + .unwrap_or_else(|| "".to_string()); - let args = vec![text_val, pattern_val]; - crate::stdlib::pattern::native_pattern_split(args, *_line, *_column) - } - Expression::StringSplit { - text, - delimiter, - line: _line, - column: _column, - } => { - let text_val = self.evaluate_expression(text, Rc::clone(&env)).await?; - let delimiter_val = self.evaluate_expression(delimiter, Rc::clone(&env)).await?; + if args.len() != func.params.len() { + return Err(RuntimeError::new( + format!( + "Expected {} arguments but got {}", + func.params.len(), + args.len() + ), + line, + column, + )); + } - // Validate types - if !matches!(text_val, Value::Text(_)) { - return Err(RuntimeError::new( - format!("Cannot split {} - expected text", text_val.type_name()), - *_line, - *_column, - )); - } - if !matches!(delimiter_val, Value::Text(_)) { + // Declared parameter types are runtime-enforced only for actions + // participating in overload dispatch: a lone member of a + // not-yet-complete overload set rejects non-matching arguments instead + // of silently running the wrong body — calls dispatch on "the + // overloads defined so far". Plain single actions keep their + // historical dynamic behavior (annotations are static hints, not + // runtime guards), preserving backward compatibility. `nothing` and + // untyped/`any` parameters accept every value. + if func.enforce_param_types.get() { + for ((param_name, param_type), arg) in + func.params.iter().zip(&func.param_types).zip(args.iter()) + { + if let Some(expected) = param_type + && !matches!(expected, Type::Any | Type::Unknown) + && !Self::value_matches_type(arg, expected) + { + let action_name = func.name.as_deref().unwrap_or("anonymous"); return Err(RuntimeError::new( - format!("Delimiter must be text - got {}", delimiter_val.type_name()), - *_line, - *_column, + format!( + "Argument '{param_name}' of '{action_name}' expects {}, but got {}", + crate::analyzer::format_param_type(expected), + arg.type_name() + ), + line, + column, )); } + } + } - let args = vec![text_val, delimiter_val]; - crate::stdlib::text::native_string_split(args) + let func_env = match func.env.upgrade() { + Some(env) => { + exec_trace!("call_function - Successfully upgraded function environment"); + env } - Expression::PropertyAccess { - object, - property, - line, - column, - } => { - let obj_value = self.evaluate_expression(object, Rc::clone(&env)).await?; - match obj_value { - Value::ContainerInstance(instance) => { - let instance_ref = instance.borrow(); - if let Some(prop_value) = instance_ref.properties.get(property) { - Ok(prop_value.clone()) - } else { - Err(RuntimeError::new( - format!("Property '{property}' not found"), - *line, - *column, - )) - } - } - Value::Object(obj_rc) => { - let obj = obj_rc.borrow(); - if let Some(prop_value) = obj.get(property) { - Ok(prop_value.clone()) - } else { - Err(RuntimeError::new( - format!("Object has no property '{property}'"), - *line, - *column, - )) - } + None => { + exec_trace!("call_function - Failed to upgrade function environment"); + return Err(RuntimeError::new( + "Environment no longer exists".to_string(), + line, + column, + )); + } + }; + + let call_env = Environment::new_child_env(&func_env); + exec_trace!("call_function - Created child environment for function call"); + + for (_i, (param, arg)) in func.params.iter().zip(args.clone()).enumerate() { + exec_trace!( + "call_function - Binding parameter {} '{}' to argument {:?}", + _i, + param, + arg + ); + + #[cfg(debug_assertions)] + exec_var_declare!(param, &arg); + // Bind parameters directly in the call scope so they shadow any + // same-named global/outer binding. `define` (which rejects names + // present in a parent scope) would otherwise leave the parameter + // unbound and let the body resolve to the global instead (#582). + let _ = call_env.borrow_mut().define_direct(param, arg.clone()); + } + + // Enforce the shared recursion ceiling before descending another level, + // turning runaway recursion into a clean error instead of a native stack + // overflow. The dedicated `call_depth` counter (not `call_stack.len()`) + // is the enforcement source of truth: it is decremented by the RAII + // guard below as the call unwinds — including when a `try`/`when` + // catches a `ResourceLimit` — so catch-and-recurse cannot under-count + // and pile onto still-live native frames. + if let Err(exceeded) = self.budget.check_call_depth(self.call_depth.get()) { + return Err(self.budget_error(exceeded, line, column)); + } + let _depth_guard = CallDepthGuard::enter(&self.call_depth); + + let frame = CallFrame::new( + func.name + .clone() + .unwrap_or_else(|| "".to_string()), + line, + column, + ); + self.call_stack.borrow_mut().push(frame); + exec_trace!("call_function - Pushed frame to call stack"); + + #[cfg(debug_assertions)] + exec_block_enter!(format!("function {}", func_name)); + + #[cfg(debug_assertions)] + let _guard = IndentGuard::new(); + + exec_trace!("call_function - Executing function body"); + let result = self.execute_block(&func.body, call_env.clone()).await; + exec_trace!("call_function - Function execution result: {:?}", result); + + #[cfg(debug_assertions)] + exec_block_exit!(format!("function {}", func_name)); + + match result { + Ok((value, control_flow)) => { + self.call_stack.borrow_mut().pop(); + + let return_value = match control_flow { + ControlFlow::Return(val) => { + exec_trace!( + "call_function - Function explicitly returned with value: {:?}", + val + ); + val } - _ => Err(RuntimeError::new( - format!("Cannot access property '{property}' on non-container value"), - *line, - *column, - )), - } - } - Expression::FileExists { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), _ => { - return Err(RuntimeError::new( - format!("Expected string for file path, got {path_value:?}"), - *line, - *column, - )); + exec_trace!("call_function - Function completed with value: {:?}", value); + value } }; - Ok(Value::Bool(tokio::fs::metadata(&*path_str).await.is_ok())) + exec_trace!( + "call_function - Function returned successfully with value: {:?}", + return_value + ); + Ok(return_value) } - Expression::DirectoryExists { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); - } - }; - - match tokio::fs::metadata(&*path_str).await { - Ok(metadata) => Ok(Value::Bool(metadata.is_dir())), - Err(_) => Ok(Value::Bool(false)), + Err(err) => { + exec_trace!( + "call_function - Function execution failed with error: {:?}", + err + ); + if let Some(last_frame) = self.call_stack.borrow_mut().last_mut() { + last_frame.capture_locals(&call_env); } - } - Expression::ListFiles { path, line, column } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); - } - }; - match tokio::fs::read_dir(&*path_str).await { - Ok(mut entries) => { - let mut files = Vec::new(); - while let Ok(Some(entry)) = entries.next_entry().await { - if let Ok(file_name) = entry.file_name().into_string() { - files.push(Value::Text(file_name.into())); - } - } - Ok(Value::List(Rc::new(RefCell::new(files)))) - } - Err(e) => Err(RuntimeError::new( - format!("Failed to list files in directory: {e}"), - *line, - *column, - )), - } + let error_with_stack = err.clone(); + + self.call_stack.borrow_mut().pop(); + + Err(error_with_stack) } - Expression::ReadContent { - file_handle, + } + } + + fn evaluate_numeric_op( + &self, + left: Value, + right: Value, + line: usize, + column: usize, + op: Op, + err_gen: ErrGen, + ) -> Result + where + Op: Fn(f64, f64) -> Result, + ErrGen: Fn(&str, &str) -> String, + { + match (left, right) { + (Value::Number(a), Value::Number(b)) => match op(a, b) { + Ok(res) => Ok(Value::Number(res)), + Err(msg) => Err(RuntimeError::new(msg, line, column)), + }, + (a, b) => Err(RuntimeError::new( + err_gen(a.type_name(), b.type_name()), line, column, - } => { - let handle_value = self - .evaluate_expression(file_handle, Rc::clone(&env)) - .await?; - let handle_str = match &handle_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for file handle, got {handle_value:?}"), - *line, - *column, - )); - } - }; + )), + } + } - match self.io_client.read_file(&handle_str, &self.budget).await { - Ok(content) => Ok(Value::Text(content.into())), - Err(e) => Err(self.file_read_error(e, *line, *column)), - } - } - Expression::ReadBinaryContent { - file_handle, + fn evaluate_comparison_op( + &self, + left: Value, + right: Value, + line: usize, + column: usize, + op_symbol: &str, + comp: Comp, + ) -> Result + where + Comp: Fn(std::cmp::Ordering) -> bool, + { + match (left, right) { + (Value::Number(a), Value::Number(b)) => match a.partial_cmp(&b) { + Some(ord) => Ok(Value::Bool(comp(ord))), + None => Ok(Value::Bool(false)), + }, + (Value::Text(a), Value::Text(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), + (Value::Date(a), Value::Date(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), + (Value::Time(a), Value::Time(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), + (Value::DateTime(a), Value::DateTime(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), + (a, b) => Err(RuntimeError::new( + format!( + "Cannot compare {} and {} with {}", + a.type_name(), + b.type_name(), + op_symbol + ), line, column, - } => { - let handle_value = self - .evaluate_expression(file_handle, Rc::clone(&env)) - .await?; - let handle_str = match &handle_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string for file handle, got {}", - handle_value.type_name() - ), - *line, - *column, - )); - } - }; + )), + } + } - match self.io_client.read_binary(&handle_str, &self.budget).await { - Ok(bytes) => Ok(Value::Binary(Arc::from(bytes))), - Err(e) => Err(self.file_read_error(e, *line, *column)), - } - } - Expression::ReadBinaryN { - file_handle, - count, + fn perform_binary_op( + &self, + operator: &Operator, + left_val: Value, + right_val: Value, + line: usize, + column: usize, + ) -> Result { + match operator { + Operator::Plus => self.add(left_val, right_val, line, column), + Operator::Minus => self.evaluate_numeric_op( + left_val, + right_val, line, column, - } => { - let handle_value = self - .evaluate_expression(file_handle, Rc::clone(&env)) - .await?; - let handle_str = match &handle_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string for file handle, got {}", - handle_value.type_name() - ), - *line, - *column, - )); + |a, b| Ok(a - b), + |a_type, b_type| format!("Cannot subtract {b_type} from {a_type}"), + ), + Operator::Multiply => self.evaluate_numeric_op( + left_val, + right_val, + line, + column, + |a, b| Ok(a * b), + |a_type, b_type| format!("Cannot multiply {a_type} and {b_type}"), + ), + Operator::Divide => self.evaluate_numeric_op( + left_val, + right_val, + line, + column, + |a, b| { + #[cfg(feature = "dhat-ad-hoc")] + dhat::ad_hoc_event(1); // Track division operations for memory profiling + + if b == 0.0 { + Err("Division by zero".to_string()) + } else { + let res = a / b; + if !res.is_finite() { + Err(format!("Division resulted in invalid number: {res}")) + } else { + Ok(res) + } } - }; + }, + |a_type, b_type| format!("Cannot divide {a_type} by {b_type}"), + ), + Operator::Modulo => self.evaluate_numeric_op( + left_val, + right_val, + line, + column, + |a, b| { + #[cfg(feature = "dhat-ad-hoc")] + dhat::ad_hoc_event(1); // Track modulo operations for memory profiling - let count_value = self.evaluate_expression(count, Rc::clone(&env)).await?; - let n = match &count_value { - Value::Number(n) => { - if !n.is_finite() || n.fract() != 0.0 || *n < 0.0 || *n > usize::MAX as f64 - { - return Err(RuntimeError::new( - format!("Invalid byte count: {n} — must be a non-negative integer"), - *line, - *column, - )); + if b == 0.0 { + Err("Modulo by zero".to_string()) + } else { + let res = a % b; + if !res.is_finite() { + Err(format!("Modulo resulted in invalid number: {res}")) + } else { + Ok(res) } - *n as usize - } - _ => { - return Err(RuntimeError::new( - format!( - "Expected number for byte count, got {}", - count_value.type_name() - ), - *line, - *column, - )); } - }; + }, + |a_type, b_type| format!("Cannot compute modulo of {a_type} by {b_type}"), + ), + Operator::Equals => Ok(Value::Bool(self.is_equal(&left_val, &right_val))), + Operator::NotEquals => Ok(Value::Bool(!self.is_equal(&left_val, &right_val))), + Operator::GreaterThan => { + self.evaluate_comparison_op(left_val, right_val, line, column, ">", |ord| { + matches!(ord, std::cmp::Ordering::Greater) + }) + } + Operator::LessThan => { + self.evaluate_comparison_op(left_val, right_val, line, column, "<", |ord| { + matches!(ord, std::cmp::Ordering::Less) + }) + } + Operator::GreaterThanOrEqual => { + self.evaluate_comparison_op(left_val, right_val, line, column, ">=", |ord| { + matches!(ord, std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) + }) + } + Operator::LessThanOrEqual => { + self.evaluate_comparison_op(left_val, right_val, line, column, "<=", |ord| { + matches!(ord, std::cmp::Ordering::Less | std::cmp::Ordering::Equal) + }) + } + Operator::And => Ok(Value::Bool(left_val.is_truthy() && right_val.is_truthy())), + Operator::Or => Ok(Value::Bool(left_val.is_truthy() || right_val.is_truthy())), + Operator::Contains => self.contains(left_val, right_val, line, column), + } + } - match self - .io_client - .read_binary_n(&handle_str, n, &self.budget) - .await - { - Ok(bytes) => Ok(Value::Binary(Arc::from(bytes))), - Err(e) => Err(self.file_read_error(e, *line, *column)), - } + fn perform_unary_op( + &self, + operator: &UnaryOperator, + value: Value, + line: usize, + column: usize, + ) -> Result { + match operator { + UnaryOperator::Not => Ok(Value::Bool(!value.is_truthy())), + UnaryOperator::Minus => match value { + Value::Number(n) => Ok(Value::Number(-n)), + _ => Err(RuntimeError::new( + format!("Cannot negate {}", value.type_name()), + line, + column, + )), + }, + } + } + + fn perform_concatenation(&self, left_val: Value, right_val: Value) -> Value { + // Optimization: Fast path for string concatenation to avoid format! machinery overhead + if let (Value::Text(left), Value::Text(right)) = (&left_val, &right_val) { + let mut s = String::with_capacity(left.len() + right.len()); + s.push_str(left); + s.push_str(right); + return Value::Text(Arc::from(s)); + } + + let result = format!("{left_val}{right_val}"); + Value::Text(Arc::from(result.as_str())) + } + + fn add( + &self, + left: Value, + right: Value, + line: usize, + column: usize, + ) -> Result { + match (left, right) { + (Value::Number(a), Value::Number(b)) => Ok(Value::Number(a + b)), + (Value::Text(a), Value::Text(b)) => { + // Optimization: Fast path for string concatenation + let mut s = String::with_capacity(a.len() + b.len()); + s.push_str(&a); + s.push_str(&b); + Ok(Value::Text(Arc::from(s))) } - Expression::FileSizeOf { - file_handle, + (Value::Text(a), b) => { + let result = format!("{a}{b}"); + Ok(Value::Text(Arc::from(result.as_str()))) + } + (a, Value::Text(b)) => { + let result = format!("{a}{b}"); + Ok(Value::Text(Arc::from(result.as_str()))) + } + (a, b) => Err(RuntimeError::new( + format!("Cannot add {} and {}", a.type_name(), b.type_name()), line, column, - } => { - let handle_value = self - .evaluate_expression(file_handle, Rc::clone(&env)) - .await?; - let handle_str = match &handle_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string for file handle, got {}", - handle_value.type_name() - ), - *line, - *column, - )); - } - }; + )), + } + } - match self.io_client.file_size(&handle_str).await { - Ok(size) => Ok(Value::Number(size as f64)), - Err(e) => Err(RuntimeError::new(e, *line, *column)), - } + fn is_equal(&self, left: &Value, right: &Value) -> bool { + left == right + } + + // Helper method to create container instance with inheritance + #[allow(clippy::only_used_in_recursion)] + fn create_container_instance_with_inheritance( + &self, + container_type: &str, + env: &Rc>, + line: usize, + column: usize, + ) -> Result { + // Look up the container definition + let container_def = match env.borrow().get(container_type) { + Some(Value::ContainerDefinition(def)) => def.clone(), + _ => { + return Err(RuntimeError::new( + format!("Container '{container_type}' not found"), + line, + column, + )); } - Expression::ListFilesRecursive { - path, - extensions, - line, - column, - } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); - } - }; + }; - // Evaluate extensions if provided - let ext_filters = if let Some(ext_exprs) = extensions { - let mut filters = Vec::new(); - for ext_expr in ext_exprs { - let ext_value = self.evaluate_expression(ext_expr, Rc::clone(&env)).await?; - match &ext_value { - Value::Text(s) => filters.push(s.to_string()), - Value::List(list) => { - // If we get a list, extract all string values from it - let list_ref = list.borrow(); - for item in list_ref.iter() { - match item { - Value::Text(s) => filters.push(s.to_string()), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string in extension list, got {item:?}" - ), - *line, - *column, - )); - } - } - } - } - _ => { - return Err(RuntimeError::new( - format!( - "Expected string or list for extension, got {ext_value:?}" - ), - *line, - *column, - )); - } - } - } - Some(filters) - } else { - None - }; + // Create parent instance if container extends another + let parent_instance = if let Some(parent_type) = &container_def.extends { + // Recursively create parent instance + let parent = + self.create_container_instance_with_inheritance(parent_type, env, line, column)?; + Some(Rc::new(RefCell::new(parent))) + } else { + None + }; - // Perform recursive directory listing - match self.list_files_recursive(&path_str, ext_filters).await { - Ok(files) => Ok(Value::List(Rc::new(RefCell::new(files)))), - Err(e) => Err(RuntimeError::new( - format!("Failed to list files recursively: {e}"), - *line, - *column, - )), - } + // Create instance with inherited properties + let mut instance_properties = HashMap::new(); + + // Copy properties from parent if exists + if let Some(ref parent) = parent_instance { + for (key, value) in &parent.borrow().properties { + instance_properties.insert(key.clone(), value.clone()); } - Expression::ListFilesFiltered { - path, - extensions, - line, - column, - } => { - let path_value = self.evaluate_expression(path, Rc::clone(&env)).await?; - let path_str = match &path_value { - Value::Text(s) => s.clone(), - _ => { - return Err(RuntimeError::new( - format!("Expected string for directory path, got {path_value:?}"), - *line, - *column, - )); - } - }; + } - // Evaluate extensions - let mut ext_filters = Vec::new(); - for ext_expr in extensions { - let ext_value = self.evaluate_expression(ext_expr, Rc::clone(&env)).await?; - match &ext_value { - Value::Text(s) => ext_filters.push(s.to_string()), - Value::List(list) => { - // If we get a list, extract all string values from it - let list_ref = list.borrow(); - for item in list_ref.iter() { - match item { - Value::Text(s) => ext_filters.push(s.to_string()), - _ => { - return Err(RuntimeError::new( - format!( - "Expected string in extension list, got {item:?}" - ), - *line, - *column, - )); - } - } - } - } - _ => { - return Err(RuntimeError::new( - format!("Expected string or list for extension, got {ext_value:?}"), - *line, - *column, - )); - } - } - } + // Initialize properties with default values from container definition + for (prop_name, prop_def) in &container_def.properties { + if let Some(default_value) = &prop_def.default_value { + instance_properties.insert(prop_name.clone(), default_value.clone()); + } + } - // List files with filtering - match self.list_files_filtered(&path_str, ext_filters).await { - Ok(files) => Ok(Value::List(Rc::new(RefCell::new(files)))), - Err(e) => Err(RuntimeError::new( - format!("Failed to list files with filter: {e}"), - *line, - *column, - )), + Ok(ContainerInstanceValue { + container_type: container_type.to_string(), + properties: instance_properties, + parent: parent_instance, + line, + column, + }) + } + + fn contains( + &self, + left: Value, + right: Value, + line: usize, + column: usize, + ) -> Result { + match (left, right) { + (Value::List(list_rc), item) => { + let list = list_rc.borrow(); + for value in list.iter() { + if self.is_equal(value, &item) { + return Ok(Value::Bool(true)); + } } + Ok(Value::Bool(false)) } - Expression::HeaderAccess { - header_name, - request, + (Value::Object(obj_rc), Value::Text(key)) => { + let obj = obj_rc.borrow(); + Ok(Value::Bool(obj.contains_key(&key.to_string()))) + } + (Value::Text(text), Value::Text(substring)) => { + Ok(Value::Bool(text.contains(&*substring))) + } + (a, b) => Err(RuntimeError::new( + format!( + "Cannot check if {} contains {}", + a.type_name(), + b.type_name() + ), line, column, - } => { - // Resolve headers from the request object first so - // `header "X" of req` works inside actions that receive - // `req` as a parameter (issue #597). Fall back to the - // loop-scoped `headers` binding for backward compatibility - // when the request expression is not a request object. - let headers_val = { - let request_val = self.evaluate_expression(request, Rc::clone(&env)).await?; - match &request_val { - Value::Object(obj) => { - let map = obj.borrow(); - if let Some(headers) = map.get("headers") { - headers.clone() - } else { - // Not a request object — try env fallback - match env.borrow().get("headers") { - Some(val) => val.clone(), - None => { - return Err(RuntimeError::new( - "Cannot access headers: no request in scope. Use 'wait for request comes in' first, or pass a request object to the action.".to_string(), - *line, - *column, - )); - } - } - } - } - _ => match env.borrow().get("headers") { - Some(val) => val.clone(), - None => { - return Err(RuntimeError::new( - "Cannot access headers: no request in scope. Use 'wait for request comes in' first, or pass a request object to the action.".to_string(), - *line, - *column, - )); - } - }, - } - }; + )), + } + } - // Get the specific header from the headers object. - match &headers_val { - Value::Object(headers_map) => { - let map = headers_map.borrow(); - match lookup_header_case_insensitive(&map, header_name) { - Some(header_value) => Ok(header_value), - // Value::Null is the runtime value of WFL's - // `nothing` literal - None => Ok(Value::Null), + async fn list_files_recursive( + &self, + path: &str, + extensions: Option>, + ) -> Result, std::io::Error> { + use tokio::fs; + + let mut files = Vec::new(); + let mut dirs_to_process = vec![path.to_string()]; + + while let Some(current_dir) = dirs_to_process.pop() { + let mut entries = fs::read_dir(¤t_dir).await?; + + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + let path_str = path.to_string_lossy().to_string(); + + if path.is_dir() { + dirs_to_process.push(path_str); + } else if path.is_file() { + // Check extension filter if provided + if let Some(ref exts) = extensions { + let file_ext = path + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| format!(".{ext}")); + + if let Some(ext) = file_ext + && exts.iter().any(|e| e == &ext) + { + files.push(Value::Text(path_str.into())); } - } - _ => { - return Err(RuntimeError::new( - format!( - "Expected headers to be an object, got {}", - headers_val.type_name() - ), - *line, - *column, - )); + } else { + files.push(Value::Text(path_str.into())); } } } - Expression::CurrentTimeMilliseconds { line: _, column: _ } => { - use std::time::{SystemTime, UNIX_EPOCH}; - let now = SystemTime::now().duration_since(UNIX_EPOCH).map_err(|e| { - RuntimeError::new(format!("Failed to get current time: {}", e), 0, 0) - })?; - Ok(Value::Number(now.as_millis() as f64)) - } - Expression::CurrentTimeFormatted { - format, - line: _, - column: _, - } => { - use chrono::{DateTime, Local}; - let now: DateTime = Local::now(); + } - // Convert WFL format to chrono format - // For now, support basic formats - let chrono_format = format - .replace("yyyy", "%Y") - .replace("MM", "%m") - .replace("dd", "%d") - .replace("HH", "%H") - .replace("mm", "%M") - .replace("ss", "%S"); + Ok(files) + } + + async fn list_files_filtered( + &self, + path: &str, + extensions: Vec, + ) -> Result, std::io::Error> { + use tokio::fs; + + let mut files = Vec::new(); + let mut entries = fs::read_dir(path).await?; + + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); - let formatted = now.format(&chrono_format).to_string(); - Ok(Value::Text(Arc::from(formatted))) + if path.is_file() { + let path_str = path.to_string_lossy().to_string(); + + // Check extension filter + let file_ext = path + .extension() + .and_then(|ext| ext.to_str()) + .map(|ext| format!(".{ext}")); + + if let Some(ext) = file_ext + && extensions.iter().any(|e| e == &ext) + { + files.push(Value::Text(path_str.into())); + } } - Expression::ProcessRunning { - process_id, - line, - column, - } => { - // Evaluate process ID expression - let proc_val = self - .evaluate_expression(process_id, Rc::clone(&env)) - .await?; - let proc_id = match &proc_val { - Value::Text(text) => text.as_ref(), - _ => { - return Err(RuntimeError::new( - format!("Process ID must be text, got {}", proc_val.type_name()), - *line, - *column, - )); - } - }; + } - // Check if process is running - let is_running = self.io_client.is_process_running(proc_id).await; - Ok(Value::Bool(is_running)) + Ok(files) + } +} + +#[cfg(test)] +mod concurrent_handler_classification_tests { + use super::*; + + fn error(kind: ErrorKind, message: &str) -> RuntimeError { + RuntimeError::with_kind(message.to_string(), 1, 1, kind) + } + + #[test] + fn more_than_the_breaker_threshold_of_request_wait_timeouts_stays_request_local() { + let timeout = error( + ErrorKind::Timeout, + &format!("{REQUEST_WAIT_TIMEOUT_PREFIX} (1 ms)"), + ); + for observed in 0..=MAX_CONSECUTIVE_HANDLER_FAILURES { + assert_eq!( + classify_concurrent_handler_error(&timeout, false), + ConcurrentHandlerDisposition::RequestLocal, + "finite request-wait timeout #{observed} must not feed the structural breaker" + ); + } + } + + #[test] + fn only_the_expected_timeout_origin_is_exempt_before_request_acceptance() { + let structural_timeout = error( + ErrorKind::Timeout, + "unrelated pre-request operation timed out", + ); + assert_eq!( + classify_concurrent_handler_error(&structural_timeout, false), + ConcurrentHandlerDisposition::Structural + ); + assert_eq!( + classify_concurrent_handler_error(&error(ErrorKind::General, "request failed"), true), + ConcurrentHandlerDisposition::RequestLocal, + "any failure after request acceptance is isolated to that request" + ); + assert_eq!( + classify_concurrent_handler_error( + &error(ErrorKind::Cancelled, "client disconnected"), + false, + ), + ConcurrentHandlerDisposition::RequestLocal + ); + } + + #[tokio::test] + async fn missing_pending_entry_is_cancelled_only_while_the_handler_owns_it() { + let interpreter = Interpreter::new(); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-1".to_string()); + + let disconnected = interpreter + .ensure_pending_response_owned("request-1", 1, 1) + .await + .expect_err("owned-but-pruned pending entry must be cancellation"); + assert_eq!(disconnected.kind, ErrorKind::Cancelled); + + interpreter.open_pending_requests.borrow_mut().clear(); + let duplicate = interpreter + .ensure_pending_response_owned("request-1", 1, 1) + .await + .expect_err("non-owned missing entry must remain a duplicate-response error"); + assert_eq!(duplicate.kind, ErrorKind::General); + assert!( + duplicate.message.contains("already been sent"), + "duplicate response diagnostic changed unexpectedly: {duplicate}" + ); + } +} + +#[cfg(test)] +mod response_expression_disconnect_tests { + use super::*; + use crate::lexer::lex_wfl_with_positions; + use crate::parser::Parser; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + struct StalledUpstream { + port: u16, + head_sent: oneshot::Receiver<()>, + peer_closed: oneshot::Receiver<()>, + release: Option>, + } + + async fn spawn_stalled_upstream() -> StalledUpstream { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind stalled upstream"); + let port = listener.local_addr().expect("upstream address").port(); + let (head_sent_tx, head_sent) = oneshot::channel(); + let (peer_closed_tx, peer_closed) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept upstream request"); + let mut request = Vec::new(); + let mut chunk = [0u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket + .read(&mut chunk) + .await + .expect("read upstream request"); + assert!(read > 0, "client closed before sending request head"); + request.extend_from_slice(&chunk[..read]); } - Expression::DatabaseQuery { - db, - sql, - parameters, - kind, - line, - column, - } => { - self.evaluate_database_query( - db, - sql, - parameters.as_deref(), - *kind, - *line, - *column, - Rc::clone(&env), + socket + .write_all( + b"HTTP/1.1 200 OK\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: close\r\n\r\n", ) .await + .expect("send stalled response head"); + let _ = head_sent_tx.send(()); + + let mut probe = [0u8; 1]; + tokio::select! { + result = socket.read(&mut probe) => { + assert!( + matches!(result, Ok(0) | Err(_)), + "client unexpectedly sent bytes while response body was stalled: {result:?}" + ); + let _ = peer_closed_tx.send(()); + } + _ = release_rx => { + let _ = socket.shutdown().await; + } } + }); + + StalledUpstream { + port, + head_sent, + peer_closed, + release: Some(release_tx), + } + } + + fn parse_statements(source: &str) -> Vec { + let tokens = lex_wfl_with_positions(source); + let mut parser = Parser::new(&tokens); + parser + .parse() + .unwrap_or_else(|errors| { + panic!("response disconnect fixture did not parse: {errors:?}") + }) + .statements + } + + fn install_pending_request( + interpreter: &Interpreter, + env: &Rc>, + request_id: &str, + ) -> oneshot::Receiver { + let (sender, receiver) = oneshot::channel(); + interpreter.pending_responses.borrow_mut().insert( + request_id.to_string(), + PendingResponse { + sender: Arc::new(tokio::sync::Mutex::new(Some(sender))), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push(request_id.to_string()); + + let mut request = HashMap::new(); + request.insert( + "_response_sender".to_string(), + Value::Text(Arc::from(request_id)), + ); + env.borrow_mut() + .define_or_replace("req", Value::Object(Rc::new(RefCell::new(request)))); + receiver + } + + fn response_eval_is_stalled(interpreter: &Interpreter) -> bool { + let owned_streams = { + let owner = Arc::clone(&interpreter.open_http_streams.borrow()); + owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len() }; - self.assert_invariants(); - result + owned_streams == 1 + && interpreter.get_call_stack().len() == 1 + && *interpreter.current_count.borrow() == Some(1.0) + && *interpreter.in_count_loop.borrow() } - /// Picks the overload whose parameter count and declared parameter types - /// match the actual argument values: filter by count, drop candidates - /// whose concrete annotations reject an argument, then prefer the - /// candidate with the most concretely-matched parameters (ties resolve to - /// definition order). - fn select_overload( - overloaded: &OverloadedFunction, - args: &[Value], - line: usize, - column: usize, - ) -> Result, RuntimeError> { - let arity_matches: Vec<&Rc> = overloaded - .overloads - .iter() - .filter(|func| func.params.len() == args.len()) - .collect(); + async fn assert_precommit_disconnect_cancels(response_statement: &str, action_return: &str) { + let mut upstream = spawn_stalled_upstream().await; + let config = Arc::new(WflConfig { + timeout_seconds: 120, + outbound_stream_max_seconds: 120, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(config); + let source = format!( + "define action called stalled_value:\n\ + \x20\x20\x20\x20count from 1 to 1:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20open url at \"http://127.0.0.1:{}/stall\" and stream response as upstream\n\ + \x20\x20\x20\x20\x20\x20\x20\x20wait for 60000 milliseconds\n\ + \x20\x20\x20\x20end count\n\ + \x20\x20\x20\x20return {action_return}\n\ + end action\n\ + {response_statement}\n", + upstream.port + ); + let statements = parse_statements(&source); + assert_eq!( + statements.len(), + 2, + "unexpected fixture AST: {statements:#?}" + ); + let env = Rc::clone(interpreter.global_env()); + interpreter + .execute_statement(&statements[0], Rc::clone(&env)) + .await + .expect("define stalled action"); - if arity_matches.is_empty() { - let mut arities: Vec = overloaded - .overloads - .iter() - .map(|func| func.params.len()) - .collect(); - arities.sort_unstable(); - arities.dedup(); - let arities_str = arities + // Simulate an already-active outer count loop. Dropping the response + // evaluation must restore this exact run-state snapshot. + *interpreter.current_count.borrow_mut() = Some(41.0); + *interpreter.in_count_loop.borrow_mut() = true; + let receiver = install_pending_request(&interpreter, &env, "request-1"); + + let mut response = Box::pin(interpreter.execute_statement(&statements[1], env)); + let mut stalled = Box::pin(async { + loop { + if response_eval_is_stalled(&interpreter) { + return; + } + tokio::task::yield_now().await; + } + }); + tokio::time::timeout(Duration::from_secs(3), async { + tokio::select! { + result = response.as_mut() => { + panic!("response evaluation finished before the disconnect latch: {result:?}") + } + _ = stalled.as_mut() => {} + } + }) + .await + .expect("response expression never reached its stalled action"); + upstream + .head_sent + .await + .expect("stalled upstream did not send its response head"); + + // Causal trigger: only after the action owns a live upstream stream and + // is sleeping inside a count loop do we drop the client receiver. + drop(receiver); + let result = match tokio::time::timeout(Duration::from_secs(2), response.as_mut()).await { + Ok(result) => result, + Err(_) => { + drop(response); + interpreter.close_open_http_streams(); + if let Some(release) = upstream.release.take() { + let _ = release.send(()); + } + panic!( + "response expression evaluation did not cancel after its request disconnected" + ); + } + }; + drop(response); + + let error = result.expect_err("disconnect must cancel response precommit evaluation"); + assert_eq!(error.kind, ErrorKind::Cancelled, "wrong error: {error:?}"); + assert!( + interpreter.get_call_stack().is_empty(), + "dropped response evaluation leaked call frames" + ); + assert_eq!(interpreter.call_depth.get(), 0, "call depth leaked"); + assert_eq!( + *interpreter.current_count.borrow(), + Some(41.0), + "outer count binding was not restored" + ); + assert!( + *interpreter.in_count_loop.borrow(), + "outer count-loop state was not restored" + ); + assert!( + !interpreter + .pending_responses + .borrow() + .contains_key("request-1"), + "cancelled request remained pending" + ); + assert!( + !interpreter + .open_pending_requests + .borrow() .iter() - .map(usize::to_string) - .collect::>() - .join(" or "); - return Err(RuntimeError::new( - format!( - "No version of '{}' takes {} argument(s). It is defined with {} parameter(s).", - overloaded.name, - args.len(), - arities_str - ), - line, - column, - )); - } + .any(|id| id == "request-1"), + "cancelled request remained handler-owned" + ); - let mut best: Option<(&Rc, usize)> = None; - for func in &arity_matches { - let mut concrete_matches = 0usize; - let mut accepts = true; - for (param_type, arg) in func.param_types.iter().zip(args) { - if let Some(expected) = param_type { - // `any`/`Unknown` annotations accept everything and earn - // no specificity credit, matching untyped parameters. - if matches!(expected, Type::Any | Type::Unknown) { - continue; - } - if Self::value_matches_type(arg, expected) { - // A `nothing` argument is accepted by every parameter - // type but earns specificity credit only for an - // explicit `as nothing` parameter (its exact match) — - // otherwise versions accepting `nothing` stay tied - // and definition order decides, as documented. - if !matches!(arg, Value::Null | Value::Nothing) - || matches!(expected, Type::Nothing) - { - concrete_matches += 1; - } - } else { - accepts = false; - break; + tokio::time::timeout(Duration::from_secs(2), &mut upstream.peer_closed) + .await + .expect("cancelled response evaluation did not drop its upstream socket") + .expect("upstream close observation task ended early"); + assert!( + interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(), + "cancelled response evaluation retained upstream ownership" + ); + } + + #[tokio::test] + async fn buffered_content_evaluation_cancels_on_request_disconnect() { + assert_precommit_disconnect_cancels("respond to req with call stalled_value", "\"late\"") + .await; + } + + #[tokio::test] + async fn streaming_head_evaluation_cancels_on_request_disconnect() { + assert_precommit_disconnect_cancels( + "start streaming response to req with status call stalled_value and content type \"text/plain\" as out", + "201", + ) + .await; + } + + #[tokio::test] + async fn buffered_request_operand_cancels_on_request_disconnect() { + assert_precommit_disconnect_cancels("respond to (call stalled_value) with \"ok\"", "req") + .await; + } + + #[tokio::test] + async fn streaming_request_operand_cancels_on_request_disconnect() { + assert_precommit_disconnect_cancels( + "start streaming response to (call stalled_value) with status 200 as out", + "req", + ) + .await; + } +} + +#[cfg(test)] +mod response_disconnect_result_tests { + use super::*; + use crate::lexer::lex_wfl_with_positions; + use crate::parser::Parser; + use std::future::Future; + use std::task::Poll; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn parse_statement(source: &str) -> Statement { + let tokens = lex_wfl_with_positions(source); + let mut parser = Parser::new(&tokens); + let program = parser + .parse() + .unwrap_or_else(|errors| panic!("disconnect fixture did not parse: {errors:?}")); + assert_eq!(program.statements.len(), 1); + program.statements.into_iter().next().expect("statement") + } + + fn parse_statements(source: &str) -> Vec { + let tokens = lex_wfl_with_positions(source); + let mut parser = Parser::new(&tokens); + parser + .parse() + .unwrap_or_else(|errors| panic!("disconnect fixture did not parse: {errors:?}")) + .statements + } + + fn request_value(request_id: &str) -> Value { + let mut request = HashMap::new(); + request.insert( + "_response_sender".to_string(), + Value::Text(Arc::from(request_id)), + ); + Value::Object(Rc::new(RefCell::new(request))) + } + + async fn assert_response_commit_disconnect_is_cancelled(statement_source: &str) { + let interpreter = Interpreter::new(); + let env = Rc::clone(interpreter.global_env()); + env.borrow_mut() + .define_or_replace("req", request_value("request-commit")); + let (sender, receiver) = oneshot::channel(); + let shared_sender = Arc::new(tokio::sync::Mutex::new(Some(sender))); + interpreter.pending_responses.borrow_mut().insert( + "request-commit".to_string(), + PendingResponse { + sender: Arc::clone(&shared_sender), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-commit".to_string()); + + // Hold the sender lock so the statement can finish evaluation and reach + // the exact commit await without taking the sender yet. + let guard = shared_sender.lock().await; + let statement = parse_statement(statement_source); + let mut execution = Box::pin(interpreter.execute_statement(&statement, env)); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if !interpreter + .pending_responses + .borrow() + .contains_key("request-commit") + { + break; + } + tokio::select! { + result = execution.as_mut() => { + panic!("response finished before reaching the commit latch: {result:?}") } + _ = tokio::task::yield_now() => {} } } - if accepts && best.is_none_or(|(_, count)| concrete_matches > count) { - best = Some((func, concrete_matches)); - } - } + }) + .await + .expect("response did not reach its commit latch"); - match best { - Some((func, _)) => Ok(Rc::clone(func)), - None => { - let provided: Vec<&str> = args.iter().map(|arg| arg.type_name()).collect(); - let mut message = format!( - "No version of '{}' matches this call.\nYou provided ({}), but '{}' accepts:", - overloaded.name, - provided.join(", "), - overloaded.name - ); - for func in &arity_matches { - message.push_str(&format!("\n {}", Self::format_overload_signature(func))); + drop(receiver); + drop(guard); + let error = tokio::time::timeout(Duration::from_secs(2), execution.as_mut()) + .await + .expect("commit did not observe the closed receiver") + .expect_err("closed receiver must cancel the response commit"); + assert_eq!(error.kind, ErrorKind::Cancelled, "wrong error: {error:?}"); + } + + async fn spawn_commit_upstream() -> (u16, oneshot::Receiver<()>, oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind commit upstream"); + let port = listener + .local_addr() + .expect("commit upstream address") + .port(); + let (head_tx, head_sent) = oneshot::channel(); + let (closed_tx, peer_closed) = oneshot::channel(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept commit upstream"); + let mut request = Vec::new(); + let mut buffer = [0u8; 512]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut buffer).await.expect("read request head"); + assert!(read > 0, "client closed before commit request"); + request.extend_from_slice(&buffer[..read]); + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 1048576\r\n\ + Connection: close\r\n\r\n", + ) + .await + .expect("write commit response head"); + socket.flush().await.expect("flush commit response head"); + let _ = head_tx.send(()); + loop { + match socket.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(_) => {} } - Err(RuntimeError::new(message, line, column)) } - } + let _ = closed_tx.send(()); + }); + (port, head_sent, peer_closed) } - /// Whether a runtime value satisfies a declared parameter type. Untyped - /// and unknown annotations accept everything; `Custom` types match a - /// container instance of that type or of a descendant (via the parent - /// instance chain). - fn value_matches_type(value: &Value, expected: &Type) -> bool { - // `nothing` is compatible with every parameter type, mirroring the - // static checkers' `(_, Type::Nothing) => true` rule; ties among - // overloads resolve by specificity then definition order. - if matches!(value, Value::Null | Value::Nothing) { - return true; - } - match expected { - Type::Number => matches!(value, Value::Number(_)), - Type::Text => matches!(value, Value::Text(_)), - Type::Boolean => matches!(value, Value::Bool(_)), - Type::Nothing => matches!(value, Value::Null | Value::Nothing), - Type::Pattern => matches!(value, Value::Pattern(_)), - Type::List(_) => matches!(value, Value::List(_)), - Type::Map(_, _) => matches!(value, Value::Object(_)), - Type::Custom(name) => { - if name.eq_ignore_ascii_case("any") { - return true; + async fn assert_commit_disconnect_closes_evaluation_stream(response_statement: &str) { + let (port, head_sent, mut peer_closed) = spawn_commit_upstream().await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 120, + timeout_seconds: 120, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(config); + let source = format!( + "define action called open_then_return:\n\ + \x20\x20\x20\x20open url at \"http://127.0.0.1:{port}/commit\" and stream response as upstream\n\ + \x20\x20\x20\x20return 201\n\ + end action\n\ + {response_statement}\n" + ); + let statements = parse_statements(&source); + assert_eq!(statements.len(), 2); + let env = Rc::clone(interpreter.global_env()); + interpreter + .execute_statement(&statements[0], Rc::clone(&env)) + .await + .expect("define commit action"); + + env.borrow_mut() + .define_or_replace("req", request_value("request-resource")); + let (sender, receiver) = oneshot::channel(); + let shared_sender = Arc::new(tokio::sync::Mutex::new(Some(sender))); + interpreter.pending_responses.borrow_mut().insert( + "request-resource".to_string(), + PendingResponse { + sender: Arc::clone(&shared_sender), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-resource".to_string()); + let guard = shared_sender.lock().await; + + let mut execution = + Box::pin(interpreter.execute_statement(&statements[1], Rc::clone(&env))); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let at_commit = !interpreter + .pending_responses + .borrow() + .contains_key("request-resource"); + let owns_stream = interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len() + == 1; + if at_commit && owns_stream { + break; } - if let Value::ContainerInstance(instance) = value { - let mut current = Some(Rc::clone(instance)); - while let Some(inst) = current { - let inst_ref = inst.borrow(); - if inst_ref.container_type == *name { - return true; - } - current = inst_ref.parent.clone(); + tokio::select! { + result = execution.as_mut() => { + panic!("response finished before resource commit latch: {result:?}") } - false - } else { - false + _ = tokio::task::yield_now() => {} } } - _ => true, - } - } + }) + .await + .expect("response did not reach resource commit latch"); + head_sent.await.expect("commit upstream did not send head"); - /// Renders an overload's signature in WFL surface syntax for error - /// messages, e.g. `depict with value as number`. - fn format_overload_signature(func: &FunctionValue) -> String { - let name = func.name.as_deref().unwrap_or("anonymous"); - if func.params.is_empty() { - return format!("{name} (no parameters)"); + drop(receiver); + drop(guard); + let error = tokio::time::timeout(Duration::from_secs(2), execution.as_mut()) + .await + .expect("commit did not observe receiver disconnect") + .expect_err("commit disconnect must cancel"); + assert_eq!(error.kind, ErrorKind::Cancelled, "wrong error: {error:?}"); + + if tokio::time::timeout(Duration::from_secs(1), &mut peer_closed) + .await + .is_err() + { + interpreter.close_open_http_streams(); + let _ = tokio::time::timeout(Duration::from_secs(2), &mut peer_closed).await; + panic!("commit-time cancellation retained a stream opened during evaluation"); } - let params = func - .params - .iter() - .zip(&func.param_types) - .map(|(param, param_type)| match param_type { - Some(t) => format!("{param} as {}", crate::analyzer::format_param_type(t)), - None => param.clone(), - }) - .collect::>() - .join(" and "); - format!("{name} with {params}") + assert!( + interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(), + "commit cancellation retained stream ownership" + ); } - async fn call_function( - &self, - func: &FunctionValue, - args: Vec, - line: usize, - column: usize, - ) -> Result { - #[cfg(feature = "dhat-ad-hoc")] - dhat::ad_hoc_event(1); + #[tokio::test] + async fn buffered_commit_disconnect_is_exactly_cancelled() { + assert_response_commit_disconnect_is_cancelled("respond to req with \"ok\"").await; + } - #[cfg(debug_assertions)] - let func_name = func - .name - .clone() - .unwrap_or_else(|| "".to_string()); + #[tokio::test] + async fn streaming_head_commit_disconnect_is_exactly_cancelled() { + assert_response_commit_disconnect_is_cancelled( + "start streaming response to req with status 200 as out", + ) + .await; + } - if args.len() != func.params.len() { - return Err(RuntimeError::new( - format!( - "Expected {} arguments but got {}", - func.params.len(), - args.len() - ), - line, - column, - )); - } + #[tokio::test] + async fn already_disconnected_precheck_removes_stale_pending_state() { + let interpreter = Interpreter::new(); + let env = Rc::clone(interpreter.global_env()); + env.borrow_mut() + .define_or_replace("req", request_value("request-closed")); + let (sender, receiver) = oneshot::channel(); + interpreter.pending_responses.borrow_mut().insert( + "request-closed".to_string(), + PendingResponse { + sender: Arc::new(tokio::sync::Mutex::new(Some(sender))), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-closed".to_string()); + drop(receiver); - // Declared parameter types are runtime-enforced only for actions - // participating in overload dispatch: a lone member of a - // not-yet-complete overload set rejects non-matching arguments instead - // of silently running the wrong body — calls dispatch on "the - // overloads defined so far". Plain single actions keep their - // historical dynamic behavior (annotations are static hints, not - // runtime guards), preserving backward compatibility. `nothing` and - // untyped/`any` parameters accept every value. - if func.enforce_param_types.get() { - for ((param_name, param_type), arg) in - func.params.iter().zip(&func.param_types).zip(args.iter()) - { - if let Some(expected) = param_type - && !matches!(expected, Type::Any | Type::Unknown) - && !Self::value_matches_type(arg, expected) - { - let action_name = func.name.as_deref().unwrap_or("anonymous"); - return Err(RuntimeError::new( - format!( - "Argument '{param_name}' of '{action_name}' expects {}, but got {}", - crate::analyzer::format_param_type(expected), - arg.type_name() - ), - line, - column, - )); - } - } - } + let statement = parse_statement("respond to req with \"late\""); + let error = interpreter + .execute_statement(&statement, env) + .await + .expect_err("closed request must cancel before evaluation"); + assert_eq!(error.kind, ErrorKind::Cancelled); + assert!( + !interpreter + .pending_responses + .borrow() + .contains_key("request-closed"), + "early cancellation retained the pending sender" + ); + assert!( + !interpreter + .open_pending_requests + .borrow() + .iter() + .any(|id| id == "request-closed"), + "early cancellation retained handler ownership" + ); + } - let func_env = match func.env.upgrade() { - Some(env) => { - exec_trace!("call_function - Successfully upgraded function environment"); - env - } - None => { - exec_trace!("call_function - Failed to upgrade function environment"); - return Err(RuntimeError::new( - "Environment no longer exists".to_string(), - line, - column, - )); - } - }; + #[tokio::test] + async fn ordinary_response_expression_errors_remain_general_and_pending() { + let interpreter = Interpreter::new(); + let env = Rc::clone(interpreter.global_env()); + env.borrow_mut() + .define_or_replace("req", request_value("request-error")); + let (sender, _receiver) = oneshot::channel(); + interpreter.pending_responses.borrow_mut().insert( + "request-error".to_string(), + PendingResponse { + sender: Arc::new(tokio::sync::Mutex::new(Some(sender))), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-error".to_string()); - let call_env = Environment::new_child_env(&func_env); - exec_trace!("call_function - Created child environment for function call"); + let statement = parse_statement("respond to req with missing_value"); + let error = interpreter + .execute_statement(&statement, env) + .await + .expect_err("undefined content must remain an ordinary expression error"); + assert_eq!(error.kind, ErrorKind::General, "wrong error: {error:?}"); + assert!( + error.message.contains("missing_value"), + "ordinary expression diagnostic changed: {error}" + ); + assert!( + interpreter + .pending_responses + .borrow() + .contains_key("request-error"), + "ordinary expression error consumed the pending response" + ); + assert!( + interpreter + .open_pending_requests + .borrow() + .iter() + .any(|id| id == "request-error"), + "ordinary expression error removed handler ownership" + ); + } - for (_i, (param, arg)) in func.params.iter().zip(args.clone()).enumerate() { - exec_trace!( - "call_function - Binding parameter {} '{}' to argument {:?}", - _i, - param, - arg - ); + #[tokio::test] + async fn successful_buffered_response_behavior_is_preserved() { + let interpreter = Interpreter::new(); + let env = Rc::clone(interpreter.global_env()); + env.borrow_mut() + .define_or_replace("req", request_value("request-success")); + let (sender, receiver) = oneshot::channel(); + interpreter.pending_responses.borrow_mut().insert( + "request-success".to_string(), + PendingResponse { + sender: Arc::new(tokio::sync::Mutex::new(Some(sender))), + }, + ); + interpreter + .open_pending_requests + .borrow_mut() + .push("request-success".to_string()); - #[cfg(debug_assertions)] - exec_var_declare!(param, &arg); - // Bind parameters directly in the call scope so they shadow any - // same-named global/outer binding. `define` (which rejects names - // present in a parent scope) would otherwise leave the parameter - // unbound and let the body resolve to the global instead (#582). - let _ = call_env.borrow_mut().define_direct(param, arg.clone()); + let statement = parse_statement( + "respond to req with \"ok\" and content_type \"text/plain\" and status 201", + ); + interpreter + .execute_statement(&statement, env) + .await + .expect("connected buffered response must still succeed"); + let reply = receiver.await.expect("buffered response was not delivered"); + match reply { + HandlerReply::Buffered(response) => { + assert_eq!(response.status, 201); + assert_eq!(response.content_type, "text/plain"); + assert_eq!(response.content, b"ok"); + } + HandlerReply::Streaming { .. } => panic!("buffered respond produced streaming reply"), } + assert!( + !interpreter + .pending_responses + .borrow() + .contains_key("request-success"), + "successful response remained pending" + ); + assert!( + !interpreter + .open_pending_requests + .borrow() + .iter() + .any(|id| id == "request-success"), + "successful response retained handler ownership" + ); + } - // Enforce the shared recursion ceiling before descending another level, - // turning runaway recursion into a clean error instead of a native stack - // overflow. The dedicated `call_depth` counter (not `call_stack.len()`) - // is the enforcement source of truth: it is decremented by the RAII - // guard below as the call unwinds — including when a `try`/`when` - // catches a `ResourceLimit` — so catch-and-recurse cannot under-count - // and pile onto still-live native frames. - if let Err(exceeded) = self.budget.check_call_depth(self.call_depth.get()) { - return Err(self.budget_error(exceeded, line, column)); - } - let _depth_guard = CallDepthGuard::enter(&self.call_depth); + #[tokio::test] + async fn buffered_commit_disconnect_closes_evaluation_streams() { + assert_commit_disconnect_closes_evaluation_stream( + "respond to req with call open_then_return", + ) + .await; + } - let frame = CallFrame::new( - func.name - .clone() - .unwrap_or_else(|| "".to_string()), - line, - column, + #[tokio::test] + async fn streaming_commit_disconnect_closes_evaluation_streams() { + assert_commit_disconnect_closes_evaluation_stream( + "start streaming response to req with status call open_then_return as out", + ) + .await; + } + + #[tokio::test] + async fn backpressured_stream_write_disconnect_is_exactly_cancelled() { + let config = Arc::new(WflConfig { + web_server_response_timeout_seconds: 0, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(config); + let env = Rc::clone(interpreter.global_env()); + let handle_id = "respstream-test"; + let (sender, receiver) = mpsc::channel(RESPONSE_STREAM_BUFFER); + for _ in 0..RESPONSE_STREAM_BUFFER { + sender + .try_send(vec![0]) + .expect("fill response stream buffer"); + } + interpreter + .server_response_streams + .borrow_mut() + .insert(handle_id.to_string(), (sender, 0)); + interpreter + .open_response_streams + .borrow_mut() + .push(handle_id.to_string()); + let mut stream = HashMap::new(); + stream.insert( + "_server_stream".to_string(), + Value::Text(Arc::from(handle_id)), ); - self.call_stack.borrow_mut().push(frame); - exec_trace!("call_function - Pushed frame to call stack"); + env.borrow_mut() + .define_or_replace("out", Value::Object(Rc::new(RefCell::new(stream)))); + + let statement = parse_statement("write chunk \"next\" to out"); + let mut execution = Box::pin(interpreter.execute_statement(&statement, env)); + futures_util::future::poll_fn(|cx| match execution.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(result) => { + panic!("full response stream write did not backpressure: {result:?}") + } + }) + .await; + drop(receiver); - #[cfg(debug_assertions)] - exec_block_enter!(format!("function {}", func_name)); + let error = tokio::time::timeout(Duration::from_secs(2), execution.as_mut()) + .await + .expect("write did not wake after receiver disconnect") + .expect_err("closed stream receiver must cancel the write"); + assert_eq!(error.kind, ErrorKind::Cancelled, "wrong error: {error:?}"); + assert!( + !interpreter + .server_response_streams + .borrow() + .contains_key(handle_id), + "cancelled write retained its response stream" + ); + } +} - #[cfg(debug_assertions)] - let _guard = IndentGuard::new(); +#[cfg(test)] +mod request_wait_timeout_tests { + use super::*; + use crate::lexer::lex_wfl_with_positions; + use crate::parser::Parser; + + fn parse_statement(source: &str) -> Statement { + let tokens = lex_wfl_with_positions(source); + let mut parser = Parser::new(&tokens); + parser + .parse() + .unwrap_or_else(|errors| panic!("timeout fixture did not parse: {errors:?}")) + .statements + .into_iter() + .next() + .expect("statement") + } - exec_trace!("call_function - Executing function body"); - let result = self.execute_block(&func.body, call_env.clone()).await; - exec_trace!("call_function - Function execution result: {:?}", result); + async fn timeout_error(value: &str) -> RuntimeError { + let interpreter = Interpreter::new(); + let env = Rc::clone(interpreter.global_env()); + let (request_sender, request_receiver) = mpsc::channel(1); + interpreter.web_servers.borrow_mut().insert( + "srv".to_string(), + WflWebServer { + request_receiver: Arc::new(tokio::sync::Mutex::new(request_receiver)), + request_sender, + server_handle: None, + }, + ); + env.borrow_mut() + .define_or_replace("srv", Value::Text(Arc::from("WebServer::127.0.0.1:1"))); + let statement = parse_statement(&format!( + "wait for request comes in on srv as req with timeout {value}" + )); + interpreter + .execute_statement(&statement, env) + .await + .expect_err("invalid sub-millisecond timeout must be rejected") + } - #[cfg(debug_assertions)] - exec_block_exit!(format!("function {}", func_name)); + #[tokio::test] + async fn zero_request_timeout_has_the_established_positive_number_error() { + let error = timeout_error("0").await; + assert_eq!(error.kind, ErrorKind::General); + assert_eq!( + error.message, + "Timeout must be a positive number (milliseconds)" + ); + } - match result { - Ok((value, control_flow)) => { - self.call_stack.borrow_mut().pop(); + #[tokio::test] + async fn fractional_request_timeout_below_one_millisecond_is_rejected() { + let error = timeout_error("0.5").await; + assert_eq!(error.kind, ErrorKind::General); + assert_eq!( + error.message, + "Timeout must be at least 1 millisecond (got 0.5 ms); fractional values below 1 would truncate to zero and spin" + ); + } +} - let return_value = match control_flow { - ControlFlow::Return(val) => { - exec_trace!( - "call_function - Function explicitly returned with value: {:?}", - val - ); - val - } - _ => { - exec_trace!("call_function - Function completed with value: {:?}", value); - value +#[cfg(test)] +mod outbound_stream_deadline_tests { + use super::*; + use futures_util::task::AtomicWaker; + use std::future::Future; + use std::pin::Pin; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::{Context, Poll}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + async fn spawn_stream_cleanup_upstream(expected_requests: usize) -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind stream cleanup upstream"); + let port = listener.local_addr().expect("upstream address").port(); + tokio::spawn(async move { + for _ in 0..expected_requests { + let (mut socket, _) = listener.accept().await.expect("accept cleanup request"); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut buf = [0u8; 512]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut buf).await.expect("read request head"); + if read == 0 { + return; + } + request.extend_from_slice(&buf[..read]); } - }; - - exec_trace!( - "call_function - Function returned successfully with value: {:?}", - return_value - ); - Ok(return_value) + let request = String::from_utf8_lossy(&request); + let truncated = request.starts_with("GET /truncated "); + let unterminated = request.starts_with("GET /unterminated "); + let response = if truncated { + "HTTP/1.1 200 OK\r\nContent-Length: 10\r\nConnection: close\r\n\r\nx" + } else if unterminated { + "HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc" + } else { + "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + }; + socket + .write_all(response.as_bytes()) + .await + .expect("write response"); + socket.flush().await.expect("flush response"); + }); } - Err(err) => { - exec_trace!( - "call_function - Function execution failed with error: {:?}", - err - ); - if let Some(last_frame) = self.call_stack.borrow_mut().last_mut() { - last_frame.capture_locals(&call_env); - } - - let error_with_stack = err.clone(); + }); + port + } - self.call_stack.borrow_mut().pop(); + async fn spawn_stalled_streams( + expected_requests: usize, + ) -> (u16, tokio::sync::mpsc::UnboundedReceiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind terminal-retention upstream"); + let port = listener.local_addr().expect("upstream address").port(); + let (peer_closed_tx, peer_closed_rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + for _ in 0..expected_requests { + let (mut socket, _) = listener + .accept() + .await + .expect("accept terminal-retention request"); + let peer_closed_tx = peer_closed_tx.clone(); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0u8; 512]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut buffer).await.expect("read request head"); + if read == 0 { + return; + } + request.extend_from_slice(&buffer[..read]); + } + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 1048576\r\n\ + Connection: close\r\n\r\n", + ) + .await + .expect("write stalled response head"); + socket.flush().await.expect("flush stalled response head"); - Err(error_with_stack) + loop { + match socket.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + let _ = peer_closed_tx.send(()); + }); } - } + }); + (port, peer_closed_rx) } - fn evaluate_numeric_op( - &self, - left: Value, - right: Value, - line: usize, - column: usize, - op: Op, - err_gen: ErrGen, - ) -> Result - where - Op: Fn(f64, f64) -> Result, - ErrGen: Fn(&str, &str) -> String, - { - match (left, right) { - (Value::Number(a), Value::Number(b)) => match op(a, b) { - Ok(res) => Ok(Value::Number(res)), - Err(msg) => Err(RuntimeError::new(msg, line, column)), - }, - (a, b) => Err(RuntimeError::new( - err_gen(a.type_name(), b.type_name()), - line, - column, - )), - } + async fn assert_reapers_drained(client: &IoClient) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if client + .active_stream_reapers + .load(std::sync::atomic::Ordering::SeqCst) + == 0 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("finished streams left hard-lifetime reaper tasks sleeping"); } - fn evaluate_comparison_op( - &self, - left: Value, - right: Value, - line: usize, - column: usize, - op_symbol: &str, - comp: Comp, - ) -> Result - where - Comp: Fn(std::cmp::Ordering) -> bool, - { - match (left, right) { - (Value::Number(a), Value::Number(b)) => match a.partial_cmp(&b) { - Some(ord) => Ok(Value::Bool(comp(ord))), - None => Ok(Value::Bool(false)), - }, - (Value::Text(a), Value::Text(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), - (Value::Date(a), Value::Date(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), - (Value::Time(a), Value::Time(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), - (Value::DateTime(a), Value::DateTime(b)) => Ok(Value::Bool(comp(a.cmp(&b)))), - (a, b) => Err(RuntimeError::new( - format!( - "Cannot compare {} and {} with {}", - a.type_name(), - b.type_name(), - op_symbol - ), - line, - column, - )), - } + fn install_owned_empty_stream(client: &IoClient, handle_id: &str) -> StreamOwner { + let owner: StreamOwner = + Arc::new(std::sync::Mutex::new(HashSet::from( + [handle_id.to_string()], + ))); + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .insert( + handle_id.to_string(), + StreamSlot { + handle: Some(HttpStreamHandle { + stream: Box::pin(futures_util::stream::empty::>>()), + buffer: Vec::new(), + done: false, + bytes_read: 0, + total_deadline: None, + }), + deadline: None, + cancel: StreamCancel::new(), + reaper_abort: None, + owner: Some(Arc::clone(&owner)), + }, + ); + owner } - fn perform_binary_op( - &self, - operator: &Operator, - left_val: Value, - right_val: Value, - line: usize, - column: usize, - ) -> Result { - match operator { - Operator::Plus => self.add(left_val, right_val, line, column), - Operator::Minus => self.evaluate_numeric_op( - left_val, - right_val, - line, - column, - |a, b| Ok(a - b), - |a_type, b_type| format!("Cannot subtract {b_type} from {a_type}"), - ), - Operator::Multiply => self.evaluate_numeric_op( - left_val, - right_val, - line, - column, - |a, b| Ok(a * b), - |a_type, b_type| format!("Cannot multiply {a_type} and {b_type}"), - ), - Operator::Divide => self.evaluate_numeric_op( - left_val, - right_val, - line, - column, - |a, b| { - #[cfg(feature = "dhat-ad-hoc")] - dhat::ad_hoc_event(1); // Track division operations for memory profiling + async fn take_and_observe_clean_eof( + client: &IoClient, + handle_id: &str, + budget: &Arc, + ) -> TakenStream { + let Some(mut taken) = client + .take_stream(handle_id) + .expect("take synthetic empty stream") + else { + panic!("live synthetic stream unexpectedly resolved as clean EOF"); + }; + assert!( + !client + .stream_pull(&mut taken.handle, budget, &taken.cancel) + .await + .expect("observe upstream clean EOF"), + "an empty upstream must return clean EOF" + ); + assert!(taken.handle.done, "observing EOF must mark the body done"); + assert_eq!( + taken.cancel.terminal(), + Some(StreamTerminal::CleanEof), + "observing upstream None must latch CleanEof before slot removal" + ); + taken + } - if b == 0.0 { - Err("Division by zero".to_string()) - } else { - let res = a / b; - if !res.is_finite() { - Err(format!("Division resulted in invalid number: {res}")) - } else { - Ok(res) - } - } - }, - |a_type, b_type| format!("Cannot divide {a_type} by {b_type}"), - ), - Operator::Modulo => self.evaluate_numeric_op( - left_val, - right_val, - line, - column, - |a, b| { - #[cfg(feature = "dhat-ad-hoc")] - dhat::ad_hoc_event(1); // Track modulo operations for memory profiling + async fn assert_one_shot_clean_eof_after_put( + client: &IoClient, + handle_id: &str, + owner: &StreamOwner, + budget: Arc, + ) { + { + let registry = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + assert!( + !registry.live.contains_key(handle_id), + "put_stream must not recreate a heavy live slot after clean EOF" + ); + assert_eq!( + registry + .recent + .iter() + .filter(|entry| { + entry.id == handle_id && entry.reason == StreamTerminal::CleanEof + }) + .count(), + 1, + "put_stream must leave exactly one CleanEof tombstone" + ); + } + assert!( + owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(), + "slot removal before put_stream must clear handler ownership" + ); + assert_eq!( + client + .next_line(handle_id, Arc::clone(&budget)) + .await + .expect("consume restored CleanEof tombstone"), + None, + "the first read after terminalization must observe clean EOF" + ); + let later = client + .next_line(handle_id, budget) + .await + .expect_err("the CleanEof tombstone must be one-shot"); + assert!( + matches!(&later, HttpClientError::Request(message) + if message.contains("Unknown or already-closed")), + "the read after consuming CleanEof must report a closed/unknown handle, got {later:?}" + ); + } - if b == 0.0 { - Err("Modulo by zero".to_string()) - } else { - let res = a % b; - if !res.is_finite() { - Err(format!("Modulo resulted in invalid number: {res}")) - } else { - Ok(res) - } - } - }, - |a_type, b_type| format!("Cannot compute modulo of {a_type} by {b_type}"), - ), - Operator::Equals => Ok(Value::Bool(self.is_equal(&left_val, &right_val))), - Operator::NotEquals => Ok(Value::Bool(!self.is_equal(&left_val, &right_val))), - Operator::GreaterThan => { - self.evaluate_comparison_op(left_val, right_val, line, column, ">", |ord| { - matches!(ord, std::cmp::Ordering::Greater) - }) + struct DelayedHeadUpstream { + port: u16, + request_received: oneshot::Receiver<()>, + release_head: Option>, + peer_closed: oneshot::Receiver<()>, + } + + async fn spawn_delayed_head_upstream() -> DelayedHeadUpstream { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind delayed-head upstream"); + let port = listener.local_addr().expect("upstream address").port(); + let (request_tx, request_received) = oneshot::channel(); + let (release_head, release_rx) = oneshot::channel(); + let (peer_closed_tx, peer_closed) = oneshot::channel(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accept delayed head"); + let mut request = Vec::new(); + let mut buffer = [0u8; 512]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let read = socket.read(&mut buffer).await.expect("read request"); + assert!(read > 0, "client closed before request head"); + request.extend_from_slice(&buffer[..read]); } - Operator::LessThan => { - self.evaluate_comparison_op(left_val, right_val, line, column, "<", |ord| { - matches!(ord, std::cmp::Ordering::Less) - }) + let _ = request_tx.send(()); + release_rx.await.expect("release delayed response head"); + socket + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 1048576\r\n\ + Connection: close\r\n\r\n", + ) + .await + .expect("write delayed response head"); + socket.flush().await.expect("flush delayed response head"); + loop { + match socket.read(&mut buffer).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } } - Operator::GreaterThanOrEqual => { - self.evaluate_comparison_op(left_val, right_val, line, column, ">=", |ord| { - matches!(ord, std::cmp::Ordering::Greater | std::cmp::Ordering::Equal) - }) + let _ = peer_closed_tx.send(()); + }); + DelayedHeadUpstream { + port, + request_received, + release_head: Some(release_head), + peer_closed, + } + } + + struct GatedChunkState { + polled: AtomicBool, + ready: AtomicBool, + dropped: AtomicBool, + waker: AtomicWaker, + } + + impl GatedChunkState { + fn new() -> Arc { + Arc::new(Self { + polled: AtomicBool::new(false), + ready: AtomicBool::new(false), + dropped: AtomicBool::new(false), + waker: AtomicWaker::new(), + }) + } + + fn make_ready(&self) { + self.ready.store(true, Ordering::SeqCst); + self.waker.wake(); + } + } + + struct GatedChunkStream { + state: Arc, + yielded: bool, + } + + impl futures_util::Stream for GatedChunkStream { + type Item = reqwest::Result>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + if self.yielded { + return Poll::Ready(None); } - Operator::LessThanOrEqual => { - self.evaluate_comparison_op(left_val, right_val, line, column, "<=", |ord| { - matches!(ord, std::cmp::Ordering::Less | std::cmp::Ordering::Equal) - }) + self.state.polled.store(true, Ordering::SeqCst); + self.state.waker.register(cx.waker()); + if self.state.ready.load(Ordering::SeqCst) { + self.yielded = true; + Poll::Ready(Some(Ok(vec![7]))) + } else { + Poll::Pending } - Operator::And => Ok(Value::Bool(left_val.is_truthy() && right_val.is_truthy())), - Operator::Or => Ok(Value::Bool(left_val.is_truthy() || right_val.is_truthy())), - Operator::Contains => self.contains(left_val, right_val, line, column), } } - fn perform_unary_op( - &self, - operator: &UnaryOperator, - value: Value, - line: usize, - column: usize, - ) -> Result { - match operator { - UnaryOperator::Not => Ok(Value::Bool(!value.is_truthy())), - UnaryOperator::Minus => match value { - Value::Number(n) => Ok(Value::Number(-n)), - _ => Err(RuntimeError::new( - format!("Cannot negate {}", value.type_name()), - line, - column, - )), - }, + impl Drop for GatedChunkStream { + fn drop(&mut self) { + self.state.dropped.store(true, Ordering::SeqCst); } } - fn perform_concatenation(&self, left_val: Value, right_val: Value) -> Value { - // Optimization: Fast path for string concatenation to avoid format! machinery overhead - if let (Value::Text(left), Value::Text(right)) = (&left_val, &right_val) { - let mut s = String::with_capacity(left.len() + right.len()); - s.push_str(left); - s.push_str(right); - return Value::Text(Arc::from(s)); - } + #[tokio::test] + async fn close_during_active_read_returns_closed_and_drops_upstream() { + let (port, mut peer_closed) = spawn_stalled_streams(1).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 60, + timeout_seconds: 30, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let (_, _, handle_id) = client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/active-close"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open stalled stream"); - let result = format!("{left_val}{right_val}"); - Value::Text(Arc::from(result.as_str())) + let mut read = Box::pin(client.next_chunk(&handle_id, budget)); + futures_util::future::poll_fn(|cx| match read.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(result) => panic!("stalled read unexpectedly completed: {result:?}"), + }) + .await; + assert!( + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .get(&handle_id) + .is_some_and(|slot| slot.handle.is_none()), + "the close latch must observe a body read actively owning the handle" + ); + + assert!( + client.finish_stream_slot_sync(&handle_id, StreamTerminal::Closed), + "close must claim the active stream slot" + ); + let error = tokio::time::timeout(Duration::from_secs(2), read.as_mut()) + .await + .expect("active read did not wake after close") + .expect_err("active read must return Closed"); + assert!( + matches!(error, HttpClientError::Closed), + "active close returned the wrong error: {error:?}" + ); + tokio::time::timeout(Duration::from_secs(2), peer_closed.recv()) + .await + .expect("active close did not drop the upstream socket") + .expect("upstream close notifier ended early"); + assert_reapers_drained(&client).await; + assert!( + !client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .contains_key(&handle_id), + "active close retained its stream slot" + ); } - fn add( - &self, - left: Value, - right: Value, - line: usize, - column: usize, - ) -> Result { - match (left, right) { - (Value::Number(a), Value::Number(b)) => Ok(Value::Number(a + b)), - (Value::Text(a), Value::Text(b)) => { - // Optimization: Fast path for string concatenation - let mut s = String::with_capacity(a.len() + b.len()); - s.push_str(&a); - s.push_str(&b); - Ok(Value::Text(Arc::from(s))) - } - (Value::Text(a), b) => { - let result = format!("{a}{b}"); - Ok(Value::Text(Arc::from(result.as_str()))) + #[tokio::test] + async fn delayed_head_keeps_the_request_start_as_the_total_deadline_origin() { + let mut upstream = spawn_delayed_head_upstream().await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 2, + timeout_seconds: 30, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let started = Instant::now(); + let url = format!("http://127.0.0.1:{}/delayed-head", upstream.port); + let mut opening = + Box::pin(client.open_http_stream("GET", &url, &[], None, Arc::clone(&budget))); + tokio::select! { + result = opening.as_mut() => { + panic!("stream opened before the response-head latch: {result:?}") } - (a, Value::Text(b)) => { - let result = format!("{a}{b}"); - Ok(Value::Text(Arc::from(result.as_str()))) + received = &mut upstream.request_received => { + received.expect("upstream did not receive request"); } - (a, b) => Err(RuntimeError::new( - format!("Cannot add {} and {}", a.type_name(), b.type_name()), - line, - column, - )), } - } + tokio::time::sleep(Duration::from_millis(1_200)).await; + upstream + .release_head + .take() + .expect("head release") + .send(()) + .expect("release response head"); + let (_, _, handle_id) = tokio::time::timeout(Duration::from_secs(2), opening.as_mut()) + .await + .expect("stream did not open after response-head release") + .expect("delayed-head stream open"); - fn is_equal(&self, left: &Value, right: &Value) -> bool { - left == right + let deadline = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .get(&handle_id) + .and_then(|slot| slot.deadline) + .expect("positive cap must register a slot deadline"); + assert!( + deadline.saturating_duration_since(started) <= Duration::from_millis(2_100), + "the total deadline was restarted after the delayed head" + ); + assert!( + deadline.saturating_duration_since(Instant::now()) < Duration::from_secs(1), + "the delayed head must leave less than one second of the original cap" + ); + + tokio::time::timeout(Duration::from_secs(2), &mut upstream.peer_closed) + .await + .expect("spawned reaper did not drop delayed-head upstream") + .expect("upstream close notifier ended early"); + let error = client + .next_chunk(&handle_id, budget) + .await + .expect_err("read after delayed-head expiry must fail"); + assert!( + matches!(error, HttpClientError::Timeout { seconds: 2 }), + "delayed-head expiry lost its typed timeout: {error:?}" + ); + assert_reapers_drained(&client).await; } - // Helper method to create container instance with inheritance - #[allow(clippy::only_used_in_recursion)] - fn create_container_instance_with_inheritance( - &self, - container_type: &str, - env: &Rc>, - line: usize, - column: usize, - ) -> Result { - // Look up the container definition - let container_def = match env.borrow().get(container_type) { - Some(Value::ContainerDefinition(def)) => def.clone(), - _ => { - return Err(RuntimeError::new( - format!("Container '{container_type}' not found"), - line, - column, - )); + #[tokio::test] + async fn spawned_reaper_wins_over_a_simultaneously_ready_chunk() { + let (port, mut peer_closed) = spawn_stalled_streams(1).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 1, + timeout_seconds: 30, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let (_, _, handle_id) = client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/ready-expiry-race"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open stalled stream"); + let state = GatedChunkState::new(); + let cancel = { + let mut registry = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + let slot = registry.live.get_mut(&handle_id).expect("live stream slot"); + let handle = slot.handle.as_mut().expect("parked stream body"); + handle.stream = Box::pin(GatedChunkStream { + state: Arc::clone(&state), + yielded: false, + }); + // Leave the slot deadline and real spawned reaper intact, but keep + // the inner read timeout from independently deciding this race. + handle.total_deadline = None; + Arc::clone(&slot.cancel) + }; + tokio::time::timeout(Duration::from_secs(2), peer_closed.recv()) + .await + .expect("replacing the real body did not drop its upstream socket") + .expect("upstream close notifier ended early"); + + let mut read = Box::pin(client.next_chunk(&handle_id, budget)); + futures_util::future::poll_fn(|cx| match read.as_mut().poll(cx) { + Poll::Pending => Poll::Ready(()), + Poll::Ready(result) => panic!("gated read unexpectedly completed: {result:?}"), + }) + .await; + assert!( + state.polled.load(Ordering::SeqCst), + "gated body was not polled" + ); + assert!( + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .get(&handle_id) + .is_some_and(|slot| slot.handle.is_none()), + "active read did not take ownership before expiry" + ); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if cancel.terminal() == Some(StreamTerminal::Timeout) + && client.active_stream_reapers.load(Ordering::SeqCst) == 0 + { + break; + } + tokio::task::yield_now().await; } + }) + .await + .expect("the production reaper did not establish Timeout"); + state.make_ready(); + + let error = tokio::time::timeout(Duration::from_secs(2), read.as_mut()) + .await + .expect("simultaneously-ready race did not resolve") + .expect_err("expiry must win over the ready body chunk"); + assert!( + matches!(error, HttpClientError::Timeout { seconds: 1 }), + "ready chunk beat the spawned reaper: {error:?}" + ); + assert!( + state.dropped.load(Ordering::SeqCst), + "expired active body was not dropped" + ); + let registry = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + assert!( + !registry.live.contains_key(&handle_id), + "expired body was reinserted after the reaper" + ); + assert_eq!( + client.active_stream_reapers.load(Ordering::SeqCst), + 0, + "spawned reaper survived the race" + ); + } + + #[tokio::test] + async fn observed_clean_eof_wins_over_a_later_deadline_claim() { + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 0, + timeout_seconds: 10, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let handle_id = "observed-clean-eof".to_string(); + let cancel = StreamCancel::new(); + let owner: StreamOwner = + Arc::new(std::sync::Mutex::new(HashSet::from([handle_id.clone()]))); + + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .insert( + handle_id.clone(), + StreamSlot { + handle: Some(HttpStreamHandle { + stream: Box::pin(futures_util::stream::empty::>>()), + buffer: b"abc".to_vec(), + done: false, + bytes_read: 3, + total_deadline: None, + }), + deadline: None, + cancel, + reaper_abort: None, + owner: Some(Arc::clone(&owner)), + }, + ); + + let Some(TakenStream { mut handle, cancel }) = client + .take_stream(&handle_id) + .expect("take synthetic stream") + else { + panic!("synthetic stream unexpectedly resolved as clean EOF"); }; - // Create parent instance if container extends another - let parent_instance = if let Some(parent_type) = &container_def.extends { - // Recursively create parent instance - let parent = - self.create_container_instance_with_inheritance(parent_type, env, line, column)?; - Some(Rc::new(RefCell::new(parent))) - } else { + assert!( + !client + .stream_pull(&mut handle, &budget, &cancel) + .await + .expect("observe clean EOF"), + "the empty body must report clean EOF" + ); + assert!(handle.done, "observing EOF must mark the body complete"); + + let winner = cancel.terminate(StreamTerminal::Timeout); + assert_eq!( + winner, + StreamTerminal::CleanEof, + "a deadline claim after the upstream yielded EOF must not overwrite clean EOF" + ); + + let final_line = std::mem::take(&mut handle.buffer); + client + .put_stream(&handle_id, handle, &cancel) + .expect("terminalize the EOF-latched body"); + assert_eq!(final_line, b"abc"); + assert!( + owner + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(), + "terminalization must remove the stream owner" + ); + + { + let registry = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + assert!( + !registry.live.contains_key(&handle_id), + "terminalization must remove the live stream slot" + ); + assert_eq!( + registry + .recent + .iter() + .filter(|entry| { + entry.id == handle_id && entry.reason == StreamTerminal::CleanEof + }) + .count(), + 1, + "terminalization must retain exactly one one-shot clean-EOF record" + ); + } + + assert_eq!( + client + .next_line(&handle_id, budget) + .await + .expect("consume one-shot clean EOF"), None - }; + ); + } + + #[tokio::test] + async fn put_stream_restores_clean_eof_after_close_removed_the_live_slot() { + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 0, + timeout_seconds: 10, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let handle_id = "clean-eof-after-close"; + let owner = install_owned_empty_stream(&client, handle_id); + let TakenStream { handle, cancel } = + take_and_observe_clean_eof(&client, handle_id, &budget).await; - // Create instance with inherited properties - let mut instance_properties = HashMap::new(); + assert!( + client.finish_stream_slot_sync(handle_id, StreamTerminal::Closed), + "close must remove the live slot while the EOF-observing read owns its body" + ); + assert_eq!( + cancel.terminal(), + Some(StreamTerminal::CleanEof), + "the later close claim must not overwrite the observed CleanEof" + ); - // Copy properties from parent if exists - if let Some(ref parent) = parent_instance { - for (key, value) in &parent.borrow().properties { - instance_properties.insert(key.clone(), value.clone()); - } - } + client + .put_stream(handle_id, handle, &cancel) + .expect("put_stream must restore CleanEof after close removed the slot"); + assert_one_shot_clean_eof_after_put(&client, handle_id, &owner, budget).await; + } - // Initialize properties with default values from container definition - for (prop_name, prop_def) in &container_def.properties { - if let Some(default_value) = &prop_def.default_value { - instance_properties.insert(prop_name.clone(), default_value.clone()); - } + #[tokio::test] + async fn put_stream_deduplicates_clean_eof_after_reaper_removed_the_live_slot() { + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 0, + timeout_seconds: 10, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let handle_id = "clean-eof-after-reaper"; + let owner = install_owned_empty_stream(&client, handle_id); + let TakenStream { handle, cancel } = + take_and_observe_clean_eof(&client, handle_id, &budget).await; + + // Mirror the production reaper's critical section without a wall-clock + // sleep: remove the slot, attempt Timeout, clear ownership, and retain + // the first-wins terminal outcome before the active read calls put. + { + let now = Instant::now(); + let mut registry = client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + registry.prune_recent(now); + let mut slot = registry + .live + .remove(handle_id) + .expect("reaper-style terminalization must claim the live slot"); + let terminal = slot.cancel.terminate(StreamTerminal::Timeout); + assert_eq!( + terminal, + StreamTerminal::CleanEof, + "a reaper Timeout after observed EOF must retain CleanEof" + ); + slot.reaper_abort = None; + drop(slot.handle.take()); + remove_stream_owner(&mut slot, handle_id); + registry.remember_recent(handle_id.to_string(), terminal, now); } - Ok(ContainerInstanceValue { - container_type: container_type.to_string(), - properties: instance_properties, - parent: parent_instance, - line, - column, - }) + client + .put_stream(handle_id, handle, &cancel) + .expect("put_stream must preserve CleanEof after the reaper removed the slot"); + assert_one_shot_clean_eof_after_put(&client, handle_id, &owner, budget).await; } - fn contains( - &self, - left: Value, - right: Value, - line: usize, - column: usize, - ) -> Result { - match (left, right) { - (Value::List(list_rc), item) => { - let list = list_rc.borrow(); - for value in list.iter() { - if self.is_equal(value, &item) { - return Ok(Value::Bool(true)); - } - } - Ok(Value::Bool(false)) - } - (Value::Object(obj_rc), Value::Text(key)) => { - let obj = obj_rc.borrow(); - Ok(Value::Bool(obj.contains_key(&key.to_string()))) - } - (Value::Text(text), Value::Text(substring)) => { - Ok(Value::Bool(text.contains(&*substring))) - } - (a, b) => Err(RuntimeError::new( - format!( - "Cannot check if {} contains {}", - a.type_name(), - b.type_name() - ), - line, - column, - )), - } + #[test] + fn extreme_outbound_stream_max_seconds_does_not_panic() { + // u64::MAX must remain a finite cap rather than panicking or silently + // disabling the hard lifetime. + let before = Instant::now(); + let extreme = outbound_stream_deadline(u64::MAX) + .expect("an extreme positive cap must still produce a finite deadline"); + let effective = extreme.saturating_duration_since(before); + assert!( + effective <= Duration::from_secs(MAX_OUTBOUND_STREAM_DEADLINE_SECS + 1), + "extreme values must be clamped to the documented implementation ceiling; \ + got {effective:?}" + ); + assert_eq!( + outbound_stream_effective_seconds(u64::MAX), + Some(MAX_OUTBOUND_STREAM_DEADLINE_SECS), + "timeout diagnostics must report the effective clamp, not u64::MAX" + ); + assert!( + outbound_stream_deadline(0).is_none(), + "0 is the documented sentinel for no absolute total cap" + ); + assert!( + outbound_stream_deadline(1).is_some(), + "a normal positive cap must produce a deadline" + ); + assert!( + outbound_stream_deadline(MAX_OUTBOUND_STREAM_DEADLINE_SECS).is_some(), + "the clamp ceiling itself must still produce a deadline" + ); } - async fn list_files_recursive( - &self, - path: &str, - extensions: Option>, - ) -> Result, std::io::Error> { - use tokio::fs; + #[tokio::test] + async fn final_unterminated_line_survives_deadline_after_clean_eof() { + let port = spawn_stream_cleanup_upstream(1).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 1, + timeout_seconds: 10, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let (_, _, handle) = tokio::time::timeout( + Duration::from_secs(3), + interpreter.io_client.open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/unterminated"), + &[], + None, + Arc::clone(&budget), + ), + ) + .await + .expect("open stream hung") + .expect("open stream"); + interpreter + .io_client + .claim_stream_owner( + &handle, + &Arc::clone(&interpreter.open_http_streams.borrow()), + ) + .expect("claim unterminated stream ownership"); - let mut files = Vec::new(); - let mut dirs_to_process = vec![path.to_string()]; + let first = tokio::time::timeout( + Duration::from_secs(3), + interpreter + .io_client + .next_line(&handle, Arc::clone(&budget)), + ) + .await + .expect("first line read hung") + .expect("first line read"); + assert_eq!( + first.as_deref(), + Some("abc"), + "the final unterminated line is returned only after clean EOF was observed" + ); - while let Some(current_dir) = dirs_to_process.pop() { - let mut entries = fs::read_dir(¤t_dir).await?; + let (live_slots, clean_eof_records) = { + let registry = interpreter + .io_client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + ( + registry.live.len(), + registry + .recent + .iter() + .filter(|entry| entry.reason == StreamTerminal::CleanEof) + .count(), + ) + }; + assert_eq!( + live_slots, 0, + "returning the final unterminated line must remove its live stream slot" + ); + assert_eq!( + clean_eof_records, 1, + "the final line must leave exactly one lightweight clean-EOF record" + ); + assert_eq!( + interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len(), + 0, + "returning the final line must remove handler ownership immediately" + ); - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); - let path_str = path.to_string_lossy().to_string(); + tokio::time::sleep(Duration::from_millis(1_100)).await; + let eof = tokio::time::timeout( + Duration::from_secs(2), + interpreter + .io_client + .next_line(&handle, Arc::clone(&budget)), + ) + .await + .expect("clean EOF read hung") + .expect("clean EOF observed before the cap must not become Timeout"); + assert_eq!(eof, None); - if path.is_dir() { - dirs_to_process.push(path_str); - } else if path.is_file() { - // Check extension filter if provided - if let Some(ref exts) = extensions { - let file_ext = path - .extension() - .and_then(|ext| ext.to_str()) - .map(|ext| format!(".{ext}")); + let later = interpreter + .io_client + .next_line(&handle, budget) + .await + .expect_err("the single clean-EOF result must consume the handle"); + assert!( + matches!(&later, HttpClientError::Request(message) if message.contains("already-closed")), + "a later read must retain the established closed-handle error, got {later:?}" + ); + } - if let Some(ext) = file_ext - && exts.iter().any(|e| e == &ext) - { - files.push(Value::Text(path_str.into())); - } - } else { - files.push(Value::Text(path_str.into())); - } - } - } + #[tokio::test] + async fn unconsumed_clean_eof_records_are_bounded() { + const STREAM_COUNT: usize = MAX_RECENT_STREAM_TERMINALS + 8; + + let port = spawn_stream_cleanup_upstream(STREAM_COUNT).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 60 * 60, + timeout_seconds: 10, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + + for sequence in 0..STREAM_COUNT { + let (_, _, handle) = interpreter + .io_client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/unterminated"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open unterminated stream"); + interpreter + .io_client + .claim_stream_owner( + &handle, + &Arc::clone(&interpreter.open_http_streams.borrow()), + ) + .expect("claim unterminated stream ownership"); + + let final_line = interpreter + .io_client + .next_line(&handle, Arc::clone(&budget)) + .await + .expect("read final unterminated line"); + assert_eq!( + final_line.as_deref(), + Some("abc"), + "stream {sequence} did not yield its final unterminated line" + ); + + let (live_slots, recent_records, all_clean_eof) = { + let registry = interpreter + .io_client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + ( + registry.live.len(), + registry.recent.len(), + registry + .recent + .iter() + .all(|entry| entry.reason == StreamTerminal::CleanEof), + ) + }; + assert_eq!( + live_slots, 0, + "stream {sequence} retained a live slot after its final line" + ); + assert_eq!( + recent_records, + (sequence + 1).min(MAX_RECENT_STREAM_TERMINALS), + "clean-EOF records must fill only the bounded recent queue" + ); + assert!( + all_clean_eof, + "the no-follow-up wave retained a non-clean-EOF terminal" + ); + assert!( + interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(), + "stream {sequence} retained handler ownership after its final line" + ); } + } - Ok(files) + #[tokio::test] + async fn unread_expired_stream_metadata_and_ownership_are_bounded() { + const EXPECTED_RECENT_TIMEOUT_CAPACITY: usize = 64; + const STREAM_COUNT: usize = EXPECTED_RECENT_TIMEOUT_CAPACITY + 8; + + let (port, mut peer_closed) = spawn_stalled_streams(STREAM_COUNT).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 2, + timeout_seconds: 10, + ..WflConfig::default() + }); + let interpreter = Interpreter::with_config(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); + let mut handles = Vec::with_capacity(STREAM_COUNT); + + for sequence in 0..STREAM_COUNT { + let (_, _, handle) = interpreter + .io_client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/stall/{sequence}"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open stalled stream"); + interpreter + .io_client + .claim_stream_owner( + &handle, + &Arc::clone(&interpreter.open_http_streams.borrow()), + ) + .expect("claim stalled stream ownership"); + handles.push(handle); + } + + tokio::time::timeout(Duration::from_secs(5), async { + for _ in 0..STREAM_COUNT { + peer_closed + .recv() + .await + .expect("upstream close notification"); + } + }) + .await + .expect("expired stream bodies were not dropped promptly"); + assert_reapers_drained(&interpreter.io_client).await; + + let (live_slots, retained_terminals) = { + let registry = interpreter + .io_client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()); + (registry.live.len(), registry.recent.len()) + }; + let retained_ownership = interpreter + .open_http_streams + .borrow() + .lock() + .unwrap_or_else(|error| error.into_inner()) + .len(); + assert_eq!( + live_slots, 0, + "expired stream bodies must leave no live registry slots" + ); + assert!( + retained_terminals <= EXPECTED_RECENT_TIMEOUT_CAPACITY, + "recent terminal records must have a hard ceiling of \ + {EXPECTED_RECENT_TIMEOUT_CAPACITY}, got {retained_terminals}" + ); + assert_eq!( + retained_ownership, 0, + "the reaper must remove expired ids from handler ownership immediately" + ); + + let newest = handles.last().expect("newest stream"); + let recent = interpreter + .io_client + .next_chunk(newest, budget) + .await + .expect_err("a recent expired stream must retain its typed terminal"); + assert!( + matches!(recent, HttpClientError::Timeout { seconds: 2 }), + "a recent read-after-expiry must report Timeout, got {recent:?}" + ); } - async fn list_files_filtered( - &self, - path: &str, - extensions: Vec, - ) -> Result, std::io::Error> { - use tokio::fs; + #[tokio::test] + async fn eof_error_and_rapid_close_cancel_reaper_tasks_on_a_retained_runtime() { + const RAPID_CLOSES: usize = 40; + let port = spawn_stream_cleanup_upstream(RAPID_CLOSES + 2).await; + let config = Arc::new(WflConfig { + outbound_stream_max_seconds: 60 * 60, + timeout_seconds: 10, + ..WflConfig::default() + }); + let client = IoClient::new(Arc::clone(&config)); + let budget = Arc::new(ExecutionBudget::from_config(&config)); - let mut files = Vec::new(); - let mut entries = fs::read_dir(path).await?; + for sequence in 0..RAPID_CLOSES { + let (_, _, handle) = client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/close/{sequence}"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open stream for explicit close"); + assert!( + client.finish_stream_slot(&handle).await, + "explicit close should remove its live stream" + ); + } - while let Some(entry) = entries.next_entry().await? { - let path = entry.path(); + let (_, _, eof_handle) = client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/empty"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open empty stream"); + assert_eq!( + client + .next_chunk(&eof_handle, Arc::clone(&budget)) + .await + .expect("clean EOF"), + None + ); - if path.is_file() { - let path_str = path.to_string_lossy().to_string(); + let (_, _, error_handle) = client + .open_http_stream( + "GET", + &format!("http://127.0.0.1:{port}/truncated"), + &[], + None, + Arc::clone(&budget), + ) + .await + .expect("open truncated stream"); + let first = client.next_chunk(&error_handle, Arc::clone(&budget)).await; + let terminal = match first { + Ok(Some(_)) => client.next_chunk(&error_handle, budget).await, + other => other, + }; + assert!( + matches!(terminal, Err(HttpClientError::Request(_))), + "a truncated body should end as a network read error, got {terminal:?}" + ); - // Check extension filter - let file_ext = path - .extension() - .and_then(|ext| ext.to_str()) - .map(|ext| format!(".{ext}")); + // Keep this Tokio runtime alive while observing the task count. Runtime + // shutdown would cancel leaked timers and make this regression false-green. + assert_reapers_drained(&client).await; + assert!( + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .is_empty(), + "EOF, error, and explicit close must remove every stream slot" + ); + } - if let Some(ext) = file_ext - && extensions.iter().any(|e| e == &ext) - { - files.push(Value::Text(path_str.into())); - } - } - } + #[test] + fn a_ready_read_result_cannot_reinsert_after_expiry_claims_the_slot() { + let client = IoClient::new(Arc::new(WflConfig { + outbound_stream_max_seconds: 1, + ..WflConfig::default() + })); + let handle_id = "httpstream-race"; + let cancel = StreamCancel::new(); + client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .insert( + handle_id.to_string(), + StreamSlot { + handle: Some(HttpStreamHandle { + // Model a network chunk that became ready while the read + // owned the handle. + stream: Box::pin(futures_util::stream::iter([Ok(vec![1u8])])), + buffer: vec![1], + done: false, + bytes_read: 1, + total_deadline: outbound_stream_deadline(1), + }), + deadline: outbound_stream_deadline(1), + cancel: Arc::clone(&cancel), + reaper_abort: None, + owner: None, + }, + ); - Ok(files) + let Some(TakenStream { handle, cancel }) = client + .take_stream(handle_id) + .expect("active read takes body") + else { + panic!("live stream unexpectedly reported clean EOF"); + }; + cancel.terminate(StreamTerminal::Timeout); + let result = client.put_stream(handle_id, handle, &cancel); + + assert!( + matches!(result, Err(HttpClientError::Timeout { .. })), + "expiry must win over a ready body result, got {result:?}" + ); + assert!( + !client + .stream_handles + .lock() + .unwrap_or_else(|error| error.into_inner()) + .live + .contains_key(handle_id), + "an expired handle must never be reinserted after its active read" + ); } } diff --git a/src/parser/ast.rs b/src/parser/ast.rs index 29fd23be..112d1da0 100644 --- a/src/parser/ast.rs +++ b/src/parser/ast.rs @@ -190,6 +190,11 @@ pub enum Statement { }, MainLoop { body: Vec, + /// `main loop concurrently:` — run iterations of the body cooperatively + /// concurrently (each in its own isolated scope) instead of strictly + /// serially, so a slow request handler does not block its siblings. + /// Plain `main loop` keeps `concurrent = false` (byte-compatible serial). + concurrent: bool, line: usize, column: usize, }, @@ -409,6 +414,42 @@ pub enum Statement { line: usize, column: usize, }, + /// Streaming outbound HTTP request: + /// `open url at "" [with method .. and headers .. and body ..] and stream response as ` + /// + /// Unlike [`HttpRequestStatement`], this returns as soon as the status and + /// headers are received, WITHOUT buffering the body. It binds `` to a + /// streaming response handle (an object exposing `status`, `ok`, `headers`, + /// and an internal `_stream` id). The body is pulled incrementally with + /// `wait for next chunk from ` / `wait for next line from `. + HttpStreamStatement { + url: Expression, + method: Option, + headers: Option, + body: Option, + variable_name: String, + line: usize, + column: usize, + }, + /// `wait for next chunk from as ` — pull the next raw byte + /// chunk from a streaming response handle. Binds `` to `Binary`, or to + /// `nothing` at a clean end of stream. + WaitForNextChunkStatement { + source: Expression, + variable_name: String, + line: usize, + column: usize, + }, + /// `wait for next line from as ` — pull the next + /// newline-delimited line (trailing newline stripped) from a streaming + /// response handle. Binds `` to `Text`, or to `nothing` at end of + /// stream. + WaitForNextLineStatement { + source: Expression, + variable_name: String, + line: usize, + column: usize, + }, PushStatement { list: Expression, value: Expression, @@ -544,6 +585,61 @@ pub enum Statement { line: usize, column: usize, }, + /// `start streaming response to [with status ] [and content type + /// ] [and headers ] as ` — begin a streamed server response. + /// Sends the status/headers immediately and binds a server response-stream + /// handle; the body is written incrementally with `write line|chunk` and + /// ended with `close`. + StartStreamingResponseStatement { + request: Expression, + status: Option, + content_type: Option, + headers: Option, + variable_name: String, + line: usize, + column: usize, + }, + /// `write line to ` / `write chunk to ` — append + /// a framed line (a trailing newline is added) or a raw chunk (text or + /// binary, verbatim) to a server response stream. + StreamWriteStatement { + value: Expression, + target: Expression, + /// true for `write line` (newline appended), false for `write chunk`. + is_line: bool, + /// Backward-compat fallback for the ambiguous surface form + /// `write line to ` — where `line ` could equally + /// be the classic file write of a variable literally named `line ` + /// (WFL allows space-separated identifiers). When present and the runtime + /// `target` is **not** a server response stream, the statement falls back + /// to `write to ` (a `WriteToStatement`), so a + /// pre-existing file write is never silently reinterpreted as a stream + /// write. `None` when the form is unambiguous (e.g. a literal value, or a + /// bare marker directly before `to`). + fallback_content: Option>, + line: usize, + column: usize, + }, + /// `flush ` — advisory flush of a server response stream: hand any + /// queued bytes to the transport. + FlushStreamStatement { + target: Expression, + /// Original lexer-merged binding (`flush cache`) used to choose the + /// pre-streaming expression-statement interpretation. This metadata is + /// retained separately because ordinary expression rewrites such as + /// explicit `find ... in`, `replace ... in`, and `split ... by` can + /// legitimately discard the seeded left operand from `action_fallback`. + legacy_binding: Option, + /// Complete old expression-statement AST for the merged `flush …` form + /// (e.g. `Variable("flush cache")`, or `IndexAccess`/`PropertyAccess` over + /// that full name). Before streaming, `flush cache[0]` was an ordinary + /// expression statement; when the root binding exists the interpreter + /// evaluates this fallback instead of a stream flush. `None` when the + /// form cannot collide with a legacy expression (bare non-merged target). + action_fallback: Option, + line: usize, + column: usize, + }, // WebSocket statements. WebSockets mirror the HTTP server's design: warp // runs the socket in background tasks and the interpreter reacts to events // through registered handler blocks (dispatched while the program is inside diff --git a/src/parser/expr/binary.rs b/src/parser/expr/binary.rs index c7192457..ac9a9d6a 100644 --- a/src/parser/expr/binary.rs +++ b/src/parser/expr/binary.rs @@ -4,10 +4,25 @@ //! comparison, pattern matching, and custom language constructs. use super::super::{Argument, Expression, Operator, ParseError, Parser}; -use super::{ExprParser, PrimaryExprParser}; +use super::PrimaryExprParser; use crate::diagnostics::Span; use crate::lexer::token::Token; +#[derive(Clone, Copy)] +enum BinaryExpressionTerminator { + With, + In, +} + +impl BinaryExpressionTerminator { + fn matches(self, token: &Token) -> bool { + matches!( + (self, token), + (Self::With, Token::KeywordWith) | (Self::In, Token::KeywordIn) + ) + } +} + /// Trait for parsing binary expressions with operator precedence pub(crate) trait BinaryExprParser<'a> { /// Parses a binary expression with operator precedence. @@ -21,6 +36,37 @@ pub(crate) trait BinaryExprParser<'a> { /// Returns an `Expression` representing the parsed binary expression, or a `ParseError` if the syntax is invalid. fn parse_binary_expression(&mut self, precedence: u8) -> Result; + /// Parse a fresh binary expression while retaining a surrounding + /// streaming-response clause boundary through recursive operands. + fn parse_binary_expression_stopping_at_clause( + &mut self, + precedence: u8, + ) -> Result; + + /// Continue a binary expression from an already-parsed left-hand side. + /// + /// `parse_binary_expression` parses a fresh primary and then runs the + /// operator loop; this exposes just the loop so a caller that consumed the + /// leading operand itself (e.g. the merged `write line ` form) can + /// still absorb trailing `with`/operator continuations — so + /// `write line payload with "!" to out` parses its value like any other + /// expression instead of stopping at the bare variable. + fn parse_binary_continuation( + &mut self, + left: Expression, + precedence: u8, + ) -> Result; + + /// Like [`parse_binary_continuation`], but stops before streaming-response + /// clause connectives (`and headers`, `and content type`, `as out`, …) so a + /// `content type` / `headers` operand does not swallow the next clause as a + /// Boolean-AND / `with` continuation. + fn parse_binary_continuation_stopping_at_clause( + &mut self, + left: Expression, + precedence: u8, + ) -> Result; + /// Parses a function/action call expression. /// /// # Parameters @@ -30,10 +76,22 @@ pub(crate) trait BinaryExprParser<'a> { &mut self, call_line: usize, call_column: usize, + stop_at_clause: bool, ) -> Result; /// Parses a comma-separated or 'and'-separated argument list for action calls. - fn parse_argument_list(&mut self) -> Result, ParseError>; + fn parse_argument_list(&mut self) -> Result, ParseError> { + self.parse_argument_list_with_clause_boundary(false) + } + + fn parse_argument_list_stopping_at_clause(&mut self) -> Result, ParseError> { + self.parse_argument_list_with_clause_boundary(true) + } + + fn parse_argument_list_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result, ParseError>; /// Parses a single argument of an `of`-call (e.g. `fibonacci of n minus 1`). /// @@ -44,17 +102,57 @@ pub(crate) trait BinaryExprParser<'a> { /// separator), `with` (concatenation), `from`/`by`/`length` (stdlib call /// separators), comparisons, and pattern keywords, leaving those for the /// caller so multi-argument and postfix forms keep working. - fn parse_of_call_argument(&mut self) -> Result; + fn parse_of_call_argument(&mut self) -> Result { + self.parse_of_call_argument_with_clause_boundary(false) + } + + fn parse_of_call_argument_stopping_at_clause(&mut self) -> Result { + self.parse_of_call_argument_with_clause_boundary(true) + } + + fn parse_of_call_argument_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result; /// Multiplicative level of an `of`-call argument: times / divided by / `/` /// / `%` / modulo (precedence 3). - fn parse_of_call_arg_term(&mut self) -> Result; + fn parse_of_call_arg_term_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result; } -impl<'a> BinaryExprParser<'a> for Parser<'a> { - fn parse_binary_expression(&mut self, precedence: u8) -> Result { - let mut left = self.parse_primary_expression()?; +impl<'a> Parser<'a> { + fn parse_binary_expression_for_context( + &mut self, + precedence: u8, + stop_at_clause: bool, + terminator: Option, + ) -> Result { + let left = if stop_at_clause { + self.parse_primary_expression_stopping_at_clause()? + } else { + self.parse_primary_expression()? + }; + self.parse_binary_continuation_inner(left, precedence, stop_at_clause, terminator) + } + fn parse_binary_expression_for_clause_context( + &mut self, + precedence: u8, + stop_at_clause: bool, + ) -> Result { + self.parse_binary_expression_for_context(precedence, stop_at_clause, None) + } + + fn parse_binary_continuation_inner( + &mut self, + mut left: Expression, + precedence: u8, + stop_at_clause: bool, + terminator: Option, + ) -> Result { while let Some(token_pos) = self.cursor.peek() { let token = &token_pos.token; let line = token_pos.line; @@ -64,6 +162,22 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { if matches!(token, Token::Eol) || Parser::is_statement_starter(token) { break; } + if terminator.is_some_and(|terminator| terminator.matches(token)) { + break; + } + // Streaming-response clause connectives must not be absorbed as + // Boolean AND / `with` concatenation inside a clause operand. + if stop_at_clause { + if matches!(token, Token::KeywordAs) { + break; + } + if matches!(token, Token::KeywordAnd | Token::KeywordWith) { + let next = self.cursor.peek_n(1).map(|t| &t.token); + if Self::is_streaming_clause_keyword(next) { + break; + } + } + } // Precedence ladder (higher binds tighter): // 0: and, or @@ -113,12 +227,20 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { // desugars to `X >= A and X <= B`. Token::KeywordBetween => { self.bump_sync(); // Consume "between" - let lower = self.parse_binary_expression(2)?; + let lower = self.parse_binary_expression_for_context( + 2, + stop_at_clause, + terminator, + )?; self.expect_token( Token::KeywordAnd, "Expected 'and' between the bounds of 'is between'", )?; - let upper = self.parse_binary_expression(2)?; + let upper = self.parse_binary_expression_for_context( + 2, + stop_at_clause, + terminator, + )?; let lower_bound = Expression::BinaryOperation { left: Box::new(left.clone()), @@ -331,7 +453,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { if name != "count" && crate::builtins::is_builtin_function(name) { // Builtin function - keep legacy syntax self.bump_sync(); // Consume "with" - let arguments = self.parse_argument_list()?; + let arguments = if stop_at_clause { + self.parse_argument_list_stopping_at_clause()? + } else { + self.parse_argument_list()? + }; left = Expression::ActionCall { name: name.clone(), @@ -346,7 +472,8 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { // For all other cases (including user-defined actions), // treat 'with' as concatenation self.bump_sync(); // Consume "with" - let right = self.parse_expression()?; + let right = + self.parse_binary_expression_for_context(0, stop_at_clause, terminator)?; left = Expression::Concatenation { left: Box::new(left), right: Box::new(right), @@ -422,7 +549,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.bump_sync(); // Consume "pattern" } - let pattern_expr = self.parse_binary_expression(precedence + 1)?; + let pattern_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::PatternMatch { text: Box::new(left), @@ -442,14 +573,22 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.bump_sync(); // Consume "pattern" } - let pattern_expr = self.parse_binary_expression(precedence + 1)?; + let pattern_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + Some(BinaryExpressionTerminator::In), + )?; if let Some(in_token) = self.cursor.peek() && matches!(&in_token.token, Token::KeywordIn) { self.bump_sync(); // Consume "in" - let text_expr = self.parse_binary_expression(precedence + 1)?; + let text_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::PatternFind { text: Box::new(text_expr), @@ -478,21 +617,33 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.bump_sync(); // Consume "pattern" } - let pattern_expr = self.parse_binary_expression(precedence + 1)?; + let pattern_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + Some(BinaryExpressionTerminator::With), + )?; if let Some(with_token) = self.cursor.peek() && matches!(&with_token.token, Token::KeywordWith) { self.bump_sync(); // Consume "with" - let replacement_expr = self.parse_binary_expression(precedence + 1)?; + let replacement_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + Some(BinaryExpressionTerminator::In), + )?; if let Some(in_token) = self.cursor.peek() && matches!(&in_token.token, Token::KeywordIn) { self.bump_sync(); // Consume "in" - let text_expr = self.parse_binary_expression(precedence + 1)?; + let text_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::PatternReplace { text: Box::new(text_expr), @@ -529,7 +680,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.consume_optional_of(); // Parse the text expression to split - let text_expr = self.parse_binary_expression(precedence + 1)?; + let text_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; // Check for "by" (string split) or "on" (pattern split) if let Some(next_token) = self.cursor.peek() { @@ -537,8 +692,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { Token::KeywordBy => { // Handle "split text by delimiter" syntax self.bump_sync(); // Consume "by" - let delimiter_expr = - self.parse_binary_expression(precedence + 1)?; + let delimiter_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::StringSplit { text: Box::new(text_expr), @@ -559,7 +717,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.bump_sync(); // Consume "pattern" } - let pattern_expr = self.parse_binary_expression(precedence + 1)?; + let pattern_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::PatternSplit { text: Box::new(text_expr), @@ -612,7 +774,8 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { self.bump_sync(); // Consume "with" // RHS binds at precedence 2 (tighter than comparison), matching // how `contains`/`is` parse their right-hand side. - let right = self.parse_binary_expression(2)?; + let right = + self.parse_binary_expression_for_context(2, stop_at_clause, terminator)?; let fn_name = if is_starts { "starts_with" } else { @@ -647,7 +810,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { { self.bump_sync(); // Consume "pattern" - let pattern_expr = self.parse_binary_expression(precedence + 1)?; + let pattern_expr = self.parse_binary_expression_for_context( + precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::PatternMatch { text: Box::new(left), @@ -717,7 +884,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { } } - let right = self.parse_binary_expression(op_precedence + 1)?; + let right = self.parse_binary_expression_for_context( + op_precedence + 1, + stop_at_clause, + terminator, + )?; left = Expression::BinaryOperation { left: Box::new(left), @@ -733,11 +904,43 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { Ok(left) } +} + +impl<'a> BinaryExprParser<'a> for Parser<'a> { + fn parse_binary_expression(&mut self, precedence: u8) -> Result { + let left = self.parse_primary_expression()?; + self.parse_binary_continuation(left, precedence) + } + + fn parse_binary_expression_stopping_at_clause( + &mut self, + precedence: u8, + ) -> Result { + let left = self.parse_primary_expression_stopping_at_clause()?; + self.parse_binary_continuation_inner(left, precedence, true, None) + } + + fn parse_binary_continuation( + &mut self, + left: Expression, + precedence: u8, + ) -> Result { + self.parse_binary_continuation_inner(left, precedence, false, None) + } + + fn parse_binary_continuation_stopping_at_clause( + &mut self, + left: Expression, + precedence: u8, + ) -> Result { + self.parse_binary_continuation_inner(left, precedence, true, None) + } fn parse_call_expression( &mut self, call_line: usize, call_column: usize, + stop_at_clause: bool, ) -> Result { // We've already consumed Token::KeywordCall in the caller @@ -791,7 +994,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { } // Parse argument list - let arguments = self.parse_argument_list()?; + let arguments = if stop_at_clause { + self.parse_argument_list_stopping_at_clause()? + } else { + self.parse_argument_list()? + }; Ok(Expression::ActionCall { name, @@ -801,9 +1008,12 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { }) } - fn parse_of_call_argument(&mut self) -> Result { + fn parse_of_call_argument_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result { // Additive level: plus / minus (precedence 2). - let mut left = self.parse_of_call_arg_term()?; + let mut left = self.parse_of_call_arg_term_with_clause_boundary(stop_at_clause)?; while let Some(token_pos) = self.cursor.peek() { let (operator, line, column) = match &token_pos.token { @@ -816,7 +1026,7 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { _ => break, }; self.bump_sync(); // Consume the additive operator - let right = self.parse_of_call_arg_term()?; + let right = self.parse_of_call_arg_term_with_clause_boundary(stop_at_clause)?; left = Expression::BinaryOperation { left: Box::new(left), operator, @@ -829,8 +1039,15 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { Ok(left) } - fn parse_of_call_arg_term(&mut self) -> Result { - let mut left = self.parse_primary_expression()?; + fn parse_of_call_arg_term_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result { + let mut left = if stop_at_clause { + self.parse_primary_expression_stopping_at_clause()? + } else { + self.parse_primary_expression()? + }; while let Some(token_pos) = self.cursor.peek() { let line = token_pos.line; @@ -859,7 +1076,11 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { } } - let right = self.parse_primary_expression()?; + let right = if stop_at_clause { + self.parse_primary_expression_stopping_at_clause()? + } else { + self.parse_primary_expression()? + }; left = Expression::BinaryOperation { left: Box::new(left), operator, @@ -872,7 +1093,10 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { Ok(left) } - fn parse_argument_list(&mut self) -> Result, ParseError> { + fn parse_argument_list_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result, ParseError> { let mut arguments = Vec::with_capacity(4); let start_pos = self.cursor.pos(); @@ -903,7 +1127,7 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { // FIX: Parse expressions with precedence >= 1 (arithmetic operators) // This stops at 'and' (precedence 0), which is then used as argument separator - let arg_value = self.parse_binary_expression(1)?; + let arg_value = self.parse_binary_expression_for_clause_context(1, stop_at_clause)?; arguments.push(Argument { name: arg_name, @@ -912,6 +1136,13 @@ impl<'a> BinaryExprParser<'a> for Parser<'a> { if let Some(token) = self.cursor.peek() { if matches!(&token.token, Token::KeywordAnd) { + if stop_at_clause + && Parser::is_streaming_clause_keyword( + self.cursor.peek_n(1).map(|t| &t.token), + ) + { + break; + } self.bump_sync(); // Consume "and" continue; // Continue parsing next argument } else { diff --git a/src/parser/expr/primary.rs b/src/parser/expr/primary.rs index 697c577d..ba00e123 100644 --- a/src/parser/expr/primary.rs +++ b/src/parser/expr/primary.rs @@ -12,17 +12,33 @@ use std::sync::Arc; /// Trait for parsing primary (atomic) expressions pub(crate) trait PrimaryExprParser<'a> { /// Parses a primary expression (atomic expression like literals, variables, etc.) - fn parse_primary_expression(&mut self) -> Result; + fn parse_primary_expression(&mut self) -> Result { + self.parse_primary_expression_with_clause_boundary(false) + } + + /// Parses a primary while preserving a surrounding streaming-response + /// clause boundary through un-delimited postfix forms such as `values at 0`. + fn parse_primary_expression_stopping_at_clause(&mut self) -> Result { + self.parse_primary_expression_with_clause_boundary(true) + } + + fn parse_primary_expression_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result; /// Parses a single list element without parsing binary operators fn parse_list_element(&mut self) -> Result; } impl<'a> PrimaryExprParser<'a> for Parser<'a> { - fn parse_primary_expression(&mut self) -> Result { + fn parse_primary_expression_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result { #[cfg(debug_assertions)] let leading = self.cursor.peek().cloned(); - let result = self.parse_primary_expression_dispatch(); + let result = self.parse_primary_expression_dispatch(stop_at_clause); // Runtime coupling check between `can_start_primary_expression` (the // predicate `display`'s multi-value fold is built on, in @@ -89,12 +105,274 @@ impl<'a> PrimaryExprParser<'a> for Parser<'a> { } impl<'a> Parser<'a> { + /// Parse an un-delimited recursive expression while retaining the + /// streaming-response clause boundary inherited from its outer operand. + fn parse_expression_with_clause_boundary( + &mut self, + stop_at_clause: bool, + ) -> Result { + if stop_at_clause { + self.parse_binary_expression_stopping_at_clause(0) + } else { + self.parse_expression() + } + } + + /// Consume postfix accessors — bracket index (`["key"]`, `[0]`) and dotted + /// property access (`.field`) — that chain off a completed lead expression, so + /// they bind to the lead instead of splitting into bogus separate statements. + /// + /// This runs where the lexer has merged the lead into one identifier token and + /// left the accessors as following tokens: an identifier property access + /// (`upstream.headers["content-type"]`, which otherwise parsed as + /// `upstream.headers` + a stray `["content-type"]` list literal), and the + /// merged-command operands (`flush streams["a"]`, `flush obj.out`). Handles + /// arbitrary chains (`grid.rows[0][1]`, `obj.a.b["k"]`). + pub(crate) fn parse_trailing_postfix( + &mut self, + expr: Expression, + ) -> Result { + self.parse_trailing_postfix_with_clause_boundary(expr, false, None) + } + + /// Clause-aware counterpart to [`Self::parse_trailing_postfix`]. Natural + /// `at` indexes are not delimited, so their recursive expression parser must + /// inherit the response-clause boundary from the surrounding operand. + pub(crate) fn parse_trailing_postfix_stopping_at_clause( + &mut self, + expr: Expression, + ) -> Result { + self.parse_trailing_postfix_with_clause_boundary(expr, true, None) + } + + fn parse_trailing_postfix_after_member( + &mut self, + expr: Expression, + stop_at_clause: bool, + member_end: usize, + ) -> Result { + self.parse_trailing_postfix_with_clause_boundary(expr, stop_at_clause, Some(member_end)) + } + + fn parse_trailing_postfix_with_clause_boundary( + &mut self, + mut expr: Expression, + stop_at_clause: bool, + mut adjacent_member_end: Option, + ) -> Result { + while let Some(tok) = self.cursor.peek() { + let (line, column) = (tok.line, tok.column); + match &tok.token { + Token::LeftBracket => { + // A bracket separated from `.property` / `.method()` by + // whitespace starts a fresh display value (`display + // alice.name [1, 2]`). Only an adjacent bracket belongs to + // the completed member expression. Seeded streaming operands + // pass `None` and retain their historical permissive spacing. + if adjacent_member_end.is_some_and(|member_end| member_end != tok.byte_start) { + break; + } + // Anchor a "missing `]`" span to the `[` token itself, not the + // start of the file. + let (bracket_start, bracket_end) = (tok.byte_start, tok.byte_end); + self.bump_sync(); // Consume '[' + + let index = self.parse_expression()?; + + match self.cursor.peek() { + Some(closing) if closing.token == Token::RightBracket => { + adjacent_member_end = Some(closing.byte_end); + self.bump_sync(); // Consume ']' + } + Some(closing) => { + return Err(ParseError::from_token( + format!("Expected ']' after index, found {:?}", closing.token), + closing, + )); + } + None => { + return Err(ParseError::from_span( + "Expected ']' after index, found end of input".to_string(), + crate::diagnostics::Span { + start: bracket_start, + end: bracket_end, + }, + line, + column, + )); + } + } + + expr = Expression::IndexAccess { + collection: Box::new(expr), + index: Box::new(index), + line, + column, + }; + } + Token::Dot => { + // Anchor an end-of-input error to the `.` token, not the start + // of the file. + let (dot_start, dot_end) = (tok.byte_start, tok.byte_end); + self.bump_sync(); // Consume '.' + let (property, property_end) = match self.cursor.peek() { + // Keywords that double as common property names (e.g. + // `response.status`) are accepted, matching the primary + // dispatch's property-access handling. + Some(prop) => match &prop.token { + Token::Identifier(name) => (name.clone(), prop.byte_end), + Token::KeywordStatus => ("status".to_string(), prop.byte_end), + _ => { + return Err(ParseError::from_token( + "Expected a property name after '.'".to_string(), + prop, + )); + } + }, + None => { + return Err(ParseError::from_span( + "Expected a property name after '.', found end of input" + .to_string(), + crate::diagnostics::Span { + start: dot_start, + end: dot_end, + }, + line, + column, + )); + } + }; + self.bump_sync(); // Consume the property name + adjacent_member_end = Some(property_end); + // `.method(args)` — a method call, not a bare property access. + // Mirrors the primary dispatch so merged-command operands like + // `write line obj.method() to out` / `flush obj.method()` compose + // the call instead of leaving the `(...)` to dangle. + if matches!(self.cursor.peek().map(|t| &t.token), Some(Token::LeftParen)) { + self.bump_sync(); // Consume '(' + let mut arguments = Vec::new(); + if let Some(next) = self.cursor.peek() + && next.token != Token::RightParen + { + arguments.push(Argument { + name: None, + value: self.parse_expression()?, + }); + while matches!(self.cursor.peek().map(|t| &t.token), Some(Token::Comma)) + { + self.bump_sync(); // Consume ',' + arguments.push(Argument { + name: None, + value: self.parse_expression()?, + }); + } + } + let method_end = self + .cursor + .peek() + .filter(|token| token.token == Token::RightParen) + .map(|token| token.byte_end); + self.expect_token( + Token::RightParen, + "Expected ')' after method arguments", + )?; + adjacent_member_end = method_end; + expr = Expression::MethodCall { + object: Box::new(expr), + method: property, + arguments, + line, + column, + }; + } else { + expr = Expression::PropertyAccess { + object: Box::new(expr), + property, + line, + column, + }; + } + } + // Direct integer indexing (`values 0`) — same form as the ordinary + // primary postfix loop. Required so classic + // `write line values 0 to "/tmp/out"` still parses (issue #642). + Token::IntLiteral(index) => { + // Direct integer indexing after a property/method was not + // part of the legacy primary grammar. Leaving it unconsumed + // lets `display alice.name 5` fold two display values. + if adjacent_member_end.is_some() { + break; + } + if matches!( + expr, + Expression::Variable(_, _, _) + | Expression::IndexAccess { .. } + | Expression::FunctionCall { .. } + | Expression::PropertyAccess { .. } + | Expression::MethodCall { .. } + ) { + let index_val = *index; + let (base_line, base_col) = match &expr { + Expression::Variable(_, l, c) + | Expression::IndexAccess { + line: l, column: c, .. + } + | Expression::FunctionCall { + line: l, column: c, .. + } + | Expression::PropertyAccess { + line: l, column: c, .. + } + | Expression::MethodCall { + line: l, column: c, .. + } => (*l, *c), + _ => (line, column), + }; + self.bump_sync(); + expr = Expression::IndexAccess { + collection: Box::new(expr), + index: Box::new(Expression::Literal( + Literal::Integer(index_val), + line, + column, + )), + line: base_line, + column: base_col, + }; + } else { + break; + } + } + // Natural-language indexing (`values at 0`) — same as primary. + Token::KeywordAt => { + self.bump_sync(); + let index = if stop_at_clause { + self.parse_binary_expression_stopping_at_clause(0)? + } else { + self.parse_expression()? + }; + expr = Expression::IndexAccess { + collection: Box::new(expr), + index: Box::new(index), + line, + column, + }; + } + _ => break, + } + } + Ok(expr) + } + /// The actual primary-expression dispatch. Call `parse_primary_expression` /// (the trait method above), not this directly — it wraps this function /// with a debug-only check that keeps `can_start_primary_expression` from /// silently drifting away from what this dispatch really accepts, and /// recursive calls from within the arms below go through that wrapper too. - fn parse_primary_expression_dispatch(&mut self) -> Result { + fn parse_primary_expression_dispatch( + &mut self, + stop_at_clause: bool, + ) -> Result { // Strided run-budget checkpoint. Every operand (list element, operator- // chain term, call argument) routes through here, so this bounds a single // huge expression that the statement-boundary checkpoint would miss. @@ -217,7 +495,7 @@ impl<'a> Parser<'a> { let call_line = token.line; let call_column = token.column; self.bump_sync(); // Consume 'call' - return self.parse_call_expression(call_line, call_column); + return self.parse_call_expression(call_line, call_column, stop_at_clause); } Token::Identifier(name) => { self.bump_sync(); @@ -230,6 +508,7 @@ impl<'a> Parser<'a> { self.bump_sync(); // Consume '.' if let Some(property_token) = self.cursor.peek() { + let property_end = property_token.byte_end; // Keywords that are also common property names // (e.g. `response.status`) are accepted here let parsed_property = match &property_token.token { @@ -272,12 +551,18 @@ impl<'a> Parser<'a> { } } + let method_end = self + .cursor + .peek() + .filter(|token| token.token == Token::RightParen) + .map(|token| token.byte_end) + .unwrap_or(property_end); self.expect_token( Token::RightParen, "Expected ')' after method arguments", )?; - return Ok(Expression::MethodCall { + let call = Expression::MethodCall { object: Box::new(Expression::Variable( name.clone(), token_line, @@ -287,11 +572,22 @@ impl<'a> Parser<'a> { arguments, line: token_line, column: token_column, - }); + }; + return self.parse_trailing_postfix_after_member( + call, + stop_at_clause, + method_end, + ); } - // Property access without method call - return Ok(Expression::PropertyAccess { + // Property access without method call. + // Route through the trailing-index helper so a + // following `["key"]`/`[i]` binds to the + // property value (e.g. + // `upstream.headers["content-type"]`) instead + // of splitting off into a bogus list-literal + // statement. + let access = Expression::PropertyAccess { object: Box::new(Expression::Variable( name.clone(), token_line, @@ -300,7 +596,12 @@ impl<'a> Parser<'a> { property: property_name.clone(), line: token_line, column: token_column, - }); + }; + return self.parse_trailing_postfix_after_member( + access, + stop_at_clause, + property_end, + ); } else { return Err(ParseError::from_token( "Expected property name after '.'".to_string(), @@ -318,7 +619,11 @@ impl<'a> Parser<'a> { { self.bump_sync(); // Consume "with" - let arguments = self.parse_argument_list()?; + let arguments = if stop_at_clause { + self.parse_argument_list_stopping_at_clause()? + } else { + self.parse_argument_list()? + }; return Ok(Expression::ActionCall { name: name.clone(), @@ -348,7 +653,8 @@ impl<'a> Parser<'a> { } Token::KeywordNot => { self.bump_sync(); // Consume "not" - let expr = self.parse_primary_expression()?; + let expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; let token_line = token.line; let token_column = token.column; Ok(Expression::UnaryOperation { @@ -360,7 +666,8 @@ impl<'a> Parser<'a> { } Token::Minus => { self.bump_sync(); // Consume "-" - let expr = self.parse_primary_expression()?; + let expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; let token_line = token.line; let token_column = token.column; Ok(Expression::UnaryOperation { @@ -372,7 +679,7 @@ impl<'a> Parser<'a> { } Token::KeywordWith => { self.bump_sync(); // Consume "with" - let expr = self.parse_expression()?; + let expr = self.parse_expression_with_clause_boundary(stop_at_clause)?; Ok(expr) } Token::KeywordCount => { @@ -502,7 +809,8 @@ impl<'a> Parser<'a> { { self.bump_sync(); // Consume "size" self.expect_token(Token::KeywordOf, "Expected 'of' after 'file size'")?; - let file_handle = self.parse_primary_expression()?; + let file_handle = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; return Ok(Expression::FileSizeOf { file_handle: Box::new(file_handle), line: token_line, @@ -516,7 +824,8 @@ impl<'a> Parser<'a> { { self.bump_sync(); // Consume "exists" self.expect_token(Token::KeywordAt, "Expected 'at' after 'file exists'")?; - let path = self.parse_primary_expression()?; + let path = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; return Ok(Expression::FileExists { path: Box::new(path), line: token_line, @@ -545,7 +854,8 @@ impl<'a> Parser<'a> { Token::KeywordAt, "Expected 'at' after 'directory exists'", )?; - let path = self.parse_primary_expression()?; + let path = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; return Ok(Expression::DirectoryExists { path: Box::new(path), line: token_line, @@ -566,7 +876,8 @@ impl<'a> Parser<'a> { let token_column = token.column; // Parse process ID expression - let process_id = self.parse_primary_expression()?; + let process_id = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; // Check if followed by "is running" if let Some(next_token) = self.cursor.peek() @@ -615,7 +926,8 @@ impl<'a> Parser<'a> { self.expect_token(Token::KeywordOf, "Expected 'of' after header name")?; // Parse request expression - let request = self.parse_primary_expression()?; + let request = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; Ok(Expression::HeaderAccess { header_name, @@ -718,7 +1030,8 @@ impl<'a> Parser<'a> { Token::KeywordIn, "Expected 'in' after 'list files [recursively]'", )?; - let path = self.parse_primary_expression()?; + let path = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; // Handle recursive listing (if not already handled) if is_recursive { @@ -727,7 +1040,7 @@ impl<'a> Parser<'a> { && with_token.token == Token::KeywordWith { self.bump_sync(); // Consume "with" - let extensions = self.parse_extension_filter()?; + let extensions = self.parse_extension_filter(stop_at_clause)?; return Ok(Expression::ListFilesRecursive { path: Box::new(path), extensions: Some(extensions), @@ -756,7 +1069,8 @@ impl<'a> Parser<'a> { && with_token.token == Token::KeywordWith { self.bump_sync(); // Consume "with" - let extensions = self.parse_extension_filter()?; + let extensions = + self.parse_extension_filter(stop_at_clause)?; return Ok(Expression::ListFilesRecursive { path: Box::new(path), extensions: Some(extensions), @@ -775,7 +1089,7 @@ impl<'a> Parser<'a> { } Token::KeywordWith => { self.bump_sync(); // Consume "with" - let extensions = self.parse_extension_filter()?; + let extensions = self.parse_extension_filter(stop_at_clause)?; return Ok(Expression::ListFilesFiltered { path: Box::new(path), extensions, @@ -815,7 +1129,8 @@ impl<'a> Parser<'a> { Token::KeywordFrom, "Expected 'from' after 'read content'", )?; - let file_handle = self.parse_primary_expression()?; + let file_handle = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; return Ok(Expression::ReadContent { file_handle: Box::new(file_handle), line: token_line, @@ -830,7 +1145,8 @@ impl<'a> Parser<'a> { Token::KeywordFrom, "Expected 'from' after 'read binary'", )?; - let file_handle = self.parse_primary_expression()?; + let file_handle = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; return Ok(Expression::ReadBinaryContent { file_handle: Box::new(file_handle), line: token_line, @@ -845,7 +1161,8 @@ impl<'a> Parser<'a> { ) { // Speculatively parse count expression, then check for "bytes" let saved_pos = self.cursor.checkpoint(); - if let Ok(count_expr) = self.parse_primary_expression() + if let Ok(count_expr) = + self.parse_primary_expression_with_clause_boundary(stop_at_clause) && let Some(bytes_tok) = self.cursor.peek() && bytes_tok.token == Token::KeywordBytes { @@ -854,7 +1171,10 @@ impl<'a> Parser<'a> { Token::KeywordFrom, "Expected 'from' after 'read N bytes'", )?; - let file_handle = self.parse_primary_expression()?; + let file_handle = self + .parse_primary_expression_with_clause_boundary( + stop_at_clause, + )?; return Ok(Expression::ReadBinaryN { file_handle: Box::new(file_handle), count: Box::new(count_expr), @@ -876,12 +1196,13 @@ impl<'a> Parser<'a> { } Token::KeywordFind => { self.bump_sync(); // Consume "find" - let pattern_expr = self.parse_expression()?; + let pattern_expr = + self.parse_expression_with_clause_boundary(stop_at_clause)?; self.expect_token( Token::KeywordIn, "Expected 'in' after pattern in find expression", )?; - let text_expr = self.parse_expression()?; + let text_expr = self.parse_expression_with_clause_boundary(stop_at_clause)?; Ok(Expression::PatternFind { pattern: Box::new(pattern_expr), text: Box::new(text_expr), @@ -891,17 +1212,19 @@ impl<'a> Parser<'a> { } Token::KeywordReplace => { self.bump_sync(); // Consume "replace" - let pattern_expr = self.parse_primary_expression()?; + let pattern_expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; self.expect_token( Token::KeywordWith, "Expected 'with' after pattern in replace expression", )?; - let replacement_expr = self.parse_expression()?; + let replacement_expr = + self.parse_expression_with_clause_boundary(stop_at_clause)?; self.expect_token( Token::KeywordIn, "Expected 'in' after replacement in replace expression", )?; - let text_expr = self.parse_expression()?; + let text_expr = self.parse_expression_with_clause_boundary(stop_at_clause)?; Ok(Expression::PatternReplace { pattern: Box::new(pattern_expr), replacement: Box::new(replacement_expr), @@ -917,7 +1240,7 @@ impl<'a> Parser<'a> { // optionally consuming "of" (equivalent to `split X by DELIM`). self.consume_optional_of(); - let text_expr = self.parse_expression()?; + let text_expr = self.parse_expression_with_clause_boundary(stop_at_clause)?; // Check for "by" (string split) or "on" (pattern split) if let Some(next_token) = self.cursor.peek() { @@ -925,7 +1248,8 @@ impl<'a> Parser<'a> { Token::KeywordBy => { // Handle "split text by delimiter" syntax self.bump_sync(); // Consume "by" - let delimiter_expr = self.parse_expression()?; + let delimiter_expr = + self.parse_expression_with_clause_boundary(stop_at_clause)?; Ok(Expression::StringSplit { text: Box::new(text_expr), delimiter: Box::new(delimiter_expr), @@ -940,7 +1264,8 @@ impl<'a> Parser<'a> { Token::KeywordPattern, "Expected 'pattern' after 'on' in split expression", )?; - let pattern_expr = self.parse_expression()?; + let pattern_expr = + self.parse_expression_with_clause_boundary(stop_at_clause)?; Ok(Expression::PatternSplit { text: Box::new(text_expr), pattern: Box::new(pattern_expr), @@ -1019,7 +1344,8 @@ impl<'a> Parser<'a> { } else { // Try to parse as "contains X in Y" // Parse the needle expression - let needle = self.parse_primary_expression()?; + let needle = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; // Check if next token is "in" if let Some(in_token) = self.cursor.peek() @@ -1028,7 +1354,9 @@ impl<'a> Parser<'a> { self.bump_sync(); // Consume "in" // Parse the haystack expression - let haystack = self.parse_primary_expression()?; + let haystack = self.parse_primary_expression_with_clause_boundary( + stop_at_clause, + )?; // Create a function call expression for contains Ok(Expression::FunctionCall { @@ -1207,7 +1535,11 @@ impl<'a> Parser<'a> { // but stops at `and`, `with`, `from`/`by`, comparisons, // and pattern keywords so multi-argument and postfix // forms keep working. - let first_arg = self.parse_of_call_argument()?; + let first_arg = if stop_at_clause { + self.parse_of_call_argument_stopping_at_clause()? + } else { + self.parse_of_call_argument()? + }; let is_function_call = matches!( expr, @@ -1237,11 +1569,23 @@ impl<'a> Parser<'a> { ); if is_separator { + if stop_at_clause + && matches!(&sep_token.token, Token::KeywordAnd) + && Self::is_streaming_clause_keyword( + self.cursor.peek_n(1).map(|t| &t.token), + ) + { + break; + } self.bump_sync(); // Consume the separator // Each argument absorbs arithmetic while // `and`/`with`/`from`/`by` stay separators. - let arg_value = self.parse_of_call_argument()?; + let arg_value = if stop_at_clause { + self.parse_of_call_argument_stopping_at_clause()? + } else { + self.parse_of_call_argument()? + }; arguments.push(Argument { name: None, @@ -1269,7 +1613,11 @@ impl<'a> Parser<'a> { Token::KeywordAt => { self.bump_sync(); // Consume "at" - let index = self.parse_expression()?; + let index = if stop_at_clause { + self.parse_binary_expression_stopping_at_clause(0)? + } else { + self.parse_expression()? + }; expr = Expression::IndexAccess { collection: Box::new(expr), diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a25d2213..62a2d0fb 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -610,6 +610,42 @@ impl<'a> StmtParser<'a> for Parser<'a> { // `send websocket message to ` and // `broadcast websocket message to `. The command // words (and a bare identifier message) lex as one merged token. + // `start streaming response to ... as `. `start` is a + // keyword; `streaming` is a contextual identifier; `response` is + // a keyword. `response` being a keyword means it never merges into + // the identifier, so `streaming` always arrives as the *exact* + // token `Identifier("streaming")`. Match only that exact token — + // NOT `starts_with("streaming ")`, which would hijack an unrelated + // `start streaming ...` (the lexer merges those into + // `Identifier("streaming ")`) into a confusing parse error. + // Any other statement-initial use of `start` (e.g. the + // `start of text` pattern anchor) falls through to its own handler. + Token::KeywordStart + if matches!( + self.cursor.peek_next().map(|t| &t.token), + Some(Token::Identifier(id)) if id == "streaming" + ) => + { + self.parse_start_streaming_response() + } + // `flush ` — the target merges into the token + // (`flush out` -> Identifier("flush out")). Only match when an + // operand follows, so a bare `flush` used as an action/variable + // name still parses as an expression statement. + Token::Identifier(id) if id.starts_with("flush ") => self.parse_flush_stream(), + // Parenthesized and explicit-call targets do not merge into the + // leading `flush` token. These two starters are unambiguous; + // broader primary-expression dispatch would steal legacy + // expressions such as `flush with suffix` and `flush at 0`. + Token::Identifier(id) + if id == "flush" + && matches!( + self.cursor.peek_next().map(|t| &t.token), + Some(Token::LeftParen | Token::KeywordCall) + ) => + { + self.parse_flush_stream() + } Token::Identifier(id) if id.starts_with("send websocket message") => { self.parse_send_websocket_message() } diff --git a/src/parser/stmt/control_flow.rs b/src/parser/stmt/control_flow.rs index 615747cf..1d19f72c 100644 --- a/src/parser/stmt/control_flow.rs +++ b/src/parser/stmt/control_flow.rs @@ -610,6 +610,19 @@ impl<'a> ControlFlowParser<'a> for Parser<'a> { { let main_token = self.bump_sync().unwrap(); // Consume "main" self.expect_token(Token::KeywordLoop, "Expected 'loop' after 'main'")?; + + // Optional `concurrently` marker (a contextual identifier). Plain + // `main loop` stays serial and byte-compatible; `main loop + // concurrently:` opts into cooperative concurrent handling. + let mut concurrent = false; + if let Some(token) = self.cursor.peek() + && let Token::Identifier(id) = &token.token + && id == "concurrently" + { + self.bump_sync(); // Consume "concurrently" + concurrent = true; + } + self.expect_token(Token::Colon, "Expected ':' after 'main loop'")?; // Skip any Eol tokens after the colon @@ -633,6 +646,7 @@ impl<'a> ControlFlowParser<'a> for Parser<'a> { Ok(Statement::MainLoop { body, + concurrent, line: main_token.line, column: main_token.column, }) diff --git a/src/parser/stmt/io.rs b/src/parser/stmt/io.rs index 494c637f..08c2b2a6 100644 --- a/src/parser/stmt/io.rs +++ b/src/parser/stmt/io.rs @@ -3,9 +3,153 @@ use super::super::{Expression, FileOpenMode, Literal, ParseError, Parser, Statement}; use super::database::DatabaseParser; use crate::lexer::token::Token; -use crate::parser::expr::{ExprParser, PrimaryExprParser}; +use crate::parser::expr::{BinaryExprParser, ExprParser, PrimaryExprParser}; use std::sync::Arc; +impl<'a> Parser<'a> { + /// Continue an operand from an already-parsed leading expression: + /// trailing postfix (`[]`, `.field`, `.method()`, `at`, direct-integer index), + /// an optional ` of ` call, then any `with`/operator + /// continuation — exactly as a normal expression value would parse. + /// + /// Shared by ordinary, merged, and unmerged `write line|chunk`, `content + /// type`, `headers`, and `flush` operands so they all support the same + /// postfix/call/operator grammar (issue #642). + /// + /// The ambiguous merged `write line|chunk ...` form has two readings + /// (stream: split-off ``; classic file write: whole `line `) + /// that differ only in the leading operand. They are parsed independently — + /// same tokens, via a cursor rewind between the two calls — because a + /// continuation can desugar differently per operand (a builtin name becomes + /// an `ActionCall`, `is between` duplicates the left, `starts/ends with` and + /// the pattern operators build calls), so deriving one AST from the other by + /// leaf-swapping silently corrupted the classic reading. + pub(crate) fn parse_seeded_expression_continuation( + &mut self, + lead: Expression, + stop_at_clause: bool, + ) -> Result { + let mut lead = lead; + loop { + // The lexer merges the command word with the operand identifier and + // leaves postfix accessors as following tokens. Compose them before + // checking for `of`, and repeat after an `of` call so + // `choose of (values)[0]` indexes the call result just like an + // ordinary expression. + lead = if stop_at_clause { + self.parse_trailing_postfix_stopping_at_clause(lead)? + } else { + self.parse_trailing_postfix(lead)? + }; + if !matches!(self.cursor.peek().map(|t| &t.token), Some(Token::KeywordOf)) { + break; + } + + // Anchor the ` of ` call to the `of` keyword itself, + // matching how the rest of the parser positions FunctionCall nodes so + // error spans point at the operator, not the lead (review feedback). + let (of_line, of_column) = self + .bump_sync() + .map(|t| (t.line, t.column)) + .expect("peeked `of` immediately above"); + // Parse the `of`-call argument(s) EXACTLY as the primary parser does: + // each argument absorbs arithmetic (`fibonacci of n minus 1` means + // `fibonacci of (n minus 1)`, not `(fibonacci of n) minus 1`), and + // `and`/`from`/`by`/`length` join multiple arguments. + let mut arguments = vec![crate::parser::ast::Argument { + name: None, + value: if stop_at_clause { + self.parse_of_call_argument_stopping_at_clause()? + } else { + self.parse_of_call_argument()? + }, + }]; + while let Some(sep) = self.cursor.peek() { + let is_separator = matches!( + &sep.token, + Token::KeywordAnd | Token::KeywordFrom | Token::KeywordBy + ) || matches!( + &sep.token, + Token::Identifier(id) if id.eq_ignore_ascii_case("length") + ); + if !is_separator { + break; + } + if stop_at_clause + && matches!(&sep.token, Token::KeywordAnd) + && Self::is_streaming_clause_keyword(self.cursor.peek_n(1).map(|t| &t.token)) + { + break; + } + self.bump_sync(); // Consume the separator + arguments.push(crate::parser::ast::Argument { + name: None, + value: if stop_at_clause { + self.parse_of_call_argument_stopping_at_clause()? + } else { + self.parse_of_call_argument()? + }, + }); + } + lead = Expression::FunctionCall { + function: Box::new(lead), + arguments, + line: of_line, + column: of_column, + }; + } + if stop_at_clause { + self.parse_binary_continuation_stopping_at_clause(lead, 0) + } else { + self.parse_binary_continuation(lead, 0) + } + } + + /// Parse a lexer-merged response-clause operand, stopping before the next + /// response clause connective. + pub(crate) fn parse_clause_operand_from_lead( + &mut self, + lead: Expression, + ) -> Result { + self.parse_seeded_expression_continuation(lead, true) + } + + /// Parse a complete ordinary/unmerged operand through the same seeded + /// continuation used for lexer-merged operands. + pub(crate) fn parse_unmerged_operand( + &mut self, + stop_at_clause: bool, + ) -> Result { + let lead = if stop_at_clause { + self.parse_primary_expression_stopping_at_clause()? + } else { + self.parse_primary_expression()? + }; + self.parse_seeded_expression_continuation(lead, stop_at_clause) + } + + pub(crate) fn is_streaming_clause_keyword(tok: Option<&Token>) -> bool { + match tok { + Some(Token::KeywordAs) | Some(Token::KeywordContent) | Some(Token::KeywordStatus) => { + true + } + Some(Token::Identifier(id)) => { + id == "headers" + || id.starts_with("headers ") + || id == "content_type" + || id.starts_with("content_type ") + || id.starts_with("content type") + } + _ => false, + } + } + + /// Alias used by the write-statement parsers. + fn parse_write_value_from_lead(&mut self, lead: Expression) -> Result { + self.parse_seeded_expression_continuation(lead, false) + } +} + pub(crate) trait IoParser<'a>: ExprParser<'a> { fn parse_display_statement(&mut self) -> Result; fn parse_open_file_statement(&mut self) -> Result; @@ -255,6 +399,28 @@ impl<'a> IoParser<'a> for Parser<'a> { } }); } + // `stream response as ` — return the status/headers + // immediately and bind a streaming handle instead of buffering + // the body. `stream` is a contextual identifier (not a + // keyword), so match it as one; `response` is a keyword. + Token::Identifier(name) if name == "stream" => { + self.bump_sync(); // Consume "stream" + self.expect_token( + Token::KeywordResponse, + "Expected 'response' after 'stream'", + )?; + self.expect_token(Token::KeywordAs, "Expected 'as' after 'stream response'")?; + let variable_name = parse_variable_name(self, open_token)?; + return Ok(Statement::HttpStreamStatement { + url: url_expr, + method, + headers, + body, + variable_name, + line: open_token.line, + column: open_token.column, + }); + } // The lexer merges consecutive identifiers into multi-word // names, so `headers auth_headers` arrives as the single // token Identifier("headers auth_headers"). Match both the @@ -771,6 +937,166 @@ impl<'a> IoParser<'a> for Parser<'a> { fn parse_write_to_statement(&mut self) -> Result { let token_pos = self.bump_sync().unwrap(); // Consume "write" + // `write line to ` / `write chunk to ` — + // append to a server response stream. `line`/`chunk` are contextual + // identifiers; the lexer merges a following bare-identifier value into + // the same token (`line payload` -> Identifier("line payload")), so + // split the value off the marker, mirroring the websocket-message form. + // + // Do NOT intercept a bare `line`/`chunk` followed by an expression + // continuation. In that shape the marker is itself the classic file-write + // value (`write line with "!" to file`, `write line[0] to file`, ...), + // not a streaming marker with a missing operand. This is common in + // line-by-line file code and predates response streaming. + let bare_marker_before_classic_continuation = matches!( + self.cursor.peek(), + Some(t) if matches!(&t.token, Token::Identifier(id) if id == "line" || id == "chunk") + ) && matches!( + self.cursor.peek_kind_n(1), + Some( + Token::KeywordTo + | Token::KeywordWith + | Token::KeywordAt + | Token::Dot + | Token::LeftBracket + | Token::KeywordOf + | Token::Plus + | Token::KeywordPlus + | Token::Minus + | Token::KeywordMinus + | Token::KeywordTimes + | Token::KeywordDividedBy + | Token::KeywordDivided + | Token::Slash + | Token::Percent + | Token::KeywordModulo + | Token::Equals + | Token::KeywordIs + | Token::KeywordAnd + | Token::KeywordOr + | Token::KeywordMatches + | Token::KeywordContains + | Token::KeywordFind + | Token::KeywordReplace + | Token::KeywordSplit + ) + ); + + if !bare_marker_before_classic_continuation + && let Some(next_token) = self.cursor.peek() + && let Token::Identifier(id) = &next_token.token + && (id == "line" + || id == "chunk" + || id.starts_with("line ") + || id.starts_with("chunk ")) + { + let id = id.clone(); + let (marker_line, marker_column) = (next_token.line, next_token.column); + let is_line = id.starts_with("line"); + let marker = if is_line { "line" } else { "chunk" }; + let rest = id + .strip_prefix(marker) + .map(str::trim_start) + .unwrap_or("") + .to_string(); + self.bump_sync(); // Consume the (possibly merged) marker + + // Build the stream-write `value` and, for the ambiguous merged- + // identifier form (`write line to `), the classic + // file-write `fallback_content`. The merged token `line ` could + // equally be a variable literally named `line ` (WFL allows + // space-separated names), so we carry the file-write interpretation + // and let the interpreter pick based on whether `target` is a stream. + let (value, fallback_content) = if rest.is_empty() { + // Non-identifier values normally have only the stream reading. + // Direct integers are the exception because legacy WFL also + // supports `line 0` as direct indexing. + let direct_index = matches!( + self.cursor.peek().map(|token| &token.token), + Some(Token::IntLiteral(_)) + ); + if direct_index { + // A direct integer is ambiguous: stream value `0` vs legacy + // direct indexing on a variable literally named `line`. + let value_start = self.cursor.checkpoint(); + let value = self.parse_unmerged_operand(false)?; + let after_stream = self.cursor.checkpoint(); + + self.cursor.rewind(value_start); + let file_left = + Expression::Variable(marker.to_string(), marker_line, marker_column); + let fallback = self.parse_write_value_from_lead(file_left).ok(); + let fallback_end = self.cursor.checkpoint(); + let fallback = fallback.filter(|_| fallback_end == after_stream); + self.cursor.rewind(after_stream); + + (value, fallback.map(Box::new)) + } else { + (self.parse_unmerged_operand(false)?, None) + } + } else { + // Ambiguous merged form: `` alone (stream) vs the full + // merged `line ` (classic file write of that variable). + // Parse the two readings INDEPENDENTLY from the same continuation + // tokens via cursor rewind — NOT by deriving one AST from the + // other. A trailing `with`/operator continuation desugars + // differently per leading operand: a builtin name becomes an + // `ActionCall`, `is between` duplicates the left operand, + // `starts/ends with` and the pattern operators build calls — none + // of which survive a leftmost-leaf swap, which silently dropped or + // mangled the continuation for the classic file-write reading. + let value_start = self.cursor.checkpoint(); + + // Stream reading: split-off `` as the leading operand. + let stream_left = Expression::Variable(rest, marker_line, marker_column); + let value = self.parse_write_value_from_lead(stream_left)?; + let after_stream = self.cursor.checkpoint(); + + // Rewind and parse the classic file-write reading with the whole + // merged `line ` as the leading operand, over the very same + // tokens. This alternate interpretation is only USED at runtime + // when the target turns out to be a file, so it must not be + // REQUIRED to parse: a value whose stream reading uses grammar the + // classic reading can't (e.g. a builtin call with named arguments, + // `write line substring with text: "x" and start: 1 to out`) still + // has a valid stream reading. If the classic reading fails to + // parse, drop the fallback rather than failing the statement. + self.cursor.rewind(value_start); + let file_left = Expression::Variable(id, marker_line, marker_column); + let fallback = self.parse_write_value_from_lead(file_left).ok(); + // Only keep the classic fallback when it consumed EXACTLY the same + // continuation span as the stream reading. A fallback that parses + // a shorter (or longer) span is a different interpretation of the + // tokens — e.g. `write line min with a: 1 and b: 2 to `, + // where the stream reading is the builtin call `min` with named + // args but `line min with a` only parses up to the `:`. Retaining + // that partial parse and pairing it with the SAME trailing + // `to ` would silently corrupt a file write, so require the + // spans to match before trusting the fallback. + let fallback_end = self.cursor.checkpoint(); + let fallback = fallback.filter(|_| fallback_end == after_stream); + // Always resume right after the stream value, whatever the + // (speculative) fallback parse consumed, so `to ` follows. + self.cursor.rewind(after_stream); + + (value, fallback.map(Box::new)) + }; + + self.expect_token( + Token::KeywordTo, + "Expected 'to ' after the value in a 'write line'/'write chunk' statement", + )?; + let target = self.parse_primary_expression()?; + return Ok(Statement::StreamWriteStatement { + value, + target, + is_line, + fallback_content, + line: token_pos.line, + column: token_pos.column, + }); + } + // Check if next token is "binary" for "write binary X into Y" syntax if let Some(next_token) = self.cursor.peek() && matches!(&next_token.token, Token::KeywordBinary) diff --git a/src/parser/stmt/patterns.rs b/src/parser/stmt/patterns.rs index 33c15ab2..b070eb6d 100644 --- a/src/parser/stmt/patterns.rs +++ b/src/parser/stmt/patterns.rs @@ -80,7 +80,10 @@ pub(crate) trait PatternParser<'a>: ExprParser<'a> { i: &mut usize, base_pattern: PatternExpression, ) -> Result; - fn parse_extension_filter(&mut self) -> Result, ParseError>; + fn parse_extension_filter( + &mut self, + stop_at_clause: bool, + ) -> Result, ParseError>; } impl<'a> PatternParser<'a> for Parser<'a> { @@ -189,14 +192,17 @@ impl<'a> PatternParser<'a> for Parser<'a> { }) } - fn parse_extension_filter(&mut self) -> Result, ParseError> { + fn parse_extension_filter( + &mut self, + stop_at_clause: bool, + ) -> Result, ParseError> { // Expect "extension", "extensions", or "pattern" if let Some(token) = self.cursor.peek() { match &token.token { Token::KeywordExtension => { self.bump_sync(); // Consume "extension" // Parse single extension - let ext = self.parse_primary_expression()?; + let ext = self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; Ok(vec![ext]) } Token::KeywordExtensions => { @@ -210,7 +216,8 @@ impl<'a> PatternParser<'a> for Parser<'a> { if has_bracket { // Parse list literal - let list_expr = self.parse_primary_expression()?; + let list_expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; if let Expression::Literal(Literal::List(items), _, _) = list_expr { Ok(items) } else { @@ -223,14 +230,16 @@ impl<'a> PatternParser<'a> for Parser<'a> { } } else { // Allow a variable containing the extensions list - let expr = self.parse_primary_expression()?; + let expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; Ok(vec![expr]) } } Token::KeywordPattern => { self.bump_sync(); // Consume "pattern" // Parse pattern expression (e.g., "*.wfl") - let expr = self.parse_primary_expression()?; + let expr = + self.parse_primary_expression_with_clause_boundary(stop_at_clause)?; Ok(vec![expr]) } _ => { diff --git a/src/parser/stmt/processes.rs b/src/parser/stmt/processes.rs index ed5af44a..4f717d64 100644 --- a/src/parser/stmt/processes.rs +++ b/src/parser/stmt/processes.rs @@ -391,6 +391,67 @@ impl<'a> ProcessParser<'a> for Parser<'a> { column: wait_token_pos.column, }); } + // "wait for next chunk from as " and + // "wait for next line from as " pull the next + // piece of a streaming response body. The lexer glues the two + // identifiers, so `next chunk` / `next line` arrive as a single + // token; a bare `next` (followed by chunk/line) is also handled. + Token::Identifier(id) + if id == "next chunk" + || id == "next line" + || (id == "next" + && matches!( + self.cursor.peek_kind_n(1), + Some(Token::Identifier(kind)) if kind == "chunk" || kind == "line" + )) => + { + // A bare `next` NOT followed by `chunk`/`line` (e.g. a + // variable named `next` in `wait for next milliseconds`) + // falls through to the duration/expression handling below. + let is_line = id.ends_with("line"); + let is_bare_next = id == "next"; + self.bump_sync(); // Consume "next chunk"/"next line" (or bare "next") + + let is_line = if is_bare_next { + // The guard already confirmed `chunk`/`line` follows. + match self.cursor.peek().map(|t| &t.token) { + Some(Token::Identifier(kind)) if kind == "chunk" => { + self.bump_sync(); + false + } + _ => { + self.bump_sync(); + true + } + } + } else { + is_line + }; + + self.expect_token( + Token::KeywordFrom, + "Expected 'from' after 'next chunk'/'next line'", + )?; + let source = self.parse_primary_expression()?; + self.expect_token(Token::KeywordAs, "Expected 'as' after the stream handle")?; + let variable_name = self.parse_variable_name_simple()?; + + return Ok(if is_line { + Statement::WaitForNextLineStatement { + source, + variable_name, + line: wait_token_pos.line, + column: wait_token_pos.column, + } + } else { + Statement::WaitForNextChunkStatement { + source, + variable_name, + line: wait_token_pos.line, + column: wait_token_pos.column, + } + }); + } _ => { // Try to parse as "wait for X milliseconds/seconds" let checkpoint = self.cursor.checkpoint(); diff --git a/src/parser/stmt/web.rs b/src/parser/stmt/web.rs index 579645b3..6fab753f 100644 --- a/src/parser/stmt/web.rs +++ b/src/parser/stmt/web.rs @@ -10,6 +10,8 @@ pub(crate) trait WebParser<'a>: ExprParser<'a> + PrimaryExprParser<'a> { fn parse_listen_statement(&mut self) -> Result; fn parse_tls_path_value(&mut self, marker: &str) -> Result; fn parse_respond_statement(&mut self) -> Result; + fn parse_start_streaming_response(&mut self) -> Result; + fn parse_flush_stream(&mut self) -> Result; fn parse_register_signal_handler_statement(&mut self) -> Result; fn parse_stop_accepting_connections_statement(&mut self) -> Result; fn parse_close_server_statement(&mut self) -> Result; @@ -362,6 +364,225 @@ impl<'a> WebParser<'a> for Parser<'a> { }) } + fn parse_start_streaming_response(&mut self) -> Result { + // Consume the `start` keyword, then the `streaming` contextual identifier. + let start_token = self.bump_sync().unwrap(); + let (line, column) = (start_token.line, start_token.column); + + match self.cursor.peek() { + Some(t) => match &t.token { + Token::Identifier(id) if id == "streaming" => { + self.bump_sync(); // Consume "streaming" + } + _ => { + return Err(ParseError::from_token( + "Expected 'streaming' after 'start'".to_string(), + t, + )); + } + }, + None => { + return Err(ParseError::from_token( + "Expected 'streaming response' after 'start'".to_string(), + start_token, + )); + } + } + + self.expect_token( + Token::KeywordResponse, + "Expected 'response' after 'start streaming'", + )?; + self.expect_token( + Token::KeywordTo, + "Expected 'to ' after 'start streaming response'", + )?; + let request = self.parse_primary_expression()?; + + let mut status = None; + let mut content_type = None; + let mut headers = None; + + // Optional clauses joined by `with`/`and`, in any order: `status `, + // `content type `, `headers `. Mirrors the `respond` clause loop. + // A connective directly before `as` is the end-of-clauses join and is + // consumed so the `as ` binding parses; a connective before any + // other unrecognized token ends the loop WITHOUT being consumed, so the + // trailing `expect_token(as)` reports the malformed clause. + loop { + let connective = matches!( + self.cursor.peek(), + Some(t) if t.token == Token::KeywordWith || t.token == Token::KeywordAnd + ); + if !connective { + break; + } + let Some(next_token) = self.cursor.peek_next() else { + break; + }; + + match &next_token.token { + Token::KeywordStatus => { + self.bump_sync(); // with/and + self.bump_sync(); // status + status = Some(self.parse_unmerged_operand(true)?); + } + // `content type ` — `content` keyword then optional `type`. + // When `` is a bare identifier the lexer merges it into the + // `type` token (`type ct` -> Identifier("type ct")), so split the + // value off rather than binding the whole thing as the variable. + Token::KeywordContent => { + self.bump_sync(); // with/and + self.bump_sync(); // content + let merged_rest = if let Some(t) = self.cursor.peek() + && let Token::Identifier(id) = &t.token + && (id == "type" || id.starts_with("type ")) + { + let id = id.clone(); + let pos = (t.line, t.column); + self.bump_sync(); // (possibly merged) type + let rest = id.strip_prefix("type").map(str::trim_start).unwrap_or(""); + (!rest.is_empty()).then(|| (rest.to_string(), pos)) + } else { + None + }; + content_type = Some(match merged_rest { + Some((rest, (l, c))) => { + // Full expression continuation (postfix + `of` + + // operators), same as ordinary `` — e.g. + // `content type mime_type of path` (issue #642). + let lead = Expression::Variable(rest, l, c); + self.parse_clause_operand_from_lead(lead)? + } + None => self.parse_unmerged_operand(true)?, + }); + } + // Merged `content_type ` / `content type ` form. + Token::Identifier(id) + if id == "content_type" + || id.starts_with("content_type ") + || id.starts_with("content type") => + { + let id = id.clone(); + let (id_line, id_column) = (next_token.line, next_token.column); + self.bump_sync(); // with/and + self.bump_sync(); // merged marker + let rest = id + .strip_prefix("content_type") + .map(str::trim_start) + .unwrap_or_else(|| { + id.strip_prefix("content type") + .map(str::trim_start) + .unwrap_or("") + }); + if rest.is_empty() { + content_type = Some(self.parse_unmerged_operand(true)?); + } else { + let lead = Expression::Variable(rest.to_string(), id_line, id_column); + content_type = Some(self.parse_clause_operand_from_lead(lead)?); + } + } + // `headers ` (bare or merged `headers `). + Token::Identifier(id) if id == "headers" || id.starts_with("headers ") => { + let id = id.clone(); + let (id_line, id_column) = (next_token.line, next_token.column); + self.bump_sync(); // with/and + self.bump_sync(); // merged marker + let rest = id + .strip_prefix("headers") + .map(str::trim_start) + .unwrap_or(""); + if rest.is_empty() { + headers = Some(self.parse_unmerged_operand(true)?); + } else { + // Clause operand: postfix/`of`/operators but stop before + // the next clause connective (`and content type`, `as`). + let lead = Expression::Variable(rest.to_string(), id_line, id_column); + headers = Some(self.parse_clause_operand_from_lead(lead)?); + } + } + // A connective directly before `as` just joins the clause list to + // the binding; consume it so `as ` parses cleanly instead of + // `expect_token(as)` tripping over the leftover `and`/`with`. + Token::KeywordAs => { + self.bump_sync(); // consume the connective; `as` stays next + break; + } + _ => break, + } + } + + self.expect_token( + Token::KeywordAs, + "Expected 'as ' after 'start streaming response ...'", + )?; + let variable_name = self.parse_variable_name_simple()?; + + Ok(Statement::StartStreamingResponseStatement { + request, + status, + content_type, + headers, + variable_name, + line, + column, + }) + } + + fn parse_flush_stream(&mut self) -> Result { + // `flush ` — the lexer merges a bare-identifier target into the + // command token (`flush out` -> Identifier("flush out")). + let token = self.bump_sync().unwrap(); + let (line, column) = (token.line, token.column); + let phrase = match &token.token { + Token::Identifier(id) => id.clone(), + _ => { + return Err(ParseError::from_token( + "Expected 'flush '".to_string(), + token, + )); + } + }; + let rest = phrase + .strip_prefix("flush") + .map(str::trim_start) + .unwrap_or(""); + let (target, legacy_binding, action_fallback) = if rest.is_empty() { + // Exact `flush` followed by an unmerged target is dispatched only for + // the unambiguous streaming starters `(` and `call`. Before streaming, + // however, a defined zero-argument action named exactly `flush` still + // auto-ran and the remaining same-line expression did not turn that + // action into a stream operation. Preserve that binding as the same + // action fallback used by the merged form. + ( + self.parse_unmerged_operand(false)?, + Some(phrase.clone()), + Some(Expression::Variable(phrase.clone(), line, column)), + ) + } else { + // Stream reading: postfix on the split-off rest (`cache` from + // `flush cache`). Legacy expression: same postfix on the FULL phrase + // (`flush cache`) so `flush cache[0]` / `.property` / `.method()` / + // `at` keep their old expression-statement AST (issue #642 re-review). + let cp = self.cursor.checkpoint(); + let stream_lead = Expression::Variable(rest.to_string(), line, column); + let target = self.parse_seeded_expression_continuation(stream_lead, false)?; + self.cursor.rewind(cp); + let legacy_binding = phrase.clone(); + let legacy_lead = Expression::Variable(legacy_binding.clone(), line, column); + let fallback = self.parse_seeded_expression_continuation(legacy_lead, false)?; + (target, Some(legacy_binding), Some(fallback)) + }; + + Ok(Statement::FlushStreamStatement { + target, + legacy_binding, + action_fallback, + line, + column, + }) + } + fn parse_register_signal_handler_statement(&mut self) -> Result { let register_token = self.bump_sync().unwrap(); // Consume "register" diff --git a/src/transpiler/javascript.rs b/src/transpiler/javascript.rs index 79e9ba9e..b2077389 100644 --- a/src/transpiler/javascript.rs +++ b/src/transpiler/javascript.rs @@ -351,7 +351,22 @@ impl JavaScriptTranspiler { Ok(result) } - Statement::MainLoop { body, .. } => { + Statement::MainLoop { + body, + concurrent, + line, + column, + } => { + // `main loop concurrently:` has cooperative-concurrency semantics + // the serial `while (true)` translation cannot express. Fail + // rather than silently emit a serial loop. + if *concurrent { + return Err(TranspileError { + message: "`main loop concurrently:` is not supported in JavaScript transpilation (its cooperative concurrency semantics require the WFL interpreter).".to_string(), + line: *line, + column: *column, + }); + } // Main loop is essentially a forever loop let mut result = format!("{}while (true) {{\n", self.indent()); self.push_indent(); @@ -620,6 +635,45 @@ impl JavaScriptTranspiler { }) } + Statement::StreamWriteStatement { + target, + fallback_content: Some(fallback_content), + .. + } => { + // The parser preserves both readings of ambiguous classic syntax + // such as `write line note to "f.txt"`. JavaScript has no response + // stream implementation, but it can still emit the pre-streaming + // file-write reading exactly as it did before. + let content_expr = self.transpile_expression(fallback_content)?; + let file_expr = self.transpile_expression(target)?; + Ok(format!( + "{}WFL.file.write({}.path, {});\n", + self.indent(), + file_expr, + content_expr + )) + } + + Statement::HttpStreamStatement { line, column, .. } + | Statement::WaitForNextChunkStatement { line, column, .. } + | Statement::WaitForNextLineStatement { line, column, .. } + | Statement::StartStreamingResponseStatement { line, column, .. } + | Statement::StreamWriteStatement { + line, + column, + fallback_content: None, + .. + } + | Statement::FlushStreamStatement { line, column, .. } => { + // Streaming HTTP relies on the interpreter's parked-stream + // handles; emitting broken JS would be worse than a clear error. + Err(TranspileError { + message: "Streaming HTTP statements are not supported in JavaScript transpilation. They require the WFL interpreter.".to_string(), + line: *line, + column: *column, + }) + } + Statement::WriteContentStatement { content, target, .. } => { @@ -2055,6 +2109,12 @@ impl JavaScriptTranspiler { | Statement::HttpGetStatement { .. } | Statement::HttpPostStatement { .. } | Statement::HttpRequestStatement { .. } + | Statement::HttpStreamStatement { .. } + | Statement::WaitForNextChunkStatement { .. } + | Statement::WaitForNextLineStatement { .. } + | Statement::StartStreamingResponseStatement { .. } + | Statement::StreamWriteStatement { .. } + | Statement::FlushStreamStatement { .. } | Statement::WaitForProcessStatement { .. } | Statement::WaitForRequestStatement { .. } => true, diff --git a/src/typechecker/mod.rs b/src/typechecker/mod.rs index 10b8690c..847a6a0f 100644 --- a/src/typechecker/mod.rs +++ b/src/typechecker/mod.rs @@ -197,6 +197,92 @@ impl TypeChecker { self.analyzer.get_action_parameters() } + fn join_type_snapshots( + states: &[Vec>>], + ) -> Vec>> { + let Some(first) = states.first() else { + return Vec::new(); + }; + let mut joined = first.clone(); + + // Analyzer pass 1 normally pre-registers branch-visible symbols, but + // include names from every state so direct AST users receive the same + // conservative result. + for state in states.iter().skip(1) { + if joined.len() < state.len() { + joined.resize_with(state.len(), HashMap::new); + } + for (layer_index, layer) in state.iter().enumerate() { + for name in layer.keys() { + joined[layer_index].entry(name.clone()).or_insert(None); + } + } + } + + for (layer_index, joined_layer) in joined.iter_mut().enumerate() { + let names: Vec = joined_layer.keys().cloned().collect(); + for name in names { + let values: Vec> = states + .iter() + .map(|state| { + state + .get(layer_index) + .and_then(|layer| layer.get(&name)) + .cloned() + .unwrap_or(None) + }) + .collect(); + let first_value = values.first().cloned().unwrap_or(None); + let merged = if values.iter().all(|value| value == &first_value) { + first_value + } else if values + .iter() + .any(|value| value.is_none() || matches!(value.as_ref(), Some(Type::Unknown))) + { + Some(Type::Unknown) + } else { + Some(Type::Any) + }; + joined_layer.insert(name, merged); + } + } + + joined + } + + /// Check a loop body under the conservative type state seen at the top of + /// every iteration. Exploratory passes contribute only their backedge + /// state; diagnostics are emitted once after the header stabilizes. + fn check_loop_body_fixed_point(&mut self, body: &[Statement]) { + let entry = self.analyzer.snapshot_symbol_types(); + let mut header = entry.clone(); + + loop { + self.analyzer.restore_symbol_types(header.clone()); + let error_count = self.errors.len(); + for statement in body { + self.check_statement_types(statement); + } + if self.budget_error.is_some() { + return; + } + self.errors.truncate(error_count); + + let backedge = self.analyzer.snapshot_symbol_types(); + let next = Self::join_type_snapshots(&[entry.clone(), header.clone(), backedge]); + if next == header { + break; + } + header = next; + } + + self.analyzer.restore_symbol_types(header.clone()); + for statement in body { + self.check_statement_types(statement); + } + self.analyzer.restore_symbol_types(header); + } + /// Get the return type for builtin functions fn get_builtin_function_type(&self, name: &str, _arg_count: usize) -> Type { match name { @@ -635,6 +721,16 @@ impl TypeChecker { line: _line, column: _column, } => { + // Runtime keeps one child environment alive for every + // iteration, so bindings from a backedge are visible at the + // next header but remain local after the loop. + self.analyzer.push_scope(); + self.check_loop_body_fixed_point(body); + if self.budget_error.is_some() { + self.analyzer.pop_scope(); + return; + } + let condition_type = self.infer_expression_type(condition); if condition_type != Type::Boolean && condition_type != Type::Unknown { self.errors.push(TypeError::new( @@ -647,10 +743,7 @@ impl TypeChecker { *_column, )); } - - for stmt in body { - self.check_statement_types(stmt); - } + self.analyzer.pop_scope(); } Statement::ExitStatement { line: _, column: _ } => {} Statement::WaitForStatement { @@ -688,9 +781,30 @@ impl TypeChecker { line: _line, column: _column, } => { + // Runtime evaluates the try body, handlers, otherwise, and + // finally block inside one shared child environment. + self.analyzer.push_scope(); + let entry_types = self.analyzer.snapshot_symbol_types(); for stmt in body { self.check_statement_types(stmt); } + if self.budget_error.is_some() { + self.analyzer.pop_scope(); + return; + } + let success_endpoint = self.analyzer.snapshot_symbol_types(); + + // An error can leave the body from any statement, so handlers + // start from the conservative entry/success join. Keep the + // success scope's symbol set as the structural baseline: + // success-only bindings remain resolvable as gradual types, + // while exact restoration prevents one handler's new symbols + // from contaminating the next handler. + let handler_entry = + Self::join_type_snapshots(&[entry_types, success_endpoint.clone()]); + let handler_scope_symbols = self.analyzer.snapshot_current_scope_symbols(); + let mut joined_scope_symbols = handler_scope_symbols.clone(); + let mut endpoints = vec![success_endpoint]; // Type check each when clause in its own scope so the bound // error name cannot clobber an outer variable of the same @@ -699,6 +813,9 @@ impl TypeChecker { // the binding lives only in the child scope (runtime does the // same via Environment::define_or_replace). for when_clause in when_clauses { + self.analyzer + .restore_current_scope_symbols(handler_scope_symbols.clone()); + self.analyzer.restore_symbol_types(handler_entry.clone()); self.analyzer.push_scope(); self.analyzer.define_or_replace_symbol(Symbol { name: when_clause.error_name.clone(), @@ -722,20 +839,64 @@ impl TypeChecker { for stmt in &when_clause.body { self.check_statement_types(stmt); } - self.analyzer.pop_scope(); + let mut excluded_aliases = vec![when_clause.error_name.clone()]; + if when_clause.error_name != "error_message" { + excluded_aliases.push("error_message".to_string()); + } + self.analyzer.pop_scope_promoting_except(&excluded_aliases); + + if self.budget_error.is_some() { + self.analyzer.pop_scope(); + return; + } + + endpoints.push(self.analyzer.snapshot_symbol_types()); + for (name, symbol) in self.analyzer.snapshot_current_scope_symbols() { + joined_scope_symbols.entry(name).or_insert(symbol); + } } if let Some(otherwise_stmts) = otherwise_block { + self.analyzer + .restore_current_scope_symbols(handler_scope_symbols.clone()); + self.analyzer.restore_symbol_types(handler_entry.clone()); for stmt in otherwise_stmts { self.check_statement_types(stmt); } + if self.budget_error.is_some() { + self.analyzer.pop_scope(); + return; + } + + endpoints.push(self.analyzer.snapshot_symbol_types()); + for (name, symbol) in self.analyzer.snapshot_current_scope_symbols() { + joined_scope_symbols.entry(name).or_insert(symbol); + } + } else if !when_clauses.iter().any(|when_clause| { + matches!( + &when_clause.error_type, + crate::parser::ast::ErrorType::General + ) + }) { + // A non-matching error reaches finally without running a + // handler when there is no catch-all or otherwise block. + endpoints.push(handler_entry.clone()); } + self.analyzer + .restore_current_scope_symbols(handler_scope_symbols); + for symbol in joined_scope_symbols.into_values() { + self.analyzer.define_or_replace_symbol(symbol); + } + let joined_endpoint = Self::join_type_snapshots(&endpoints); + self.analyzer.restore_symbol_types(joined_endpoint); + if let Some(finally_stmts) = finally_block { for stmt in finally_stmts { self.check_statement_types(stmt); } } + self.analyzer.pop_scope(); } Statement::HttpGetStatement { url, @@ -824,13 +985,10 @@ impl TypeChecker { } if let Some(headers) = headers { let headers_type = self.infer_expression_type(headers); - if !matches!( - headers_type, - Type::Map(_, _) | Type::Unknown | Type::Any | Type::Error - ) { + if !self.is_valid_header_map_type(&headers_type) { self.type_error( "HTTP headers must be a map of header names to values".to_string(), - Some(Type::Map(Box::new(Type::Text), Box::new(Type::Text))), + Some(Type::Map(Box::new(Type::Text), Box::new(Type::Any))), Some(headers_type), *_line, *_column, @@ -851,8 +1009,11 @@ impl TypeChecker { | Type::Error ) { self.type_error( - "HTTP request body must be text".to_string(), - Some(Type::Text), + "HTTP request body must be text, a number, or a boolean (numbers and booleans are converted to text)".to_string(), + // No single "expected" type — the accepted set is + // Text|Number|Boolean, so a bare `Text` hint would + // misrender the expected-vs-actual diagnostic. + None, Some(body_type), *_line, *_column, @@ -870,6 +1031,291 @@ impl TypeChecker { }); } } + Statement::HttpStreamStatement { + url, + method, + headers, + body, + variable_name, + line: _line, + column: _column, + } => { + let url_type = self.infer_expression_type(url); + if url_type != Type::Text && url_type != Type::Unknown && url_type != Type::Error { + self.type_error( + "URL must be a text string".to_string(), + Some(Type::Text), + Some(url_type), + *_line, + *_column, + ); + } + if let Some(method) = method { + let method_type = self.infer_expression_type(method); + if method_type != Type::Text + && method_type != Type::Unknown + && method_type != Type::Error + { + self.type_error( + "HTTP method must be a text string".to_string(), + Some(Type::Text), + Some(method_type), + *_line, + *_column, + ); + } + } + if let Some(headers) = headers { + let headers_type = self.infer_expression_type(headers); + if !self.is_valid_header_map_type(&headers_type) { + self.type_error( + "HTTP headers must be a map of header names to values".to_string(), + Some(Type::Map(Box::new(Type::Text), Box::new(Type::Any))), + Some(headers_type), + *_line, + *_column, + ); + } + } + if let Some(body) = body { + let body_type = self.infer_expression_type(body); + if !matches!( + body_type, + Type::Text + | Type::Number + | Type::Boolean + | Type::Unknown + | Type::Any + | Type::Error + ) { + self.type_error( + "HTTP request body must be text, a number, or a boolean (numbers and booleans are converted to text)".to_string(), + // No single "expected" type — the accepted set is + // Text|Number|Boolean, so a bare `Text` hint would + // misrender the expected-vs-actual diagnostic. + None, + Some(body_type), + *_line, + *_column, + ); + } + } + + // Binds an outbound streaming-response handle (exposes + // status/ok/headers via index/member access, and is closeable). + // A distinct handle type — not a bare `Map` — so `close` accepts + // it without also accepting an ordinary user map. + if !variable_name.is_empty() + && let Some(symbol) = self.analyzer.get_symbol_mut(variable_name) + { + symbol.symbol_type = Some(Type::Custom("HttpStream".to_string())); + } + } + Statement::WaitForNextChunkStatement { + source, + variable_name, + line, + column, + } + | Statement::WaitForNextLineStatement { + source, + variable_name, + line, + column, + } => { + // The source must be an outbound stream handle. Gradual types + // (Unknown/Any/Error) pass; a concrete non-stream operand is a + // static error rather than a runtime "not a stream" surprise. + let source_type = self.infer_expression_type(source); + if !self.is_http_stream_source_type(&source_type) { + self.type_error( + "`wait for next chunk|line` requires an outbound stream handle \ + (from `open url ... and stream response as ...`)" + .to_string(), + Some(Type::Custom("HttpStream".to_string())), + Some(source_type), + *line, + *column, + ); + } + // The binding may be a chunk/line value or `nothing` at end of + // stream, so leave the bound variable's type open (Any) to avoid + // false errors on the `check if is nothing` termination. + if !variable_name.is_empty() + && let Some(symbol) = self.analyzer.get_symbol_mut(variable_name) + { + symbol.symbol_type = Some(Type::Any); + } + } + Statement::StartStreamingResponseStatement { + request, + status, + content_type, + headers, + variable_name, + line: _line, + column: _column, + } => { + let _ = self.infer_expression_type(request); + // Enforce the clause types (like RespondStatement) so obvious + // mistakes fail at typecheck rather than at runtime. + if let Some(status) = status { + let status_type = self.infer_expression_type(status); + if !matches!( + status_type, + Type::Number | Type::Unknown | Type::Any | Type::Error + ) { + self.type_error( + "Streaming response status must be a number".to_string(), + Some(Type::Number), + Some(status_type), + *_line, + *_column, + ); + } + } + if let Some(content_type) = content_type { + let ct_type = self.infer_expression_type(content_type); + if !matches!( + ct_type, + Type::Text | Type::Unknown | Type::Any | Type::Error + ) { + self.type_error( + "Streaming response content type must be text".to_string(), + Some(Type::Text), + Some(ct_type), + *_line, + *_column, + ); + } + } + if let Some(headers) = headers { + let headers_type = self.infer_expression_type(headers); + if !self.is_valid_header_map_type(&headers_type) { + self.type_error( + "Streaming response headers must be a map of header names to values" + .to_string(), + Some(Type::Map(Box::new(Type::Text), Box::new(Type::Any))), + Some(headers_type), + *_line, + *_column, + ); + } + } + if !variable_name.is_empty() { + // A distinct server-response-stream handle type (not a bare + // `Map`) so `close out` is accepted without `close` also + // type-checking an ordinary user map. + // + // Always bind in the *current* scope only (shadow, do not + // mutate an outer symbol of the same name via get_symbol_mut + // parent walk). Analyzer loop scopes are discarded after + // body analysis, so we re-create the binding here. + self.analyzer.define_or_replace_symbol(Symbol { + name: variable_name.clone(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: Some(Type::Custom("ResponseStream".to_string())), + line: *_line, + column: *_column, + }); + } + } + Statement::StreamWriteStatement { + value, + target, + fallback_content, + line, + column, + .. + } => { + // Branch-aware: the runtime picks the reading by the TARGET's type + // (a response-stream handle -> stream write of `value`; anything + // else with a fallback -> classic file write of `fallback_content`). + // Type-check the reading the runtime will actually take, so a valid + // pre-existing file write is never rejected on the stream branch it + // never runs (and vice versa). + let target_type = self.infer_expression_type(target); + let has_fallback = fallback_content.is_some(); + + if self.is_response_stream_target_type(&target_type) + && !self.is_gradual_type(&target_type) + { + // Concrete response stream: the stream reading is taken. + // Report undefined names on this branch (analyzer may have + // stayed silent because the classic lead alone was defined). + self.check_expression_names_defined(value); + let value_type = self.infer_expression_type(value); + self.check_streamable_payload(&value_type, *line, *column); + } else if has_fallback + && (matches!(target_type, Type::Text) + || matches!(&target_type, Type::Custom(n) if n == "File")) + { + // Concrete text path OR open-file handle (`Custom("File")`): + // the classic file-write reading is taken. Validate the + // fallback (including definedness), not the stream `value`. + if let Some(fallback) = fallback_content { + self.check_expression_names_defined(fallback); + let _ = self.infer_expression_type(fallback); + } + } else if self.is_gradual_type(&target_type) { + // Gradual/unknown target: both readings are viable and the + // runtime decides by the target's runtime type. Conservatively + // validate EVERY viable branch (not "accept if either is ok"), + // so a valid file fallback cannot mask an invalid stream + // payload or an undefined stream lead (issue #642). + self.check_expression_names_defined(value); + let value_type = self.infer_expression_type(value); + self.check_streamable_payload(&value_type, *line, *column); + if let Some(fallback) = fallback_content { + self.check_expression_names_defined(fallback); + let _ = self.infer_expression_type(fallback); + } + } else { + // Concrete non-stream, non-text target (or a text target with no + // fallback): an unambiguous stream write to the wrong type. + self.type_error( + "`write line|chunk` requires a response-stream handle \ + (from `start streaming response ... as ...`)" + .to_string(), + Some(Type::Custom("ResponseStream".to_string())), + Some(target_type), + *line, + *column, + ); + } + } + Statement::FlushStreamStatement { + target, + legacy_binding, + action_fallback, + line, + column, + } => { + // Legacy full-name expression (e.g. Variable("flush cache") or + // IndexAccess over it): when its root is bound, typecheck that + // expression. Otherwise this is a stream flush. + let is_expression_fallback = legacy_binding + .as_deref() + .is_some_and(|name| self.name_is_defined_for_write(name)); + if is_expression_fallback { + if let Some(fb) = action_fallback { + let _ = self.infer_expression_type(fb); + } + } else { + let target_type = self.infer_expression_type(target); + if !self.is_response_stream_target_type(&target_type) { + self.type_error( + "`flush` requires a response-stream handle \ + (from `start streaming response ... as ...`)" + .to_string(), + Some(Type::Custom("ResponseStream".to_string())), + Some(target_type), + *line, + *column, + ); + } + } + } Statement::VariableDeclaration { name, value, @@ -1137,15 +1583,23 @@ impl TypeChecker { ); } + let entry_types = self.analyzer.snapshot_symbol_types(); for stmt in then_block { self.check_statement_types(stmt); } + let then_types = self.analyzer.snapshot_symbol_types(); + self.analyzer.restore_symbol_types(entry_types.clone()); - if let Some(else_stmts) = else_block { + let else_types = if let Some(else_stmts) = else_block { for stmt in else_stmts { self.check_statement_types(stmt); } - } + self.analyzer.snapshot_symbol_types() + } else { + entry_types + }; + let joined = Self::join_type_snapshots(&[then_types, else_types]); + self.analyzer.restore_symbol_types(joined); } Statement::SingleLineIf { condition, @@ -1168,11 +1622,19 @@ impl TypeChecker { ); } + let entry_types = self.analyzer.snapshot_symbol_types(); self.check_statement_types(then_stmt); + let then_types = self.analyzer.snapshot_symbol_types(); + self.analyzer.restore_symbol_types(entry_types.clone()); - if let Some(else_stmt) = else_stmt { + let else_types = if let Some(else_stmt) = else_stmt { self.check_statement_types(else_stmt); - } + self.analyzer.snapshot_symbol_types() + } else { + entry_types + }; + let joined = Self::join_type_snapshots(&[then_types, else_types]); + self.analyzer.restore_symbol_types(joined); } Statement::ForEachLoop { item_name, @@ -1233,6 +1695,7 @@ impl TypeChecker { body, line: _line, column: _column, + variable_name, .. } => { let start_type = self.infer_expression_type(start); @@ -1277,14 +1740,22 @@ impl TypeChecker { } } - // Register the "count" variable with type Number - if let Some(symbol) = self.analyzer.get_symbol_mut("count") { - symbol.symbol_type = Some(Type::Number); - } + // Runtime creates the loop variable in a child environment, + // shadowing rather than retyping an outer `count` or custom + // loop-variable binding. + self.analyzer.push_scope(); + self.analyzer.define_or_replace_symbol(Symbol { + name: variable_name.as_deref().unwrap_or("count").to_string(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: Some(Type::Number), + line: *_line, + column: *_column, + }); for stmt in body { self.check_statement_types(stmt); } + self.analyzer.pop_scope(); } Statement::WhileLoop { condition, @@ -1292,6 +1763,11 @@ impl TypeChecker { line: _line, column: _column, } => { + self.check_loop_body_fixed_point(body); + if self.budget_error.is_some() { + return; + } + let condition_type = self.infer_expression_type(condition); if condition_type != Type::Boolean && condition_type != Type::Unknown @@ -1305,10 +1781,6 @@ impl TypeChecker { *_column, ); } - - for stmt in body { - self.check_statement_types(stmt); - } } Statement::RepeatUntilLoop { condition, @@ -1335,14 +1807,22 @@ impl TypeChecker { } } Statement::ForeverLoop { body, .. } => { + // Push a scope so bindings introduced in the body (e.g. + // `start streaming response ... as out`) remain visible to later + // statements in the same body for type checking. Analyzer loop + // scopes are discarded after analysis. + self.analyzer.push_scope(); for stmt in body { self.check_statement_types(stmt); } + self.analyzer.pop_scope(); } Statement::MainLoop { body, .. } => { + self.analyzer.push_scope(); for stmt in body { self.check_statement_types(stmt); } + self.analyzer.pop_scope(); } Statement::DisplayStatement { value, .. } => { self.infer_expression_type(value); @@ -1379,9 +1859,17 @@ impl TypeChecker { ); } - if let Some(symbol) = self.analyzer.get_symbol_mut(variable_name) { - symbol.symbol_type = Some(Type::Custom("File".to_string())); - } + // Runtime binds the opened handle in the current environment. + // Analyzer body scopes are discarded before this pass, so + // recreate the local symbol here and shadow (rather than + // parent-walk/retype) any outer binding with the same name. + self.analyzer.define_or_replace_symbol(Symbol { + name: variable_name.clone(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: Some(Type::Custom("File".to_string())), + line: *_line, + column: *_column, + }); } Statement::ReadFileStatement { path, @@ -1448,13 +1936,13 @@ impl TypeChecker { column: _column, } => { let file_type = self.infer_expression_type(file); - if file_type != Type::Custom("File".to_string()) - && file_type != Type::Unknown - && file_type != Type::Error - { + if !self.is_closeable_type(&file_type) { + // No single `expected` type: `close` accepts a File *or* a + // (map-shaped) stream handle, so pinning the hint to `File` + // would mis-render the expected-vs-found diagnostic. self.type_error( - "Expected a File object".to_string(), - Some(Type::Custom("File".to_string())), + "Expected a file or stream handle".to_string(), + None, Some(file_type), *_line, *_column, @@ -2187,9 +2675,13 @@ impl TypeChecker { line: _line, column: _column, } => { + self.analyzer.push_scope(); + let outer_type_snapshot = self.analyzer.snapshot_symbol_types(); for stmt in handler_body { self.check_statement_types(stmt); } + self.analyzer.restore_symbol_types(outer_type_snapshot); + self.analyzer.pop_scope(); } Statement::ParentMethodCall { method_name: _method_name, @@ -2390,13 +2882,10 @@ impl TypeChecker { // outbound HttpRequestStatement headers check. if let Some(headers_expr) = headers { let headers_type = self.infer_expression_type(headers_expr); - if !matches!( - headers_type, - Type::Map(_, _) | Type::Unknown | Type::Any | Type::Error - ) { + if !self.is_valid_header_map_type(&headers_type) { self.type_error( "Response headers must be a map of header names to values".to_string(), - Some(Type::Map(Box::new(Type::Text), Box::new(Type::Text))), + Some(Type::Map(Box::new(Type::Text), Box::new(Type::Any))), Some(headers_type), *_line, *_column, @@ -2456,9 +2945,13 @@ impl TypeChecker { // bound variable resolves as an object at runtime (gradual typing // keeps member access like `body of msg` permissive). self.check_server_expression_type(server, *line, *column); + self.analyzer.push_scope(); + let outer_type_snapshot = self.analyzer.snapshot_symbol_types(); for stmt in body { self.check_statement_types(stmt); } + self.analyzer.restore_symbol_types(outer_type_snapshot); + self.analyzer.pop_scope(); } Statement::SendWebSocketMessageStatement { message, target, .. @@ -2827,6 +3320,8 @@ impl TypeChecker { // where a concrete type is required at runtime. Type::Unknown } + } else if let Some(property_type) = self.current_container_property_type(name) { + property_type } else { // Check if this is an action parameter, builtin function, or special function name before reporting it as undefined if self.analyzer.get_action_parameters().contains(name) @@ -3367,6 +3862,27 @@ impl TypeChecker { // result) is indexable; the element type is only known // at runtime (issue #553). Type::Any => Type::Any, + // Stream handles expose fields (`status`/`ok`/`headers`) by + // index; the key must be text (runtime object indexing rejects + // a numeric key), and the field type is only known at runtime. + Type::Custom(ref name) if name == "HttpStream" || name == "ResponseStream" => { + if index_type != Type::Text + && index_type != Type::Unknown + && index_type != Type::Any + && index_type != Type::Error + { + self.type_error( + format!("Stream handle field name must be text, got {index_type}"), + Some(Type::Text), + Some(index_type), + *line, + *column, + ); + Type::Error + } else { + Type::Any + } + } _ => { self.type_error( format!("Cannot index into {collection_type}"), @@ -4042,6 +4558,12 @@ impl TypeChecker { // the value type is whatever the map stores. Type::Map(_, value_type) => *value_type, Type::Unknown | Type::Any | Type::Error => Type::Unknown, + // Stream handles expose fields (`status`/`ok`/`headers`) via + // the documented dot form too; the field type is only known at + // runtime. + Type::Custom(ref name) if name == "HttpStream" || name == "ResponseStream" => { + Type::Unknown + } _ => { self.type_error( format!( @@ -4495,6 +5017,303 @@ impl TypeChecker { .push(TypeError::new(message, expected, found, line, column)); } + /// Whether an inferred type is acceptable as an HTTP header map. Header names + /// must be text, and header values are what the interpreter accepts and + /// stringifies — text, number, or boolean (see the `respond`/HTTP header + /// handling); a concretely-typed non-text key or a value type the runtime + /// rejects (e.g. `Map`) is flagged. `Unknown`/`Any`/`Error` — + /// whether as the whole type or as the key/value type of a map the checker + /// could not fully pin down (map literals often infer unknown key/value + /// types) — are always accepted so a header set the checker cannot resolve is + /// never falsely flagged. + fn is_valid_header_map_type(&self, ty: &Type) -> bool { + match ty { + Type::Unknown | Type::Any | Type::Error => true, + Type::Map(key, value) => { + let key_ok = matches!(**key, Type::Text | Type::Unknown | Type::Any | Type::Error); + let value_ok = matches!( + **value, + Type::Text + | Type::Number + | Type::Boolean + | Type::Unknown + | Type::Any + | Type::Error + ); + key_ok && value_ok + } + _ => false, + } + } + + /// Whether a type can name a closeable resource: a file handle + /// (`Custom("File")`), a stream handle (`Custom("HttpStream")` outbound or + /// `Custom("ResponseStream")` server-side), or a statically-unresolved value. + /// These are the only things the runtime can close, so an ordinary map, + /// another custom type (`Database`/`Request`), or a scalar (`close 5`) is + /// rejected — keeping real mistakes as static errors rather than runtime-only. + fn is_closeable_type(&self, ty: &Type) -> bool { + match ty { + Type::Custom(name) => { + name == "File" || name == "HttpStream" || name == "ResponseStream" + } + Type::Unknown | Type::Any | Type::Error => true, + _ => false, + } + } + + /// The operand of `wait for next chunk|line from ` must be an outbound + /// stream handle (`stream response as ...` binds `HttpStream`). Unknown/Any/ + /// Error pass for gradual typing; a concrete non-stream type is rejected. + fn is_http_stream_source_type(&self, ty: &Type) -> bool { + match ty { + Type::Custom(name) => name == "HttpStream", + Type::Unknown | Type::Any | Type::Error => true, + _ => false, + } + } + + /// The `` of `write line|chunk` / `flush` must be a server response + /// stream handle (`start streaming response as ...` binds `ResponseStream`). + /// Unknown/Any/Error pass for gradual typing. + fn is_response_stream_target_type(&self, ty: &Type) -> bool { + match ty { + Type::Custom(name) => name == "ResponseStream", + Type::Unknown | Type::Any | Type::Error => true, + _ => false, + } + } + + /// A type that inference has not pinned down (gradual typing): it may turn out + /// to be anything at runtime, so a static check must stay lenient rather than + /// reject it. + fn is_gradual_type(&self, ty: &Type) -> bool { + matches!(ty, Type::Unknown | Type::Any | Type::Error) + } + + /// The value types `write line|chunk` can send to a response stream — the + /// runtime stringifies numbers/booleans and sends text/binary as-is, and + /// rejects everything else (Map/List/Nothing/...). Gradual types pass. + fn is_streamable_payload(&self, ty: &Type) -> bool { + matches!(ty, Type::Text | Type::Number | Type::Boolean | Type::Binary) + || self.is_gradual_type(ty) + } + + /// Emit a type error if `ty` is a concrete value the runtime would reject as a + /// response-stream payload (a Map/List/Nothing/... reaches `write` only to fail + /// at runtime otherwise). + fn check_streamable_payload(&mut self, ty: &Type, line: usize, column: usize) { + if !self.is_streamable_payload(ty) { + self.type_error( + "`write line|chunk` can only send text, binary, a number, or a boolean \ + to a response stream" + .to_string(), + Some(Type::Text), + Some(ty.clone()), + line, + column, + ); + } + } + + /// Whether a bare name is known to the typechecker/analyzer scopes (or is a + /// builtin / action parameter / loop counter). Used when validating the + /// concrete `write line|chunk` branch the runtime will select — the analyzer + /// may have stayed silent on a one-sided undefined lead because the other + /// reading was defined (issue #642). + fn name_is_defined_for_write(&self, name: &str) -> bool { + if self.analyzer.name_is_defined_for_write(name) { + return true; + } + + self.current_container_property_type(name).is_some() + } + + /// Resolve a direct or inherited property against the typechecker's live + /// container context. The analyzer has restored its own container context + /// by the time method bodies are typechecked, so both definedness and type + /// inference must use this view. + fn current_container_property_type(&self, name: &str) -> Option { + // Analyzer has already completed its container walk and restored its + // own `current_container` by the time TypeChecker revisits method + // bodies. Use TypeChecker's live container context here so direct and + // inherited properties remain defined on the selected write branch. + let mut container_name = self.current_container.as_deref(); + while let Some(container_key) = container_name { + let Some(container) = self.analyzer.get_container(container_key) else { + break; + }; + if let Some(property) = container + .properties + .get(name) + .or_else(|| container.static_properties.get(name)) + { + return Some(property.property_type.clone()); + } + container_name = container.extends.as_deref(); + } + + None + } + + /// Walk an expression and report every undefined bare name. Used for the + /// selected (or every viable gradual) `write line|chunk` branch so a missing + /// classic `line ` lead is not accepted just because the stream lead + /// alone exists (and vice versa). + fn check_expression_names_defined(&mut self, expression: &Expression) { + match expression { + Expression::Literal(Literal::List(items), ..) => { + for item in items { + self.check_expression_names_defined(item); + } + } + Expression::Literal(_, _, _) + | Expression::StaticMemberAccess { .. } + | Expression::CurrentTimeMilliseconds { .. } + | Expression::CurrentTimeFormatted { .. } => {} + Expression::Variable(name, l, c) => { + if !self.name_is_defined_for_write(name) { + self.type_error( + format!("Variable '{name}' is not defined"), + None, + None, + *l, + *c, + ); + } + } + Expression::BinaryOperation { left, right, .. } + | Expression::Concatenation { left, right, .. } + | Expression::PatternMatch { + text: left, + pattern: right, + .. + } + | Expression::PatternFind { + text: left, + pattern: right, + .. + } + | Expression::PatternSplit { + text: left, + pattern: right, + .. + } + | Expression::StringSplit { + text: left, + delimiter: right, + .. + } => { + self.check_expression_names_defined(left); + self.check_expression_names_defined(right); + } + Expression::UnaryOperation { + expression: inner, .. + } + | Expression::AwaitExpression { + expression: inner, .. + } + | Expression::FileExists { path: inner, .. } + | Expression::DirectoryExists { path: inner, .. } + | Expression::ListFiles { path: inner, .. } + | Expression::ReadContent { + file_handle: inner, .. + } + | Expression::ReadBinaryContent { + file_handle: inner, .. + } + | Expression::FileSizeOf { + file_handle: inner, .. + } + | Expression::ProcessRunning { + process_id: inner, .. + } => { + self.check_expression_names_defined(inner); + } + Expression::IndexAccess { + collection, index, .. + } => { + self.check_expression_names_defined(collection); + self.check_expression_names_defined(index); + } + Expression::PropertyAccess { object, .. } | Expression::MemberAccess { object, .. } => { + self.check_expression_names_defined(object); + } + Expression::MethodCall { + object, arguments, .. + } => { + self.check_expression_names_defined(object); + for arg in arguments { + self.check_expression_names_defined(&arg.value); + } + } + Expression::FunctionCall { + function, + arguments, + .. + } => { + self.check_expression_names_defined(function); + for arg in arguments { + self.check_expression_names_defined(&arg.value); + } + } + Expression::ActionCall { arguments, .. } => { + for arg in arguments { + self.check_expression_names_defined(&arg.value); + } + } + Expression::PatternReplace { + text, + pattern, + replacement, + .. + } => { + self.check_expression_names_defined(text); + self.check_expression_names_defined(pattern); + self.check_expression_names_defined(replacement); + } + Expression::HeaderAccess { request, .. } => { + self.check_expression_names_defined(request); + } + Expression::ReadBinaryN { + file_handle, count, .. + } => { + self.check_expression_names_defined(file_handle); + self.check_expression_names_defined(count); + } + Expression::ListFilesRecursive { + path, extensions, .. + } => { + self.check_expression_names_defined(path); + if let Some(extensions) = extensions { + for extension in extensions { + self.check_expression_names_defined(extension); + } + } + } + Expression::ListFilesFiltered { + path, extensions, .. + } => { + self.check_expression_names_defined(path); + for extension in extensions { + self.check_expression_names_defined(extension); + } + } + Expression::DatabaseQuery { + db, + sql, + parameters, + .. + } => { + self.check_expression_names_defined(db); + self.check_expression_names_defined(sql); + if let Some(parameters) = parameters { + self.check_expression_names_defined(parameters); + } + } + } + } + fn are_types_compatible(&self, target_type: &Type, source_type: &Type) -> bool { #[allow(clippy::only_used_in_recursion)] let _self = self; // Suppress the warning for self parameter @@ -4562,6 +5381,53 @@ mod tests { use crate::parser::ast::{Argument, Expression, Literal, Parameter, Program, Statement, Type}; use std::sync::Arc; + #[test] + fn test_header_map_type_requires_text_keys() { + // HTTP header names must be text. The header-map validity check (shared by + // outbound HTTP, streaming-response, and `respond` header clauses) must + // reject a map with a concretely-typed non-text key, while accepting + // text-keyed maps and any map whose key the checker could not pin down. + let tc = TypeChecker::new(); + + // Accepted: text keys with a value type the runtime stringifies + // (text/number/bool), or an unresolved/loose key or value type. + for ok in [ + Type::Map(Box::new(Type::Text), Box::new(Type::Text)), + Type::Map(Box::new(Type::Text), Box::new(Type::Number)), + Type::Map(Box::new(Type::Text), Box::new(Type::Boolean)), + Type::Map(Box::new(Type::Text), Box::new(Type::Any)), + Type::Map(Box::new(Type::Text), Box::new(Type::Unknown)), + Type::Map(Box::new(Type::Unknown), Box::new(Type::Unknown)), + Type::Map(Box::new(Type::Any), Box::new(Type::Text)), + Type::Unknown, + Type::Any, + ] { + assert!( + tc.is_valid_header_map_type(&ok), + "expected {ok:?} to be a valid header map type" + ); + } + + // Rejected: a concrete non-text key, a value type the runtime rejects + // (e.g. Binary or a nested list), or a non-map entirely. + for bad in [ + Type::Map(Box::new(Type::Number), Box::new(Type::Text)), + Type::Map(Box::new(Type::Boolean), Box::new(Type::Any)), + Type::Map(Box::new(Type::Text), Box::new(Type::Binary)), + Type::Map( + Box::new(Type::Text), + Box::new(Type::List(Box::new(Type::Text))), + ), + Type::Number, + Type::Text, + ] { + assert!( + !tc.is_valid_header_map_type(&bad), + "expected {bad:?} to be rejected as a header map type" + ); + } + } + #[test] fn test_variable_declaration_type_inference() { let program = Program { diff --git a/testing.md b/testing.md new file mode 100644 index 00000000..96f1abf3 --- /dev/null +++ b/testing.md @@ -0,0 +1,723 @@ +# WFL Testing — Policy & Project Profile + +This repository adopts the **Logbie Testing Policy** (reproduced verbatim in +[§ Logbie Testing Policy](#logbie-testing-policy) below) and defines the WFL +project testing profile required by that policy's §4. + +> **Adopted organization-policy version:** 1.0 +> **Testing-profile review date:** 2026-07-22 +> **Test-suite / infrastructure owner:** Maintainer (Brad, Logbie LLC) + +--- + +## WFL project testing profile (Logbie Testing Policy §4) + +### Supported configuration tuples + +| Tuple | Presubmit | Release | +|---|---|---| +| Linux x86-64 (ubuntu-latest), Rust stable (MSRV **1.94+**, edition 2024) | ✅ | ✅ | +| Windows x86-64 (windows-latest), Rust stable | ✅ (integration) | ✅ | +| macOS | — | — | + +> **macOS is not a gated tuple.** CI runs only `ubuntu-latest` and +> `windows-latest` (`.github/workflows/ci.yml`); there is no macOS runner in +> presubmit or release. macOS is supported only best-effort by contributors and +> is not verified by this pipeline. Add a macOS matrix entry before claiming any +> gated macOS coverage here. + +Key runtime dependencies: `tokio`, `warp`/`hyper`, `reqwest`, `sqlx`, `logos`, +`tower-lsp`. The interpreter core is single-threaded (`Rc`/`RefCell`); async I/O +runs on Tokio. + +### Test layers — one documented command each + +| Layer | Command | +|---|---| +| Format (static) | `cargo fmt --all -- --check` | +| Lint (static) | `cargo clippy --all-targets --all-features -- -D warnings` | +| Unit + Rust integration | `cargo test --all` | +| WFL end-to-end programs | `cargo build --release` then `./scripts/run_integration_tests.sh` (`.ps1` on Windows) | +| Web-server end-to-end | `./scripts/run_web_tests.sh` (`.ps1` on Windows) | +| Docs examples validation | `python scripts/validate_docs_examples.py` | +| Benchmarks (perf, non-gating) | `cargo bench` | + +**Run all presubmit checks** (clean checkout): + +```bash +cargo fmt --all -- --check \ + && cargo clippy --all-targets --all-features -- -D warnings \ + && cargo test --all \ + && cargo build --release \ + && ./scripts/run_integration_tests.sh \ + && ./scripts/run_web_tests.sh \ + && python scripts/validate_docs_examples.py +``` + +Every command returns a non-zero exit status on failure. The presubmit suite is +hermetic: web/HTTP tests use local ephemeral servers and MUST NOT depend on the +public internet. Non-executable docs examples that need a live upstream or +external client are marked with a first-line `// CI-SKIP: ` and are +validated statically (layers 1–4) via the docs-examples manifest instead. + +### Required services, fixtures, credentials + +- No external credentials or network for presubmit. Local TCP servers on + ephemeral ports stand in for HTTP peers. +- SQLite for DB tests; no production data. +- TLS tests generate throwaway certs (`rcgen`). + +### Critical user/operator journeys (release-blocking end-to-end) + +1. **Run a program**: `wfl ` — lex → parse → analyze → typecheck → + interpret, correct exit code (`tests/`, `TestPrograms/`, `run_integration_tests`). +2. **Web server request/response**: `listen` → `wait for request` → `respond` + over a real socket (`run_web_tests`, `tests/web_server_*`). +3. **Streaming (client)**: `open url ... stream response` → `wait for next + line|chunk` → `nothing` at EOF (`tests/http_stream_test.rs`). +4. **Streaming (server)**: `start streaming response` → `write line|chunk` → + `close` (`tests/http_server_streaming_test.rs`). +5. **Concurrent handlers**: `main loop concurrently:` — a slow handler does not + block a fast sibling; failures are contained (`tests/concurrent_main_loop_test.rs`). +6. **File I/O**, **outbound HTTP**, **REPL**, **crypto/hashing**. + +### Risk triggers (§11) + +- **Concurrency / streaming / lifecycle (§11.3):** any change to + `main loop concurrently:`, request handling, or the streaming statements MUST + add tests for races/ordering, cancellation, timeouts, disconnects, bounded + queues/backpressure, resource limits, clean shutdown, and writes-after-close, + and MUST prove one slow/failed operation does not block unrelated work. **R3.** +- **Untrusted input (§11.1):** lexer, parser, pattern VM, HTTP/multipart, and + config readers require malformed/oversized/adversarial cases; fuzz targets + (`cargo fuzz`) with a retained corpus for parser/pattern paths. +- **Security/crypto:** WFLHASH, password hashing, subprocess sanitization — + positive/negative + invariant (constant-time, zeroize) tests. **R3.** +- **Backward compatibility (§11.6):** WFL is a language — every existing + `TestPrograms/*.wfl` MUST keep passing; new syntax MUST NOT steal previously + valid programs (regression tests required). + +### Coverage & budgets + +- Baseline: the policy's defaults (≥80% line / 70% branch overall; ≥90/85 on + changed code) are the target. **Known gap:** the repo does not yet run an + automated coverage gate in CI; establishing one is a tracked conformance item + (§19). Until then, changed behavior MUST still ship behavior/boundary/negative + tests at the lowest useful layer plus every affected higher layer. +- Performance budgets: Criterion benches under `benches/` are informational; no + release-blocking latency budget is defined yet (tracked gap). + +### CI jobs & gating + +GitHub Actions (`.github/workflows/`) runs, per push/PR: format, clippy, +debug/release build, `cargo test`, Linux + Windows integration, **Run WFL +Programs**, web tests, database tests, and fuzz-target compilation. All are +required checks; the default branch MUST stay green. The nightly build is a +release artifact and is separately monitored. + +### Evidence, runtimes, retention + +- Expected presubmit runtime: minutes (Rust build dominates). Slow web/timeout + tests use bounded deadlines, not unbounded sleeps. +- Evidence (Red→Green, CI run ids, commands) lives on the PR per §15; retained + per §15 retention windows. + +### Justified non-applicable layers + +- **Accessibility/UI (§11.7):** WFL ships a CLI/LSP, no first-party GUI — + UI/a11y layers are structurally N/A (LSP behavior is covered by `wfl-lsp` + tests). Reviewer-confirmed. + +### Conformance gaps (tracked, §19) + +- No automated coverage gate in CI yet. +- No scheduled extended fuzz/soak profile yet (fuzz targets compile in CI; longer + campaigns are not scheduled). +- No formal release-candidate artifact gate beyond the nightly build. + +These gaps are owned by the Maintainer and do not authorize new untested +behavior; touched code still follows Red→Green and the risk triggers above. + +--- + +## Logbie Testing Policy + +> **Adoption note (this repository):** the text below is the organization policy +> reproduced verbatim, so it keeps the canonical **Status: Proposed** label and +> **Effective date: Upon adoption**. For WFL specifically, that adoption has +> happened: this repository has adopted policy version 1.0 as **binding and in +> force** (see the header at the top of this file, and `CLAUDE.md` / `AGENTS.md`), +> effective **2026-07-22** (the testing-profile review date). Read the "Upon +> adoption" below as "as of 2026-07-22" for this repo. + +**Status:** Proposed organization policy, version 1.0 +**Owner:** Logbie LLC Engineering +**Effective date:** Upon adoption +**Last updated:** July 22, 2026 +**Applies to:** Every Logbie-owned software project, repository, package, service, application, game, agent, infrastructure definition, and release artifact + +### 1. Purpose + +Testing is executable evidence that a change behaves as intended, fails safely, and does not break supported behavior. It is part of design and implementation, not a cleanup task performed after the code appears finished. + +This policy establishes the minimum testing standard for all Logbie projects. Individual projects may impose stricter rules, but they may not weaken this policy without a time-limited, recorded exception. + +The objective is not to manufacture green dashboards. The objective is to make trustworthy changes and retain enough evidence for another engineer to understand what was proved. + +### 2. Policy language + +The terms in this document are normative: + +- **MUST / MUST NOT** — mandatory. A violation blocks merge or release unless this policy explicitly permits an exception. +- **SHOULD / SHOULD NOT** — the normal expectation. Deviations require a written reason in the change record. +- **MAY** — optional and permitted. +- **Required test** — a test selected by this policy, the project's testing profile, the change's risk, or an acceptance criterion. +- **Change record** — the durable issue, ticket, or pull request that owns the work and its evidence. +- **Public contract** — behavior relied on outside the changed implementation, including user and operator workflows, public APIs, CLI behavior, protocols, events, schemas, stored formats, packages, configuration, and documented compatibility. +- **Critical journey** — an end-to-end workflow whose failure would prevent a user or operator from receiving a core outcome or would create material security, privacy, data-integrity, availability, or recovery risk. +- **Independent reviewer** — a qualified person or separately instructed review agent that did not author the implementation, examines the actual diff and evidence, and has no ability to approve merely by repeating the author's claims. +- **Release** — any production promotion, continuous-deployment rollout, package or container publication, app-store submission, infrastructure apply, or externally distributed prerelease. Renaming a release "just a deployment" does not change its gates. + +### 3. Non-negotiable rules + +1. Every behavioral change MUST have automated regression coverage at the lowest useful layer and every affected higher layer. +2. Every R1–R3 behavioral change MUST follow **Red → Green → Refactor → Broaden → Record**, except for the Green → Green maintenance rule in Section 6.3 or the incident rule in Section 17. +3. Every new, modified, or removed behavior MUST include auditable evidence that the relevant test failed for the expected reason before the production change made it pass. A defect fix MUST reproduce the defect. +4. A releasable product MUST have real end-to-end tests for its critical user and operator journeys. Those tests are release-blocking. +5. A test MUST NOT mock, stub, or bypass the boundary it claims to verify. +6. A mocked component test MUST NOT be labeled end-to-end. +7. Required tests MUST NOT be made green through automatic retries, skips, ignores, quarantine, muted failures, relaxed assertions, or unexplained snapshot regeneration. +8. A flaky required test is a failing test. It blocks merge until repaired or until the responsible change is reverted. +9. CI MUST test the integrated change from a clean checkout. A passing developer machine is useful evidence, not final evidence. +10. Coverage numbers MUST NOT substitute for behavior, boundary, negative-path, recovery, or end-to-end tests. +11. Tests and test infrastructure are production-quality code. They receive review, ownership, maintenance, and security controls. +12. AI-generated code, tests, summaries, and claims receive exactly the same verification as human-written work. "The model said it works" is not test output; it is optimism wearing a tiny hard hat. + +### 4. Repository testing profile + +Every repository MUST contain a root-level `testing.md`. It MUST either include this policy or link to the canonical version, and it MUST define a project-specific testing profile containing: + +- The supported platform, architecture, runtime, browser, database, and dependency configuration tuples, including which run in presubmit and which run at release +- One documented command for each available test layer +- A single command or workflow that runs all presubmit checks +- Required services, containers, fixtures, credentials, hardware, and test data +- The project's critical user and operator journeys +- The risk triggers that require security, migration, performance, concurrency, recovery, fuzz, compatibility, or accessibility testing +- Coverage measurement and thresholds +- Performance and resource budgets, when applicable +- CI job names and which jobs block pull requests, merges, and releases +- The cadence, maximum evidence age, and invalidation triggers for required scheduled suites +- Expected test runtimes and the location of retained evidence +- Owners for the test suites and test infrastructure +- Any justified layer that does not apply to the project +- The adopted organization-policy version and the testing-profile review date + +Commands MUST work from a clean documented environment and MUST return a nonzero exit status on failure. Local project rules may be stricter than this policy but MUST NOT silently redefine terms such as "end-to-end," "pass," or "release-ready." + +The canonical organization copy of this file takes precedence over stale copied text. Repositories MUST adopt a new policy version before their next release and within 30 days unless a valid Section 17 exception says otherwise. + +A monorepo MAY use one root profile, but every independently releasable package, service, application, or artifact MUST have an identifiable subprofile covering its commands, critical journeys, owners, compatibility matrix, and release gates. + +A project-level "not applicable" declaration is permitted only when a layer is structurally impossible for that project type. It requires engineering-owner approval, a review date, and a technical explanation. It cannot override a risk introduced by a particular change. + +If a repository lacks a valid testing profile, behavioral changes to that repository are not ready to merge. + +### 5. Change risk classes + +Every change MUST be assigned the highest applicable risk class before implementation. Executable changes default to R2 until the change record justifies another class. Risk may be raised during review; it MUST NOT be lowered merely to avoid a test gate. + +When classification is ambiguous, the higher class applies. An R1 classification MUST explain why the change cannot affect a public contract, persistent state, security boundary, process boundary, or critical journey, and a reviewer MUST confirm it. + +| Class | Typical changes | Minimum verification | +| --- | --- | --- | +| **R0 — Non-behavioral** | Prose-only documentation, comments, spelling, and assets proven not to affect shipped output or an acceptance criterion | Formatting, link or documentation build checks as applicable; confirmation that no executable behavior changed | +| **R1 — Local behavior** | Isolated logic with no public contract, persistence, security, or process boundary | Auditable Red → Green evidence, focused unit tests, relevant component tests, full affected suite, static checks | +| **R2 — Product or boundary behavior** | Public API, CLI behavior, UI flow, database access, filesystem behavior, service integration, packaging, configuration with runtime effect | R1 plus real integration or contract tests, affected critical-journey end-to-end tests, compatibility checks, clean CI | +| **R3 — Critical behavior** | Material changes to authentication, authorization, cryptography, secrets, protected user data, destructive operations, schema migration, money, safety, untrusted-input boundaries, protocol guarantees, concurrency, cancellation, lifecycle, recovery, release controls, or high-availability behavior | R2 plus negative and failure-path tests, applicable security/property/fuzz/concurrency/recovery/performance tests, independent review, recovery evidence, and the full release-relevant end-to-end suite | + +All product releases, regardless of the individual changes they contain, MUST pass the full release gate in Section 14. + +### 6. Test-driven development + +#### 6.1 Required loop + +For each acceptance criterion or defect: + +1. **Specify** — express the behavior as an observable outcome, including relevant failure behavior. +2. **Red** — add or identify the smallest useful automated test and run it. Confirm that it fails for the intended reason. +3. **Green** — make the smallest coherent production change that satisfies the test. +4. **Refactor** — improve the implementation and tests while keeping them green. +5. **Broaden** — run the affected integration, contract, end-to-end, security, compatibility, and other risk-triggered suites. +6. **Record** — attach the evidence required by Section 15 to the change record. + +The Red step is invalid if the test fails because of a syntax error, broken fixture, missing dependency, unrelated failure, or an assertion that does not represent the required behavior. + +#### 6.2 Acceptable Red → Green evidence + +At least one of the following MUST be retained: + +- A focused test-only Red commit that is an ancestor of the Green implementation commit +- An independently timestamped CI or change-record artifact created before the Green implementation commit, tied to a Red revision and showing the test name, command, expected behavior, actual failure, and failure reason +- For a reproduced defect, an automated regression test applied to the recorded affected base revision and retained in a Red commit before the Green fix + +The evidence MUST identify the base, Red, and Green commit identifiers. The final history may be squashed after the evidence is attached to the change record. A newly written test that was observed only after the implementation already passed it does not establish the required Red step. Reverting or disabling completed code may prove that a regression test is capable of failing, but it does not retroactively prove TDD chronology. + +#### 6.3 Refactors and non-behavioral changes + +A behavior-preserving refactor or maintenance change MUST establish adequate characterization coverage and record a passing baseline before the change, then pass the same coverage afterward. Dependency, toolchain, packaging, infrastructure, and configuration maintenance also require applicable compatibility, security, integration, and end-to-end evidence. They do not need an artificial failing test when no behavior is intended to change. + +R0 changes do not require a manufactured Red step. Configuration, build, workflow, dependency, infrastructure, schema, and documentation-generator changes are not R0 when they can change executable behavior. + +#### 6.4 Incidents + +During an active incident, the minimum reversible mitigation may precede the normal Red step only under Section 17. The defect MUST receive regression coverage before the incident ticket is closed. An emergency is a reason to reorder evidence, not to delete it. + +### 7. Required test layers + +The risk table, project testing profile, acceptance criteria, and Section 11 triggers determine the required layers. Within that set, a change MUST use every layer needed to prove the affected contract without duplicating tests that add no distinct evidence. "Not applicable" requires a specific technical explanation in the change record and reviewer acceptance. + +#### 7.1 Static verification + +Projects MUST run applicable formatting, compilation, linting, type checking, schema validation, policy checks, secret scanning, dependency checks, and generated-file consistency checks. + +Static verification supplements executable tests; it does not replace them. + +#### 7.2 Unit tests + +Unit tests MUST cover new or changed business rules, validation, algorithms, state transitions, parsers, error classification, and policy decisions when those behaviors can be isolated. + +Unit tests SHOULD be fast, deterministic, precise, and independent of network or shared external state. They SHOULD assert observable behavior rather than private implementation details. + +#### 7.3 Component and service tests + +Component tests verify a complete module, package, process, or service through its public interface. They MUST use real internal components for the behavior under test and MAY substitute only dependencies outside the declared component boundary. + +#### 7.4 Integration tests + +Integration tests MUST exercise real boundaries whenever the change affects them, including as applicable: + +- Database engines, schemas, transactions, migrations, and queries +- Filesystems, permissions, paths, locks, and storage formats +- Processes, signals, standard streams, exit codes, and packaged binaries +- HTTP, WebSocket, streaming, queue, event, RPC, and protocol behavior +- Authentication, authorization, redaction, and policy enforcement +- Containers, operating-system facilities, and service discovery +- Timeouts, retries, cancellation, disconnects, backpressure, restart, and idempotency + +An in-memory replacement is not evidence that the actual database, filesystem, queue, protocol, or operating-system integration works. + +#### 7.5 Contract tests + +Every public or cross-service interface MUST have contract tests covering successful responses, errors, versioning, required fields, optional fields, limits, malformed input, and backward compatibility. + +Cross-repository contracts MUST name the provider owner, consumer owner, compatible version range, and repository responsible for candidate compatibility testing. Coordinated provider and consumer changes MUST test supported version skew before either side releases. + +Provider simulators and deterministic adapters MAY support fast tests, but a project that claims compatibility with an external provider MUST also verify the contract against that provider's official test environment, sandbox, or independently controlled conformance reference defined in the project testing profile. If the provider offers no safe test environment, a versioned signed recording or reference corpus MAY substitute only with owner approval, a declared freshness limit, and proof that it covers the claimed provider version. The project MUST state that live interoperability was not verified. + +#### 7.6 End-to-end tests + +End-to-end tests exercise a complete user- or operator-visible journey through the production entry points and shipped artifact. They MUST: + +- Start from a clean, production-like state +- Use the real application binary or packaged artifact +- Cross the real in-scope process, storage, protocol, and UI boundaries +- Assert both the final outcome and important externally visible side effects +- Exercise cleanup or recovery where the journey changes state +- Produce enough evidence to diagnose a failure + +Browser products MUST use a real supported browser for browser journeys. Service products MUST use their real network interface. CLI and desktop products MUST execute the packaged binary. Libraries MUST provide consumer, conformance, or system-harness tests that exercise the published artifact as a downstream user would. + +If an external paid or unsafe system is replaced, the test MUST be labeled as a system test rather than a true end-to-end test of that external integration. The external integration then requires a separate credentialed sandbox or release smoke profile, or the approved conformance-reference path in Section 7.5 when no safe provider environment exists. + +Every product MUST define a small, reliable critical-journey suite that blocks merge when affected and blocks every release in full. + +Every new or changed user- or operator-visible boundary behavior MUST add or update an end-to-end assertion unless an existing test already asserts that exact observable outcome. The critical-journey suite is the always-release-blocking subset, not a loophole for leaving noncritical workflows unproved. + +#### 7.7 Exploratory and manual testing + +Manual and exploratory testing MAY discover issues and provide useful product evidence. They MUST NOT replace required automated regression tests. Any defect found manually MUST receive automated coverage. A platform limitation that makes automation impossible requires a Section 17 exception and cannot waive a critical release gate. + +### 8. Test integrity + +#### 8.1 Determinism + +Tests MUST control or record time, randomness, locale, time zone, network assumptions, identifiers, and ordering when those inputs affect results. Randomized tests MUST report the seed and retain failing inputs. + +Date and time behavior MUST cover applicable expiration boundaries, daylight-saving transitions, leap dates, time zones, and locale changes. Tests capable of blocking MUST have an explicit bounded timeout. + +Tests MUST be isolated from one another. They MUST NOT rely on execution order, shared mutable fixtures, production state, or residue from a previous run. + +#### 8.2 Failures, flakes, and retries + +- A required test that fails once has failed. +- Required tests MUST NOT automatically retry at the test, framework, or CI layer. +- Required-suite configuration MUST expose first-attempt results and disable hidden framework or CI retries. +- A CI job MAY be rerun only when independent evidence shows that the runner or external test infrastructure failed before a product-test result was produced. The original run, evidence, and reason MUST remain visible. This permitted infrastructure rerun is not a test retry. +- A test that passes only on retry is flaky and blocks merge. +- Required tests MUST NOT be skipped, ignored, muted, quarantined, marked "allowed to fail," or removed from the gating suite to obtain green CI. +- Platform-specific tests MAY be selected only on their declared matrix entries, but the release gate MUST execute every supported entry. +- A known flaky test on the default branch is an urgent repository defect. The default branch MUST be restored to trustworthy green before unrelated behavioral work merges. + +#### 8.3 Assertions and snapshots + +Tests MUST assert meaningful outcomes, error behavior, and side effects. "Did not crash" is insufficient when the behavior has a defined result. + +Snapshots and golden files MUST be human-reviewable. Their changes MUST be reviewed like production code. Bulk regeneration without explaining each intentional behavioral difference is prohibited. + +Negative assertions MUST be used where absence matters, including authorization denial, secret non-disclosure, duplicate prevention, rollback, cancellation, and writes outside an allowed boundary. + +#### 8.4 Test doubles + +Mocks, fakes, stubs, emulators, and simulators MUST be named accurately and confined to a declared boundary. A test double MUST NOT make the behavior under test impossible to fail. + +Important doubles SHOULD be checked against the real implementation through contract tests so they do not become cheerful little liars with perfect uptime. + +### 9. Coverage and test strength + +Coverage is a diagnostic and regression floor, not a target that proves correctness. + +Unless a stricter project profile applies: + +- New projects MUST maintain at least **80% line coverage** and **70% branch coverage** across instrumentable first-party executable code before their first production release. +- Changed instrumentable executable code MUST achieve at least **90% line coverage** and **85% branch coverage**. +- A change MUST NOT reduce repository line or branch coverage by more than 0.5 percentage points without an approved exception. The stored baseline MUST never be silently lowered. +- Security, authorization, destructive-operation, financial, migration, and other R3 decision logic MUST have explicit tests for every identified policy outcome and failure mode, regardless of the aggregate percentage. + +Generated code, vendored code, build output, and provably unreachable platform shims MAY be excluded. Exclusions MUST be reviewable configuration, not ad hoc command-line omissions. + +"Changed code" means added or modified first-party executable lines relative to the target branch's merge base, with renames tracked when the tool supports them. Base and head MUST be measured in the same CI job using the same version-controlled coverage configuration, tool version, platform, exclusions, and rounding to two decimal places. A tool or configuration change that alters the denominator requires an old-versus-new comparison and test-infrastructure-owner approval. + +If reliable branch coverage is unavailable for a language, the testing profile MUST name the limitation and define reviewed condition, decision, scenario, or mutation coverage that supplies equivalent evidence. Calling code "non-instrumentable" without this approved alternative is not an exclusion. + +When conventional coverage is meaningless—such as declarative infrastructure, visual assets, or hardware workflows—the project testing profile MUST define scenario, requirement, state, or interface coverage instead. + +R3 projects SHOULD use mutation testing or an equivalent test-strength analysis for critical logic before a major release. Surviving meaningful mutations indicate missing assertions even when line coverage looks impressive. + +### 10. Test data, fixtures, and environments + +- Tests MUST NOT use production secrets, credentials, private keys, or uncontrolled personal data. +- Production-derived data MUST be minimized, sanitized, approved, and documented before use. +- Destructive tests MUST run only in explicitly disposable environments with guardrails that make production targeting impossible. +- Test resources MUST use unique names or isolated namespaces and MUST clean up on success, failure, cancellation, and timeout. +- Disposable environments MUST also have an independent time-to-live or janitor cleanup path for hard runner termination, where test code cannot execute cleanup. +- Fixtures MUST be small enough to review and version unless a justified artifact store is defined. +- Schema, protocol, and file-format fixtures MUST include the oldest supported version, current version, malformed cases, boundary sizes, and forward-compatibility cases where applicable. +- Credentials for sandbox or release profiles MUST be short-lived, least-privileged, redacted from output, and unavailable to untrusted pull requests. +- Test logs and artifacts MUST be sanitized before retention. + +The standard presubmit suite SHOULD be hermetic and MUST NOT depend on the public internet. Credentialed, hardware, provider, load, soak, and privileged tests belong in explicitly named profiles with controlled environments. + +### 11. Risk-triggered testing + +The following requirements apply whenever the corresponding risk exists. + +#### 11.1 Security and privacy + +Changes affecting trust boundaries, identity, authorization, secrets, untrusted input, or personal data MUST include: + +- Positive and negative authorization cases +- Role, tenant, and ownership-boundary tests +- Malformed, oversized, replayed, duplicated, and adversarial input cases +- Secret-redaction and sensitive-log assertions +- Session, token, timeout, revocation, and failure behavior as applicable +- Abuse-limit and resource-exhaustion tests where applicable +- A security-focused independent review for R3 changes + +Parsers, decoders, protocol handlers, and file readers exposed to untrusted input MUST have property or fuzz tests with a retained regression corpus. A bounded fuzz smoke run SHOULD execute on pull requests; longer campaigns SHOULD run on a scheduled profile. + +A known Critical security finding is non-waivable for a normal release. A known High finding blocks release unless the security owner approves a narrowly scoped Section 17 exception with demonstrated mitigation. A documented false positive supported by evidence is a resolved finding, not an exception. + +#### 11.2 Persistence and migrations + +Schema, data, and storage-format changes MUST be tested from every supported prior version using representative data. Tests MUST prove: + +- Upgrade correctness and idempotency +- Preservation of required data and constraints +- Behavior during partial failure, interruption, and restart +- Compatibility during any rolling or mixed-version deployment window +- The documented rollback, restore, or forward-repair strategy + +Destructive or irreversible migrations require explicit approval, a verified backup or recovery artifact, and a rehearsal in a production-like disposable environment. + +#### 11.3 Concurrency, streaming, and lifecycle + +Concurrent, asynchronous, networked, streaming, or long-running behavior MUST test applicable races, ordering, cancellation, timeouts, disconnects, bounded queues, backpressure, resource limits, clean shutdown, restart, and writes after close. + +Queue and event-driven behavior MUST test duplicates, delays, reordering, replay, poison messages, partial acknowledgement, and idempotent recovery when those conditions are possible. + +Tests MUST prove that one slow or failed operation does not improperly block unrelated work. Where the language or platform supplies race detection, concurrency modeling, sanitizers, or deterministic schedulers, the project SHOULD include them in CI or scheduled testing. + +#### 11.4 Reliability and recovery + +Stateful or continuously running systems MUST test crash recovery, restart, duplicate delivery, partial completion, idempotency, lost dependencies, corrupted or stale inputs, and degraded operation. + +High-availability projects MUST define scheduled chaos, failover, and soak profiles. A release MUST NOT claim a recovery property that has never been exercised. + +#### 11.5 Performance and resource use + +Projects with latency, throughput, memory, storage, startup-time, battery, network, or cost requirements MUST define measurable budgets in their testing profile. + +Performance tests MUST use controlled workloads, warmup rules, environments, and comparison methods. A statistically meaningful budget regression blocks release unless explicitly accepted under Section 17. Microbenchmarks alone do not prove system capacity. + +#### 11.6 Compatibility and packaging + +Every supported matrix entry MUST be tested before release. Projects MUST test the artifact users actually install or run, including package metadata, default configuration, startup, upgrade, and uninstall or cleanup behavior when applicable. + +Feature flags MUST be tested in their default and non-default states, including authorization and migration behavior affected by the flag. Removing a flag requires tests for the resulting permanent path. + +Examples and published code snippets SHOULD compile or execute in CI. Public libraries MUST test supported consumers and backward compatibility according to the project's versioning policy. + +#### 11.7 User interfaces and accessibility + +User-facing applications MUST test critical journeys through the real UI. Applicable flows MUST cover keyboard operation, focus behavior, accessible names, error presentation, and the project's accessibility target. Automated accessibility checks MUST be supplemented by documented manual checks for major releases where automation cannot establish the behavior. + +Visual regression tests MAY supplement behavioral tests but MUST NOT replace them. + +#### 11.8 Infrastructure and deployment + +Infrastructure-as-code and deployment changes MUST pass syntax, policy, plan, idempotency, least-privilege, secret-handling, and rollback, restore, or forward-repair checks. R2 and R3 changes MUST be exercised in a disposable or staging environment before production. + +Tests MUST make destructive plans obvious and MUST prevent production mutation from ordinary CI. + +#### 11.9 Games and deterministic simulations + +Game and simulation projects MUST make authoritative logic testable deterministically. If presentation coupling prevents direct isolation, the project testing profile MUST define a deterministic system harness. Tests MUST cover seeded replay, invariants, boundary conditions, save/load round trips, version compatibility, economic or scoring conservation rules, and long-run stability as applicable. + +Balance evaluation and bot simulations are evidence for design decisions, but they do not replace correctness tests. + +#### 11.10 AI and model-backed features + +Model-backed behavior MUST keep deterministic application rules under ordinary automated tests. Provider adapters require contract tests. Prompt, model, retrieval, or tool-policy changes MUST use versioned evaluation sets with documented pass thresholds, safety cases, cost limits, and regression comparisons. + +Live model evaluations MUST run only in an authorized credentialed profile. Their nondeterminism MUST be measured and reported; it MUST NOT be hidden with retries until a preferred answer appears. + +#### 11.11 Observability and audit behavior + +R3 services and agents MUST test required audit events, security-relevant logs, metrics, trace propagation, alert conditions, and redaction for privileged operations and critical failures. Tests MUST prove both that required signals appear and that secrets or protected data do not. + +### 12. Test design and maintenance + +- Test names MUST describe the behavior and relevant condition. +- A test SHOULD have one clear reason to fail, while a scenario test MAY make several related assertions needed to diagnose the journey. +- Tests SHOULD use public interfaces and stable contracts. +- Shared fixtures and helpers MUST reduce accidental complexity without hiding important setup or assertions. +- Timing-based waits and arbitrary sleeps SHOULD be replaced by observable readiness conditions and bounded deadlines. +- Test suites MUST be runnable in parallel only when their isolation supports it. +- Slow tests MUST be measured and improved, split by profile, or assigned appropriate infrastructure; they MUST NOT be silently removed from required evidence. +- Deleted behavior SHOULD have obsolete tests removed. Changed tests MUST explain whether the product contract changed or the prior test was incorrect. +- A production defect that escaped existing tests MUST add or improve the layer that should have caught it, not only the layer where it was easiest to reproduce. + +Test-only hooks in production code MUST NOT weaken security or alter normal behavior. If unavoidable, they require review and must be inaccessible in production builds or deployments. + +Changes to CI workflows, test filters, impact maps, retry settings, coverage tools or exclusions, thresholds, required-check names, risk rules, or `testing.md` MUST receive test-infrastructure-owner approval and run the complete presubmit suite. A change MUST NOT weaken the machinery that judges that same change. + +CI MUST fail closed when the change record lacks a risk class, a required job has no result, or the executed job set does not match the approved testing profile and risk triggers. + +### 13. Continuous-integration profiles + +Projects MUST define the following profiles where applicable. + +Default and release branches MUST be protected. Required checks and review rules apply to maintainers, administrators, bots, and merge queues; ordinary work MUST NOT use an administrative bypass. Section 17 is the only bypass path and it MUST be logged. + +#### 13.1 Pull request / presubmit + +Runs from a clean checkout with locked dependencies and pinned or recorded tool versions, and includes: + +- Formatting, build, lint, type, policy, secret, and dependency checks +- Focused and complete unit suites +- Affected component, integration, and contract suites +- Affected critical-journey end-to-end tests +- Coverage and changed-code thresholds +- Bounded property, fuzz, security, or concurrency smoke tests triggered by risk + +Required checks MUST pass on the final proposed commit. + +Path-based or impact-based test selection MAY reduce presubmit work only when backed by a maintained dependency map. The merge queue or another pre-merge gate MUST still run the complete impacted suite. + +#### 13.2 Merge queue / integrated branch + +The change MUST be tested with the latest target branch and other queued changes. The merge result MUST pass all required presubmit checks from a clean environment. Stale green results from an earlier base are insufficient. + +The default branch MUST remain green. A red default branch is an incident owned ahead of feature work. + +#### 13.3 Scheduled extended profile + +Longer fuzzing, property exploration, compatibility matrices, credentialed sandboxes, race detection, sanitizers, load, chaos, failover, and soak tests SHOULD run on a documented schedule according to project risk. + +Failures MUST create or update an owned change record and block release. They MUST block further affected merges when they invalidate presubmit evidence. + +The project profile MUST define freshness for each scheduled suite; seven days is the default maximum. A new run is required sooner when relevant production code, dependencies, toolchain, tests, configuration, environment, or candidate artifacts change. + +#### 13.4 Release candidate + +Runs against the immutable release candidate artifact set and includes: + +- The complete supported platform and dependency matrix +- The full critical-journey end-to-end suite +- Installation, startup, upgrade, migration, compatibility, and rollback, restore, or forward-repair checks +- All R3 security, recovery, concurrency, and data-integrity suites +- Required performance, load, soak, provider, hardware, and accessibility evidence +- Artifact integrity, provenance, license, dependency, and secret checks + +Every released binary, package, image, installer, or other artifact MUST be identified and tested. The tested artifact set MUST be the set released, and each digest or equivalent immutable identifier MUST be recorded. Rebuilding after the gate invalidates the gate. + +A platform-controlled signing, notarization, store-repackaging, or deployment transformation MAY occur after the main gate only when its input and output provenance are recorded, the transformation is deterministic or independently verified, and the distributed result passes a post-transformation smoke test. Unchanged evidence MAY be reused only for the exact same immutable artifact and while every required scheduled result remains fresh. + +#### 13.5 Post-deployment smoke + +Deployed services SHOULD run a small non-destructive smoke suite that verifies health and critical external paths. Production smoke tests MUST be explicitly authorized, isolated from ordinary user data, safe to repeat, monitored, and incapable of destructive action. + +Post-deployment checks supplement the release gate; they do not excuse missing pre-release evidence. + +### 14. Merge and release gates + +Section 17 is the sole override path. An exception may authorize a clearly labeled emergency merge or build, but it never converts missing or failed evidence into a pass. A reproducible product-test failure, authorization bypass, data loss or corruption, exposed secret, unresolved Critical vulnerability, or failed destructive-migration recovery test is non-waivable. + +#### 14.1 A pull request MUST NOT merge when + +- A required check failed, did not run, or produced ambiguous results +- A required test was automatically retried, skipped, ignored, muted, quarantined, allowed to fail, or rerun outside the proven infrastructure-failure rule in Section 8.2 +- Red → Green evidence is missing for changed behavior +- Coverage or a declared budget regressed beyond policy +- The affected critical journey lacks end-to-end coverage +- The target branch or required baseline is red, unless this pull request is narrowly scoped to repairing or reverting that failure and all unrelated required checks pass +- Required test evidence cannot be tied to the final commit +- An unresolved blocking review, security finding, migration risk, or rollback gap remains +- Test changes weaken protection without an approved contract change + +#### 14.2 A release MUST NOT proceed when + +- The immutable candidate did not pass the full release-candidate profile +- Any supported platform or critical journey is untested or failed +- A required scheduled test has an unresolved failure that affects the release +- A Critical security issue remains unresolved, or a High issue lacks the valid security-owner exception allowed by Section 11.1 +- Data migration, compatibility, and rollback, restore, or forward-repair evidence is incomplete +- A declared performance or reliability claim lacks passing evidence +- Required artifacts, test reports, or approvals are missing + +"It is probably fine," a deadline, and repeated clicking of the rerun button are not release criteria. + +### 15. Required change evidence + +Every behavioral pull request or equivalent change record MUST include: + +- Change and risk-class summary +- Impacted repositories, release artifacts, public contracts, supported configurations, and critical journeys, with reviewer acceptance of the impact analysis +- Acceptance criteria mapped to tests +- Red evidence for each new behavior or defect fix +- Green evidence tied to the final commit +- Tests added, changed, or removed +- Exact commands or CI run identifiers for unit, integration, contract, end-to-end, and risk-triggered tests +- Coverage results and any exclusions +- Supported matrix entries exercised +- Known limitations, residual risks, and specifically justified non-applicable layers +- Review evidence required by the risk class +- Rollback, restore, or forward-repair instructions when external state can change + +Recommended pull-request section: + +```markdown +## Test evidence + +- Risk class: +- Acceptance criteria → tests: +- Red evidence: +- Unit/component: +- Integration/contract: +- End-to-end: +- Security/migration/concurrency/performance/other: +- Coverage: +- Platforms: +- Not applicable, with reason: +- Rollback/recovery: +- Residual risk: +``` + +Pull-request evidence MUST remain accessible for at least 90 days after merge or closure. Default-branch and scheduled evidence MUST remain accessible for at least 180 days. Release, security, migration, rollback, and exception evidence MUST remain accessible for the supported lifetime of the release plus one year, and never less than 24 months. Evidence MUST NOT contain secrets or uncontrolled personal data. + +Every release MUST archive an organization-controlled evidence manifest containing the commit, artifact-set identifiers, policy and testing-profile versions, required job results, tested configuration tuples, commands, toolchain and environment, seeds, coverage and scan summaries, exceptions, approvals, and recovery evidence. Short-lived CI links alone do not satisfy retention. + +### 16. Ownership and defect handling + +The author of a change owns its tests until the change is accepted. The project owner owns the ongoing health of the suites and infrastructure. + +When a production defect escapes: + +1. Reproduce it with the strongest feasible automated test. +2. Identify which test layer should have prevented the escape. +3. Fix the defect using the Red → Green loop. +4. Repair the missing assertion, fixture, environment, or gate. +5. Search for the same gap in adjacent behavior. +6. Record the cause and prevention evidence. + +CI infrastructure failures MUST be distinguished from product failures using evidence, not guesses. Repeated infrastructure instability is itself a blocking engineering defect. + +### 17. Exceptions and emergency changes + +An exception records temporary risk; it does not convert missing evidence into a pass. + +Every exception MUST identify: + +- The exact rule and affected scope +- Why compliance is technically impossible or would worsen an active incident +- The approving project owner and, for security or privacy rules, the security owner +- Start time, expiration, and maximum affected releases +- Compensating verification and containment +- Rollback plan +- A linked repair ticket with owner and deadline + +An exception is limited to one repository, an exact commit or candidate, and at most one release. The requester MUST NOT be the sole approver. Ordinary exceptions expire within 30 days; R3 exceptions expire within 7 days and require the affected security, data, reliability, or other domain owner. Renewal requires new evidence and approval; repeated renewal requires escalation to the Logbie engineering policy owner. An expired exception blocks the affected merge or release. + +Schedule pressure, test duration, missing CI setup, inconvenience, and "the change is small" are not valid reasons. + +Exceptions MUST NOT permit a normal release to claim that an untested critical journey, unsupported migration, or failed required suite passed. The release remains blocked or is explicitly classified as an emergency build with its limitations visible. + +An ordinary exception MAY address temporarily unavailable evidence or an unavailable environment. It MUST NOT waive a known product failure or any non-waivable condition in Section 14. A required test cannot be retried, skipped, quarantined, muted, or relabeled through an exception to manufacture a pass. + +During an active incident, an authorized minimum reversible mitigation MAY merge with reordered Red → Green evidence when delay would cause greater harm. All available focused tests and static checks MUST still run, the rollback path MUST be prepared, and regression coverage plus the full affected suites MUST pass within 24 hours, before the incident is closed, and before another normal deployment of the affected component. + +Expired exceptions fail closed. + +### 18. Definition of done + +A change is done only when all applicable conditions are true: + +- Acceptance criteria are observable and mapped to passing tests. +- Required Red → Green evidence exists. +- Relevant unit, component, integration, contract, and end-to-end tests pass. +- Static, coverage, security, compatibility, performance, migration, recovery, and other risk-triggered gates pass. +- The final integrated commit passes in a clean CI environment. +- Required independent review is complete and blocking findings are resolved. +- Documentation, examples, configuration, fixtures, and operational instructions are updated. +- Rollback or recovery covers both repository content and external state. +- Evidence is attached to the durable change record and contains no secrets. +- No required test is flaky, skipped, retried, quarantined, muted, or unexplained. +- Remaining work and risks are explicitly ticketed and do not violate a merge or release gate. + +If a required condition is unmet, the correct state is **blocked**, **in progress**, or **ready for approval**—not **done**. + +### 19. Adoption checklist + +An existing repository MUST add a valid testing profile before its first behavioral pull request after policy adoption or within 30 days, whichever comes first. A temporary delay requires Section 17; no production release may occur before profile adoption. Each repository MUST complete the following before its next production release: + +- [ ] Add a root-level `testing.md` adopting this policy +- [ ] Name test owners +- [ ] Document clean-environment commands for every applicable layer +- [ ] Inventory critical user and operator journeys +- [ ] Establish the supported compatibility matrix +- [ ] Establish coverage baselines and required thresholds +- [ ] Configure presubmit and merge protection +- [ ] Build a release-candidate workflow against the immutable artifact +- [ ] Remove or repair required retries, skips, quarantines, and allowed failures +- [ ] Define security, migration, concurrency, recovery, performance, and other risk triggers +- [ ] Define test-data and credential controls +- [ ] Define evidence retention and artifact locations +- [ ] Record and prioritize gaps that prevent full compliance + +Until adoption is complete, gaps MUST be visible as owned tickets. A repository MUST NOT describe itself as fully compliant while a mandatory gate is missing. + +Existing projects MUST apply this policy immediately to new and changed behavior. Touched legacy behavior MUST gain characterization coverage, and known repository-wide gaps MUST have a dated conformance plan. Existing debt does not authorize new untested behavior. + +### 20. Canonical rule + +**No behavioral change without an honest failing test. No boundary claim without a real boundary test. No release without real end-to-end proof. No green build manufactured from retries, skips, quarantine, or wishful thinking.** diff --git a/tests/ambiguous_write_analyzer_test.rs b/tests/ambiguous_write_analyzer_test.rs new file mode 100644 index 00000000..3e6de591 --- /dev/null +++ b/tests/ambiguous_write_analyzer_test.rs @@ -0,0 +1,53 @@ +//! Analyzer coverage for the ambiguous `write line|chunk ... to ` shared +//! continuation across call/pattern shapes (maintainer re-review, P1). +//! +//! The two readings of an ambiguous write differ only at the leftmost leaf; every +//! other sub-expression is shared continuation. When the continuation desugars to +//! a call/pattern shape (`starts with`, `contains pattern`, ...), the analyzer must +//! still walk the shared operands so an undefined name there is reported at analysis +//! time instead of surfacing only at runtime — without falsely rejecting either +//! valid reading of the lead. + +use wfl::analyzer::Analyzer; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +fn analyze(code: &str) -> Result<(), String> { + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + Analyzer::new() + .analyze(&program) + .map(|_| ()) + .map_err(|e| format!("{e:?}")) +} + +#[test] +fn undefined_name_in_a_starts_with_continuation_is_reported() { + // Both leads are defined (`greeting` for the stream reading, `line greeting` for + // the classic file reading), so the ONLY undefined name is the shared + // `starts with` operand. It must be reported rather than deferred to runtime. + let code = "store greeting as \"hello\"\n\ + store line greeting as \"world\"\n\ + write line greeting starts with missing_operand to \"/tmp/wfl_analyzer_out\""; + let err = analyze(code) + .expect_err("an undefined name in the shared `starts with` continuation must be reported"); + assert!( + err.contains("missing_operand"), + "expected the undefined shared-continuation name to be reported, got: {err}" + ); +} + +#[test] +fn defined_name_in_a_starts_with_continuation_is_not_a_false_positive() { + // Same shape, but the shared operand is defined: neither reading is broken, so + // analysis must pass (the parallel walk must not over-report). + let code = "store greeting as \"hello\"\n\ + store line greeting as \"world\"\n\ + store suffix as \"lo\"\n\ + write line greeting starts with suffix to \"/tmp/wfl_analyzer_out\""; + assert!( + analyze(code).is_ok(), + "a fully-defined ambiguous write must analyze cleanly: {:?}", + analyze(code).err() + ); +} diff --git a/tests/ambiguous_write_branch_typecheck_test.rs b/tests/ambiguous_write_branch_typecheck_test.rs new file mode 100644 index 00000000..6303f06f --- /dev/null +++ b/tests/ambiguous_write_branch_typecheck_test.rs @@ -0,0 +1,338 @@ +//! Backward-compatibility coverage for the ambiguous `write line|chunk ... to +//! ` type check (maintainer re-review, P1). +//! +//! The statement has two readings parsed from the same tokens: a STREAM write of +//! `value` (when the target is a response-stream handle) and a classic FILE write +//! of `fallback_content` (when the target is anything else). The runtime picks by +//! the target's runtime type. The type checker must check the reading the runtime +//! actually takes — checking the stream `value` unconditionally rejected a valid +//! pre-existing file write on a branch that never runs (and the reverse let a +//! broken file write pass because only the stream reading was checked). + +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::typechecker::TypeChecker; + +fn typecheck(code: &str) -> Result<(), String> { + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + TypeChecker::new() + .check_types(&program) + .map_err(|e| format!("{e:?}")) +} + +#[test] +fn text_target_valid_file_write_is_not_rejected_on_the_stream_branch() { + // Target is a concrete text path, so the runtime takes the FILE reading: + // `line value minus n` = 10 - 1 (Number - Number), which is valid. The stream + // reading `value minus n` would be Text - Number, but the runtime never + // evaluates it here — so this MUST type-check. (Before the fix it was rejected + // on the never-run stream branch.) + let code = "store value as \"wrong stream type\"\n\ + store line value as 10\n\ + store n as 1\n\ + write line value minus n to \"/tmp/wfl_branch_out\""; + assert!( + typecheck(code).is_ok(), + "a valid classic file write must not be rejected on the unused stream reading: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn text_target_broken_file_write_is_caught() { + // Reverse the types: `line value` is Text, so the FILE reading + // `line value minus n` = Text - Number is ill-typed. The runtime takes the file + // reading (target is a concrete text path), so this MUST be a static error. + // (Before the fix only the stream reading `value minus n` = Number - Number was + // checked, so this wrongly passed and failed only at runtime.) + let code = "store value as 10\n\ + store line value as \"text\"\n\ + store n as 1\n\ + write line value minus n to \"/tmp/wfl_branch_out\""; + assert!( + typecheck(code).is_err(), + "a file write whose content is Text minus Number must be a static error, \ + not deferred to runtime" + ); +} + +#[test] +fn concrete_non_streamable_payload_to_a_stream_is_rejected() { + // An unambiguous stream write (the target is a real response-stream handle) of + // a concrete List payload must be a static error — the runtime only accepts + // text/binary/number/boolean, so a Map/List/Nothing reaching `write` fails at + // runtime otherwise. + let code = "listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 and content type \"text/plain\" as out\n\ + store items as [1 and 2 and 3]\n\ + write line items to out"; + assert!( + typecheck(code).is_err(), + "writing a concrete List to a response stream must be a static type error" + ); +} + +#[test] +fn text_and_binary_payloads_to_a_stream_still_typecheck() { + // The valid stream payloads must keep passing. + let code = "listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 and content type \"text/plain\" as out\n\ + write line \"hello\" to out\n\ + write chunk 42 to out"; + assert!( + typecheck(code).is_ok(), + "text/number stream payloads must type-check: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn text_target_one_sided_undefined_classic_lead_is_caught() { + // Stream lead `value` is defined; classic lead `line value` is not. Target is + // concrete text, so runtime takes the classic branch — must be a static error + // (issue #642: previously analysis passed because only the stream lead existed). + let code = "store value as \"x\"\n\ + write line value to \"/tmp/wfl_onesided_out\""; + let errors = typecheck(code).expect_err("undefined classic lead must be rejected"); + assert!( + errors.contains("Variable 'line value' is not defined"), + "expected the selected classic-lead diagnostic, got: {errors}" + ); +} + +#[test] +fn main_loop_stream_binding_rejects_list_payload() { + // Inside main loop, `out` must be typed as ResponseStream so a list payload + // is rejected rather than masked by a gradual/file fallback (issue #642). + let code = "listen on port 8080 as s\n\ + main loop:\n\ + \x20\x20\x20\x20wait for request comes in on s as req\n\ + \x20\x20\x20\x20start streaming response to req with status 200 as out\n\ + \x20\x20\x20\x20store items as [1 and 2]\n\ + \x20\x20\x20\x20store line items as \"legacy\"\n\ + \x20\x20\x20\x20write line items to out\n\ + end loop"; + let errors = typecheck(code).expect_err("list payload must be rejected"); + assert!( + errors.contains("can only send text, binary, a number, or a boolean"), + "expected the concrete stream-payload diagnostic, got: {errors}" + ); +} + +#[test] +fn property_access_undefined_object_on_text_target_is_caught() { + // Merged lead with a `.field` postfix: stream reading is `upstream.status`, + // classic is `line upstream.status`. Neither object is defined; PropertyAccess + // must not evade definedness on the concrete text-target branch (issue #642). + let code = "write line upstream.status to \"/tmp/wfl_prop_out\""; + let errors = typecheck(code).expect_err("undefined property root must be rejected"); + assert!( + errors.contains("Variable 'line upstream' is not defined"), + "expected the selected classic property-root diagnostic, got: {errors}" + ); +} + +#[test] +fn property_access_checks_the_one_sided_root_for_each_concrete_target() { + let file_errors = typecheck( + "store upstream as \"stream-only\"\n\ + write line upstream.status to \"/tmp/wfl_prop_out\"", + ) + .expect_err("the selected classic property root is undefined"); + assert!( + file_errors.contains("Variable 'line upstream' is not defined"), + "text target must diagnose the classic property root, got: {file_errors}" + ); + + let stream_errors = typecheck( + "store line upstream as \"classic-only\"\n\ + listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 as out\n\ + write line upstream.status to out", + ) + .expect_err("the selected stream property root is undefined"); + assert!( + stream_errors.contains("Variable 'upstream' is not defined"), + "response-stream target must diagnose the stream property root, got: {stream_errors}" + ); +} + +#[test] +fn container_property_is_defined_on_the_concrete_file_branch() { + // The analyzer recognizes `line value` as a property while it is visiting + // `Writer`, then restores its own container context before the typechecker + // revisits the method. Branch-specific definedness must use the + // typechecker's live container context rather than falsely rejecting the + // valid classic file-write reading. + let code = "create container Writer:\n\ + \x20\x20\x20\x20property line value: Text\n\ + \x20\x20\x20\x20action dump:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20write line value to \"C:/tmp/out\"\n\ + \x20\x20\x20\x20end\n\ + end"; + assert!( + typecheck(code).is_ok(), + "a container property used by the selected file-write branch must remain defined: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn response_stream_branch_rejects_an_undefined_stream_lead() { + // Only the classic merged lead (`line value`) exists. Because `out` is a + // concrete ResponseStream, runtime selects the stream reading (`value`), + // which must still be rejected as undefined. + let code = "store line value as \"legacy file payload\"\n\ + listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 as out\n\ + write line value to out"; + let errors = typecheck(code).expect_err("undefined stream lead must be rejected"); + assert!( + errors.contains("Variable 'value' is not defined"), + "expected the selected stream-lead diagnostic, got: {errors}" + ); +} + +#[test] +fn action_body_stream_binding_remains_concrete_for_payload_checking() { + let errors = typecheck( + "define action called handle with parameters request_value:\n\ + start streaming response to request_value with status 200 as out\n\ + store items as [1 and 2]\n\ + store line items as \"classic\"\n\ + write line items to out\n\ + end action", + ) + .expect_err("an action-local ResponseStream must reject a List payload"); + assert!( + errors.contains("can only send text, binary, a number, or a boolean"), + "expected the action-body stream-payload diagnostic, got: {errors}" + ); +} + +#[test] +fn action_local_is_defined_on_the_concrete_file_branch() { + let code = "define action called dump:\n\ + \x20\x20\x20\x20store line value as \"action-local\"\n\ + \x20\x20\x20\x20write line value to \"C:/tmp/out\"\n\ + end action"; + assert!( + typecheck(code).is_ok(), + "an action-local merged lead must remain visible during branch checking: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn main_loop_local_is_defined_on_the_concrete_file_branch() { + let code = "main loop:\n\ + \x20\x20\x20\x20store line value as \"loop-local\"\n\ + \x20\x20\x20\x20write line value to \"C:/tmp/out\"\n\ + end loop"; + assert!( + typecheck(code).is_ok(), + "a main-loop-local merged lead must remain visible during branch checking: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn wrapped_missing_name_is_rejected_on_the_concrete_file_branch() { + let errors = typecheck( + "store line value as \"prefix\"\n\ + write line value with file exists at missing_path to \"C:/tmp/wfl_wrapped_out\"", + ) + .expect_err("the selected classic branch must validate names inside FileExists"); + assert!( + errors.contains("Variable 'missing_path' is not defined"), + "expected the wrapped path diagnostic, got: {errors}" + ); +} + +#[test] +fn wrapped_missing_name_is_rejected_on_the_concrete_stream_branch() { + let errors = typecheck( + "store value as \"prefix\"\n\ + listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 as out\n\ + write line value with file exists at missing_path to out", + ) + .expect_err("the selected stream branch must validate names inside FileExists"); + assert!( + errors.contains("Variable 'missing_path' is not defined"), + "expected the wrapped path diagnostic, got: {errors}" + ); +} + +#[test] +fn wrapped_missing_name_is_rejected_on_every_gradual_branch() { + let errors = typecheck( + "define action called send with parameters destination:\n\ + store value as \"stream\"\n\ + store line value as \"classic\"\n\ + write line value with file exists at missing_path to destination\n\ + end action", + ) + .expect_err("a gradual target must validate wrapped names on every viable branch"); + assert!( + errors.contains("Variable 'missing_path' is not defined"), + "expected the wrapped path diagnostic, got: {errors}" + ); +} + +#[test] +fn gradual_target_requires_both_candidate_leads_to_be_defined() { + let undefined_stream_lead = "define action called send with parameters target:\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to target\n\ + end action"; + let stream_errors = + typecheck(undefined_stream_lead).expect_err("the viable stream lead must be defined"); + assert!( + stream_errors.contains("Variable 'value' is not defined"), + "expected the gradual stream-lead diagnostic, got: {stream_errors}" + ); + + let undefined_classic_lead = "define action called send with parameters target:\n\ + \x20\x20\x20\x20store value as \"stream\"\n\ + \x20\x20\x20\x20write line value to target\n\ + end action"; + let classic_errors = + typecheck(undefined_classic_lead).expect_err("the viable classic lead must be defined"); + assert!( + classic_errors.contains("Variable 'line value' is not defined"), + "expected the gradual classic-lead diagnostic, got: {classic_errors}" + ); +} + +#[test] +fn gradual_target_validates_every_viable_payload_branch() { + let invalid_stream_payload = "define action called send with parameters target:\n\ + \x20\x20\x20\x20store value as [1 and 2]\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to target\n\ + end action"; + let payload_errors = + typecheck(invalid_stream_payload).expect_err("the viable stream payload must be valid"); + assert!( + payload_errors.contains("can only send text, binary, a number, or a boolean"), + "expected the gradual stream-payload diagnostic, got: {payload_errors}" + ); + + let both_valid = "define action called send with parameters target:\n\ + \x20\x20\x20\x20store value as \"stream\"\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to target\n\ + end action"; + assert!( + typecheck(both_valid).is_ok(), + "both viable gradual branches are valid: {:?}", + typecheck(both_valid).err() + ); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 00000000..fd84316a --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1,21 @@ +//! Shared helpers for integration tests. +#![allow(dead_code)] + +use std::net::TcpListener; + +/// Ask the OS for a currently-free TCP port on loopback, then release it so the +/// caller can bind it via WFL's `listen on port `. +/// +/// WFL takes a *literal* port in `listen on port `, so the port must be chosen +/// before the program source is built — we cannot bind an ephemeral `:0` and read +/// the assigned port back the way the mock upstreams do. Picking a free port from +/// the OS (instead of a hardcoded constant) avoids collisions under parallel test +/// runs and on busy runners. A small TOCTOU window remains between releasing the +/// probe socket and WFL re-binding it, but it is far less flaky than a fixed port. +pub fn free_tcp_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .expect("bind an ephemeral TCP port") + .local_addr() + .expect("read the ephemeral local address") + .port() +} diff --git a/tests/concurrent_disconnect_burst_test.rs b/tests/concurrent_disconnect_burst_test.rs new file mode 100644 index 00000000..e07fe18d --- /dev/null +++ b/tests/concurrent_disconnect_burst_test.rs @@ -0,0 +1,250 @@ +//! Real-socket regression for P1 (#2): a BURST of downstream (browser) +//! disconnects must NOT tear down the concurrent `main loop`. +//! +//! A client disconnect is an EXPECTED, normal cancellation of one handler — not a +//! handler *failure*. The concurrent loop keeps a single global consecutive- +//! failure counter that backs off after every failed handler and breaks the whole +//! loop once it reaches `MAX_CONSECUTIVE_FAILURES` (256). If each disconnect is +//! (mis)counted as a failure, then 256 disconnects with no interleaved success +//! trip that structural breaker and the server stops serving entirely — so an +//! ordinary "client hung up" event, repeated, becomes a denial of service. +//! +//! Topology: a stalling mock upstream <- WFL concurrent proxy -> many short-lived +//! clients. Each client makes the proxy open the (stalling) upstream, start a +//! streaming response, and block reading the upstream; the client then reads the +//! response head and disconnects, cancelling that handler. After a burst of >256 +//! such disconnects (more than the breaker threshold, with NO successful request +//! in between), an unrelated `/ping` request must still be served — proving the +//! loop survived the burst. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::Semaphore; +use tokio::sync::mpsc; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// How many disconnecting clients to fire. Must exceed the concurrent loop's +/// `MAX_CONSECUTIVE_FAILURES` (256) so, under the buggy behavior, the burst trips +/// the structural breaker. +const DISCONNECT_BURST: usize = 270; +/// Wait for at least this many upstream closes (== handler disconnects) before +/// probing, guaranteeing the burst has driven the breaker past its threshold. +const CLOSES_BEFORE_PROBE: usize = 256; +/// Bounded client concurrency: well under the 256 handler cap and the request +/// queue bound, so no request is shed with 503. +const CLIENT_CONCURRENCY: usize = 48; + +/// Mock upstream: for each connection, send a chunked head then STALL (send no +/// body). Signal on `closes` every time a connection is observed closing — which +/// only happens when the proxy handler cancels its upstream (client disconnect). +async fn spawn_counting_stall_upstream() -> (u16, mpsc::UnboundedReceiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (closes_tx, closes_rx) = mpsc::unbounded_channel(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let closes_tx = closes_tx.clone(); + tokio::spawn(async move { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + // Stall: never send a body chunk. Block reading; when the proxy + // cancels the upstream (its client disconnected), the peer close + // surfaces here as Ok(0)/Err. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = closes_tx.send(()); + return; + } + Ok(_) => {} + } + } + }); + } + }); + (port, closes_rx) +} + +fn start_proxy_server(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + if let Err(errors) = interp.interpret(&ast).await { + panic!("proxy interpreter failed: {errors:?}"); + } + }); + }) +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("proxy server on {addr} did not become ready"); +} + +/// One disconnecting client: open `/proxy`, read the response head (proving the +/// handler reached `start streaming response` and is now blocked on the upstream), +/// then drop the socket to disconnect. +async fn fire_disconnect(proxy_port: u16) { + let Ok(mut sock) = tokio::net::TcpStream::connect(("127.0.0.1", proxy_port)).await else { + return; + }; + let req = "GET /proxy HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"; + if sock.write_all(req.as_bytes()).await.is_err() { + return; + } + // Read until the end of the response head (\r\n\r\n) so the handler has an + // open response stream when we disconnect (that is what makes the disconnect + // observable to the blocked upstream read). + let mut acc = Vec::new(); + let mut tmp = [0u8; 256]; + loop { + match tokio::time::timeout(Duration::from_secs(5), sock.read(&mut tmp)).await { + Ok(Ok(0)) | Err(_) => break, + Ok(Ok(n)) => { + acc.extend_from_slice(&tmp[..n]); + if acc.windows(4).any(|w| w == b"\r\n\r\n") { + break; + } + } + Ok(Err(_)) => break, + } + } + // Drop `sock` -> disconnect while the handler is blocked reading the upstream. +} + +#[tokio::test] +async fn test_disconnect_burst_does_not_kill_concurrent_loop() { + let (upstream_port, mut upstream_closes) = spawn_counting_stall_upstream().await; + + let proxy_port = common::free_tcp_port(); + let code = format!( + r#" + listen on port {proxy_port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{upstream_port}/" and stream response as up + start streaming response to req with status 200 and content type "text/plain" as down + count from 1 to 100000: + wait for next chunk from up as c + check if c is nothing: + break + otherwise: + write chunk c to down + end check + end count + end check + end check + end loop + "# + ); + let server = start_proxy_server(code); + wait_for_server(proxy_port).await; + + // Fire a burst of disconnecting clients, bounded so none is shed with 503. + let sem = Arc::new(Semaphore::new(CLIENT_CONCURRENCY)); + let fired = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::with_capacity(DISCONNECT_BURST); + for _ in 0..DISCONNECT_BURST { + let sem = Arc::clone(&sem); + let fired = Arc::clone(&fired); + tasks.push(tokio::spawn(async move { + let _permit = sem.acquire().await.expect("semaphore"); + fire_disconnect(proxy_port).await; + fired.fetch_add(1, Ordering::Relaxed); + })); + } + + // Wait until the mock has seen enough upstream closes to guarantee the burst + // drove the buggy breaker past its 256-failure threshold (no `/ping` sent yet, + // so every one of these is a "consecutive failure" under the old behavior). + let mut closed = 0usize; + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + while closed < CLOSES_BEFORE_PROBE { + match tokio::time::timeout_at(deadline, upstream_closes.recv()).await { + Ok(Some(())) => closed += 1, + Ok(None) => break, + Err(_) => panic!( + "only observed {closed} upstream closes before timeout (expected {CLOSES_BEFORE_PROBE}); \ + the disconnect burst did not fully drive the handlers" + ), + } + } + assert!( + closed >= CLOSES_BEFORE_PROBE, + "expected at least {CLOSES_BEFORE_PROBE} upstream closes, saw {closed}" + ); + + // Grace so the loop finishes counting the final failure (and, under the bug, + // actually breaks) before we probe. + tokio::time::sleep(Duration::from_millis(750)).await; + + // The unrelated `/ping` MUST still be served. Under the bug the loop has torn + // itself down after 256 "failures" and this hangs / is refused. + let ping = tokio::time::timeout( + Duration::from_secs(10), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/ping")) + .send(), + ) + .await + .expect("`/ping` timed out after the disconnect burst — concurrent loop was torn down") + .expect("`/ping` request failed after the disconnect burst"); + assert_eq!( + ping.status().as_u16(), + 200, + "`/ping` should be served after the disconnect burst" + ); + let body = ping.text().await.expect("read /ping body"); + assert_eq!( + body, "pong", + "`/ping` should return the live handler's response" + ); + + // Shut the server down and join everything. + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/shutdown")) + .send() + .await; + for t in tasks { + let _ = t.await; + } + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("server join task failed: {e}"), + } +} diff --git a/tests/concurrent_disconnect_paths_burst_test.rs b/tests/concurrent_disconnect_paths_burst_test.rs new file mode 100644 index 00000000..d4f9dba4 --- /dev/null +++ b/tests/concurrent_disconnect_paths_burst_test.rs @@ -0,0 +1,937 @@ +//! Real-socket regression (maintainer re-review, P1 / #642): EVERY transport-confirmed +//! client disconnect must be classified as a cancellation, not a handler failure — +//! including the buffered `respond` send, the streaming-response head path (before the +//! head is sent), and the streaming write path — not only a cancelled upstream chunk +//! read. +//! +//! The concurrent loop breaks after `MAX_CONSECUTIVE_FAILURES` (256) consecutive +//! *structural* failures. Request-local outcomes (disconnects, wait timeouts, errors +//! after a request was accepted) must never feed that breaker. These bursts drive +//! more than 256 disconnects of each kind with no successful request in between; an +//! unrelated `/ping` must still be served afterward. +//! +//! Also: every client that is intended to exercise a path must actually connect and +//! reach that lifecycle point (no silent early-return that leaves the burst under the +//! breaker threshold), and an explicit handler-start barrier proves every intended +//! result was consumed before probing `/ping` (so a General-classified disconnect +//! cannot race past a premature success that resets the counter). +//! Exact `ErrorKind::Cancelled` assertions remain at the interpreter unit layer; these +//! real-boundary bursts prove the externally observable breaker and liveness contract. + +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; +use tempfile::TempDir; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::{mpsc, watch}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// Exactly fill the concurrent-handler cap. A second full wave cannot reach its +/// lifecycle checkpoint until every result from the first wave has been consumed. +const DISCONNECT_WAVE: usize = 256; +/// Both waves disconnect before `/ping`, so each path exercises 512 clients (>256). +const DISCONNECT_TOTAL: usize = DISCONNECT_WAVE * 2; +const _: () = assert!(DISCONNECT_TOTAL > 256); +const WAVE_DEADLINE: Duration = Duration::from_secs(30); +const ITERATION_PROOF_DEADLINE: Duration = Duration::from_secs(20); + +/// These cases deliberately fill the 256-handler cap. Rust's test harness otherwise +/// runs all four cases in this binary in parallel, multiplying the live socket/task +/// peak. Serializing the heavyweight cases keeps the test-host resource bound at one +/// full handler wave while preserving the real >256-request breaker proof. +static HEAVY_CASE_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// A causal release latch visible to WFL through `file exists at`. +/// +/// The marker is created before a wave begins. Handlers poll its existence after +/// reaching the precise response lifecycle point under test. The test removes it +/// only after every browser task has returned, which is positive proof that all +/// intended client sockets were dropped. The short wait inside the WFL loop is only +/// a cooperative polling yield; marker removal, not elapsed time, releases handlers. +struct WaveLatch { + _directory: TempDir, + marker: PathBuf, + wfl_path: String, +} + +impl WaveLatch { + fn new(context: &str) -> Self { + let directory = tempfile::Builder::new() + .prefix("wfl-disconnect-release-") + .tempdir() + .unwrap_or_else(|error| panic!("{context}: create release-latch directory: {error}")); + let marker = directory.path().join("hold-wave"); + let wfl_path = marker + .to_string_lossy() + .replace('\\', "/") + .replace('"', "\\\""); + Self { + _directory: directory, + marker, + wfl_path, + } + } + + fn hold(&self, wave: usize, context: &str) { + assert!( + !self.marker.exists(), + "{context}: release marker unexpectedly existed before wave {wave}" + ); + std::fs::write(&self.marker, format!("hold {context} wave {wave}\n")).unwrap_or_else( + |error| panic!("{context}: create release marker for wave {wave}: {error}"), + ); + } + + fn release(&self, wave: usize, context: &str) { + std::fs::remove_file(&self.marker) + .unwrap_or_else(|error| panic!("{context}: release wave {wave}: {error}")); + } +} + +struct CountingGate { + port: u16, + arrivals: mpsc::UnboundedReceiver, + acknowledgements: mpsc::UnboundedReceiver, + errors: mpsc::UnboundedReceiver, + release_wave: watch::Sender, +} + +/// An HTTP checkpoint shared by the two request waves. Each handler opens the +/// checkpoint before its response operation. The mock reports all arrivals, then +/// withholds the HTTP head until the test releases that wave. This proves all 256 +/// handlers reached the intended path without relying on sleeps or successful TCP +/// writes alone. +async fn spawn_counting_gate() -> CountingGate { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind counting gate"); + let port = listener.local_addr().expect("counting gate address").port(); + let next_checkpoint_ordinal = Arc::new(AtomicUsize::new(0)); + let next_ack_ordinal = Arc::new(AtomicUsize::new(0)); + let (arrival_tx, arrivals) = mpsc::unbounded_channel(); + let (ack_tx, acknowledgements) = mpsc::unbounded_channel(); + let (error_tx, errors) = mpsc::unbounded_channel(); + let (release_wave, release_guard) = watch::channel(0usize); + + tokio::spawn(async move { + // Keep one receiver alive between waves so `release_wave.send()` cannot + // fail in the brief interval after the prior wave's connections close. + let release_guard = release_guard; + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let next_checkpoint_ordinal = Arc::clone(&next_checkpoint_ordinal); + let next_ack_ordinal = Arc::clone(&next_ack_ordinal); + let arrival_tx = arrival_tx.clone(); + let ack_tx = ack_tx.clone(); + let error_tx = error_tx.clone(); + let mut release_rx = release_guard.clone(); + tokio::spawn(async move { + let result = async { + let head = read_http_head(&mut sock).await?; + let request = String::from_utf8_lossy(&head); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .ok_or_else(|| { + "counting gate received a malformed request line".to_string() + })?; + match path { + "/checkpoint" => { + let ordinal = + next_checkpoint_ordinal.fetch_add(1, Ordering::Relaxed) + 1; + let wave = ((ordinal - 1) / DISCONNECT_WAVE) + 1; + arrival_tx + .send(ordinal) + .map_err(|_| "checkpoint arrival receiver dropped".to_string())?; + release_rx + .wait_for(|released_wave| *released_wave >= wave) + .await + .map_err(|_| "counting-gate release sender dropped".to_string())?; + sock.write_all( + b"HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + Connection: close\r\n\r\n", + ) + .await + .map_err(|error| format!("write checkpoint response: {error}"))?; + sock.flush() + .await + .map_err(|error| format!("flush checkpoint response: {error}")) + } + "/ack" => { + let ordinal = next_ack_ordinal.fetch_add(1, Ordering::Relaxed) + 1; + // Deliberately leave this chunked response unfinished. The + // handler reads the marker and explicitly closes its + // outbound stream; observing EOF below is an exact + // handler-side acknowledgement that the earlier + // `/checkpoint` open returned and the local post-checkpoint + // wait is now the next operation. + sock.write_all( + b"HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: keep-alive\r\n\r\n\ + 1\r\nA\r\n", + ) + .await + .map_err(|error| format!("write acknowledgement marker: {error}"))?; + sock.flush().await.map_err(|error| { + format!("flush acknowledgement marker: {error}") + })?; + let deadline = tokio::time::Instant::now() + WAVE_DEADLINE; + let mut byte = [0u8; 1]; + loop { + let read = tokio::time::timeout_at(deadline, sock.read(&mut byte)) + .await + .map_err(|_| { + format!( + "handler did not close acknowledgement stream {ordinal}" + ) + })? + .map_err(|error| { + format!("read acknowledgement close {ordinal}: {error}") + })?; + if read == 0 { + break; + } + } + ack_tx + .send(ordinal) + .map_err(|_| "handler acknowledgement receiver dropped".to_string()) + } + other => Err(format!("unexpected counting-gate path {other:?}")), + } + } + .await; + if let Err(error) = result { + let _ = error_tx.send(error); + } + }); + } + }); + + CountingGate { + port, + arrivals, + acknowledgements, + errors, + release_wave, + } +} + +async fn read_http_head(sock: &mut tokio::net::TcpStream) -> Result, String> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let mut head = Vec::new(); + let mut buf = [0u8; 512]; + loop { + let n = tokio::time::timeout_at(deadline, sock.read(&mut buf)) + .await + .map_err(|_| "timed out waiting for HTTP head".to_string())? + .map_err(|error| format!("read HTTP head: {error}"))?; + if n == 0 { + return Err("connection closed before the complete HTTP head".to_string()); + } + head.extend_from_slice(&buf[..n]); + if head.windows(4).any(|window| window == b"\r\n\r\n") { + return Ok(head); + } + if head.len() > 16 * 1024 { + return Err("HTTP head exceeded 16 KiB".to_string()); + } + } +} + +struct IterationCounter { + port: u16, + arrivals: mpsc::UnboundedReceiver, + errors: mpsc::UnboundedReceiver, + observed: HashSet, +} + +/// Count a handler-start request and return an empty response immediately. A +/// concurrent handler calls this before waiting for a request. Since the handler +/// then remains parked in `wait for request`, each later start is observable proof +/// that the outer loop consumed one prior handler result and refilled its slot. +async fn spawn_iteration_counter() -> IterationCounter { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind iteration counter"); + let port = listener + .local_addr() + .expect("iteration counter address") + .port(); + let ordinal = Arc::new(AtomicUsize::new(0)); + let (arrival_tx, arrivals) = mpsc::unbounded_channel(); + let (error_tx, errors) = mpsc::unbounded_channel(); + + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let ordinal = Arc::clone(&ordinal); + let arrival_tx = arrival_tx.clone(); + let error_tx = error_tx.clone(); + tokio::spawn(async move { + let result = async { + let head = read_http_head(&mut sock).await?; + let request = String::from_utf8_lossy(&head); + if !request.starts_with("GET /tick ") { + return Err(format!( + "unexpected iteration-counter request: {:?}", + request.lines().next() + )); + } + let current = ordinal.fetch_add(1, Ordering::Relaxed) + 1; + arrival_tx + .send(current) + .map_err(|_| "iteration arrival receiver dropped".to_string())?; + sock.write_all( + b"HTTP/1.1 200 OK\r\n\ + Content-Length: 0\r\n\ + Connection: close\r\n\r\n", + ) + .await + .map_err(|error| format!("write iteration response: {error}"))?; + sock.flush() + .await + .map_err(|error| format!("flush iteration response: {error}")) + } + .await; + if let Err(error) = result { + let _ = error_tx.send(error); + } + }); + } + }); + + IterationCounter { + port, + arrivals, + errors, + observed: HashSet::new(), + } +} + +async fn wait_for_proven_handler_iterations( + counter: &mut IterationCounter, + expected: usize, + context: &str, +) { + let deadline = tokio::time::Instant::now() + ITERATION_PROOF_DEADLINE; + while !(1..=expected).all(|ordinal| counter.observed.contains(&ordinal)) { + let ordinal = tokio::time::timeout_at(deadline, async { + tokio::select! { + ordinal = counter.arrivals.recv() => { + ordinal.expect("iteration arrival channel closed") + } + error = counter.errors.recv() => { + panic!( + "iteration counter failed before proving {expected} {context} \ + handler starts: {}", + error.expect("iteration counter error channel closed") + ) + } + } + }) + .await + .unwrap_or_else(|_| { + let proven = (1..=expected) + .filter(|ordinal| counter.observed.contains(ordinal)) + .count(); + panic!( + "observed only {} of {expected} {context} handler starts; the \ + {expected}th start is required to prove the loop consumed every \ + intended result before the liveness probe", + proven + ) + }); + assert!( + counter.observed.insert(ordinal), + "iteration counter duplicated handler-start ordinal {ordinal}" + ); + } +} + +fn post_wave_iteration_target(wave: usize) -> usize { + // The loop initially fills all 256 slots. Each completed disconnect wave must + // then yield another 256 starts. Reaching this exact prefix proves every result + // from this wave left FuturesUnordered before the next wave or `/ping`. + DISCONNECT_WAVE * (wave + 1) +} + +async fn wait_for_gate_arrivals( + arrivals: &mut mpsc::UnboundedReceiver, + errors: &mut mpsc::UnboundedReceiver, + wave: usize, + context: &str, +) { + let first = (wave - 1) * DISCONNECT_WAVE + 1; + let last = wave * DISCONNECT_WAVE; + let deadline = tokio::time::Instant::now() + WAVE_DEADLINE; + let mut seen = HashSet::with_capacity(DISCONNECT_WAVE); + while seen.len() < DISCONNECT_WAVE { + let ordinal = tokio::time::timeout_at(deadline, async { + tokio::select! { + ordinal = arrivals.recv() => { + ordinal.expect("counting-gate arrival channel closed") + } + error = errors.recv() => { + panic!( + "{context} counting gate failed while waiting for wave {wave}: {}", + error.expect("counting-gate error channel closed") + ) + } + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "only {} of {DISCONNECT_WAVE} {context} handlers reached the \ + counting checkpoint in wave {wave}", + seen.len() + ) + }); + assert!( + (first..=last).contains(&ordinal), + "unexpected counting-gate ordinal {ordinal} while waiting for wave {wave} \ + ({first}..={last})" + ); + assert!( + seen.insert(ordinal), + "duplicate counting-gate arrival ordinal {ordinal}" + ); + } +} + +async fn wait_for_handler_acknowledgements( + acknowledgements: &mut mpsc::UnboundedReceiver, + errors: &mut mpsc::UnboundedReceiver, + wave: usize, + context: &str, +) { + let first = (wave - 1) * DISCONNECT_WAVE + 1; + let last = wave * DISCONNECT_WAVE; + let deadline = tokio::time::Instant::now() + WAVE_DEADLINE; + let mut seen = HashSet::with_capacity(DISCONNECT_WAVE); + while seen.len() < DISCONNECT_WAVE { + let ordinal = tokio::time::timeout_at(deadline, async { + tokio::select! { + ordinal = acknowledgements.recv() => { + ordinal.expect("handler acknowledgement channel closed") + } + error = errors.recv() => { + panic!( + "{context} counting gate failed while waiting for acknowledgements \ + in wave {wave}: {}", + error.expect("counting-gate error channel closed") + ) + } + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "only {} of {DISCONNECT_WAVE} {context} handlers acknowledged the \ + completed checkpoint in wave {wave}", + seen.len() + ) + }); + assert!( + (first..=last).contains(&ordinal), + "unexpected handler acknowledgement ordinal {ordinal} while waiting for wave {wave}" + ); + assert!( + seen.insert(ordinal), + "duplicate handler acknowledgement ordinal {ordinal}" + ); + } +} + +struct ProxyServer { + thread: std::thread::JoinHandle<()>, + abort: Option>, +} + +fn start_proxy_server(code: String) -> ProxyServer { + let (abort, abort_rx) = tokio::sync::oneshot::channel(); + let thread = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + tokio::select! { + result = interp.interpret(&ast) => { + if let Err(errors) = result { + panic!("server interpreter failed: {errors:?}"); + } + } + _ = abort_rx => { + // Test cleanup: dropping the interpret future closes listeners, + // pending responses, and response streams. + } + } + }); + }); + ProxyServer { + thread, + abort: Some(abort), + } +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready"); +} + +async fn hold_client_until_disconnect( + port: u16, + path: &'static str, + mut disconnect: watch::Receiver, +) -> Result<(), String> { + let mut sock = tokio::time::timeout( + Duration::from_secs(5), + tokio::net::TcpStream::connect(("127.0.0.1", port)), + ) + .await + .map_err(|_| format!("timed out connecting to {path}"))? + .map_err(|error| format!("connect {path}: {error}"))?; + let req = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + sock.write_all(req.as_bytes()) + .await + .map_err(|error| format!("send {path} request: {error}"))?; + sock.flush() + .await + .map_err(|error| format!("flush {path} request: {error}"))?; + disconnect + .wait_for(|should_disconnect| *should_disconnect) + .await + .map_err(|_| format!("{path} disconnect signal sender dropped"))?; + drop(sock); + Ok(()) +} + +async fn disconnect_after_stream_head(port: u16, path: &'static str) -> Result<(), String> { + let mut sock = tokio::time::timeout( + Duration::from_secs(5), + tokio::net::TcpStream::connect(("127.0.0.1", port)), + ) + .await + .map_err(|_| format!("timed out connecting to {path}"))? + .map_err(|error| format!("connect {path}: {error}"))?; + let req = format!("GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + sock.write_all(req.as_bytes()) + .await + .map_err(|error| format!("send {path} request: {error}"))?; + sock.flush() + .await + .map_err(|error| format!("flush {path} request: {error}"))?; + let head = read_http_head(&mut sock).await?; + let head = String::from_utf8_lossy(&head); + assert!( + head.starts_with("HTTP/1.1 200"), + "{path} must reach a successful streaming response head before disconnect; \ + got {head:?}" + ); + drop(sock); + Ok(()) +} + +fn spawn_gated_client_wave( + port: u16, + path: &'static str, +) -> ( + watch::Sender, + Vec>>, +) { + let (disconnect, disconnect_rx) = watch::channel(false); + let mut tasks = Vec::with_capacity(DISCONNECT_WAVE); + for _ in 0..DISCONNECT_WAVE { + let disconnect_rx = disconnect_rx.clone(); + tasks.push(tokio::spawn(async move { + hold_client_until_disconnect(port, path, disconnect_rx).await + })); + } + (disconnect, tasks) +} + +fn spawn_stream_client_wave( + port: u16, + path: &'static str, +) -> Vec>> { + (0..DISCONNECT_WAVE) + .map(|_| tokio::spawn(disconnect_after_stream_head(port, path))) + .collect() +} + +async fn join_client_wave( + tasks: Vec>>, + wave: usize, + context: &str, +) { + assert_eq!( + tasks.len(), + DISCONNECT_WAVE, + "each {context} wave must contain exactly {DISCONNECT_WAVE} clients" + ); + tokio::time::timeout(WAVE_DEADLINE, async move { + for (index, task) in tasks.into_iter().enumerate() { + let result = task + .await + .unwrap_or_else(|error| panic!("{context} client task {index} panicked: {error}")); + result.unwrap_or_else(|error| { + panic!("{context} client task {index} failed in wave {wave}: {error}") + }); + } + }) + .await + .unwrap_or_else(|_| panic!("{context} client joins timed out in wave {wave}")); +} + +/// Drive two full checkpointed waves. All 256 first-wave handlers are held inside +/// the checkpoint simultaneously. The second wave cannot put all 256 handlers into +/// that checkpoint until the loop has consumed every first-wave result. The marker +/// latch remains held until all 256 browser tasks confirm their sockets are dropped; +/// after release, an exact handler-start prefix proves this wave's results were +/// consumed before the next wave or `/ping`. +async fn drive_two_gated_disconnect_waves( + port: u16, + path: &'static str, + gate: &mut CountingGate, + latch: &WaveLatch, + iterations: &mut IterationCounter, + context: &str, +) { + for wave in 1..=2 { + latch.hold(wave, context); + let (disconnect, clients) = spawn_gated_client_wave(port, path); + wait_for_gate_arrivals(&mut gate.arrivals, &mut gate.errors, wave, context).await; + gate.release_wave + .send(wave) + .expect("counting-gate release receiver stays alive"); + wait_for_handler_acknowledgements( + &mut gate.acknowledgements, + &mut gate.errors, + wave, + context, + ) + .await; + disconnect + .send(true) + .expect("all gated clients remain alive until explicitly disconnected"); + join_client_wave(clients, wave, context).await; + latch.release(wave, context); + wait_for_proven_handler_iterations(iterations, post_wave_iteration_target(wave), context) + .await; + } +} + +/// Receiving a valid streaming head is the lifecycle proof for the write path. +/// Every browser task drops its socket while the marker remains held. Only after +/// all 256 tasks return does the test release the handlers to write, then the +/// iteration prefix proves all results were consumed. +async fn drive_two_stream_disconnect_waves( + port: u16, + path: &'static str, + latch: &WaveLatch, + iterations: &mut IterationCounter, + context: &str, +) { + for wave in 1..=2 { + latch.hold(wave, context); + let clients = spawn_stream_client_wave(port, path); + join_client_wave(clients, wave, context).await; + latch.release(wave, context); + wait_for_proven_handler_iterations(iterations, post_wave_iteration_target(wave), context) + .await; + } +} + +async fn assert_ping_survives(port: u16, context: &str) { + let ping = tokio::time::timeout( + Duration::from_secs(10), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/ping")) + .send(), + ) + .await + .unwrap_or_else(|_| panic!("`/ping` timed out after the {context} burst — loop torn down")) + .expect("`/ping` request failed"); + assert_eq!( + ping.status().as_u16(), + 200, + "`/ping` should be served after the {context} burst" + ); + assert_eq!(ping.text().await.unwrap(), "pong"); +} + +async fn shutdown(port: u16, mut server: ProxyServer) { + let _ = tokio::time::timeout( + Duration::from_secs(10), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/shutdown")) + .send(), + ) + .await; + let graceful = tokio::time::timeout(Duration::from_secs(10), async { + while !server.thread.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .is_ok(); + if !graceful { + let _ = server + .abort + .take() + .expect("proxy abort signal is sent at most once") + .send(()); + tokio::time::timeout(Duration::from_secs(5), async { + while !server.thread.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("proxy server ignored both graceful shutdown and forced test cleanup"); + } + match server.thread.join() { + Ok(()) => {} + Err(panic) => std::panic::resume_unwind(panic), + } +} + +#[tokio::test] +async fn test_disconnect_before_buffered_respond_does_not_kill_the_loop() { + let _heavy_case = HEAVY_CASE_LOCK.lock().await; + let mut gate = spawn_counting_gate().await; + let mut iterations = spawn_iteration_counter().await; + let latch = WaveLatch::new("buffered-respond disconnect"); + let port = common::free_tcp_port(); + // `/slow` reaches the counting checkpoint and then parks on a filesystem + // marker. The test removes that marker only after every client task confirms + // its socket was dropped, so the subsequent buffered response deterministically + // sees a disconnected receiver. + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + open url at "http://127.0.0.1:{counter_port}/tick" and stream response as iteration + close iteration + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{gate_port}/checkpoint" and stream response as checkpoint + close checkpoint + open url at "http://127.0.0.1:{gate_port}/ack" and stream response as acknowledgement + wait for next chunk from acknowledgement as acknowledged + close acknowledgement + repeat while file exists at "{release_path}": + wait for 1 milliseconds + end repeat + respond to req with "late" + end check + end check + end loop + "#, + gate_port = gate.port, + counter_port = iterations.port, + release_path = latch.wfl_path.as_str(), + ); + let server = start_proxy_server(code); + wait_for_server(port).await; + drive_two_gated_disconnect_waves( + port, + "/slow", + &mut gate, + &latch, + &mut iterations, + "buffered-respond disconnect", + ) + .await; + assert_ping_survives(port, "buffered-respond disconnect").await; + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_disconnect_before_stream_write_does_not_kill_the_loop() { + let _heavy_case = HEAVY_CASE_LOCK.lock().await; + let mut iterations = spawn_iteration_counter().await; + let latch = WaveLatch::new("stream-write disconnect"); + let port = common::free_tcp_port(); + // `/stream` sends the head and parks on a filesystem marker. Each client reads + // that head and drops its socket; only after all client tasks return does the + // test remove the marker and let the handler attempt its writes. + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + open url at "http://127.0.0.1:{counter_port}/tick" and stream response as iteration + close iteration + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + start streaming response to req with status 200 and content type "text/plain" as out + repeat while file exists at "{release_path}": + wait for 1 milliseconds + end repeat + store payload as "0123456789" + count from 1 to 9: + store payload as payload with payload + end count + count from 1 to 400: + write chunk payload to out + end count + close out + end check + end check + end loop + "#, + counter_port = iterations.port, + release_path = latch.wfl_path.as_str(), + ); + let server = start_proxy_server(code); + wait_for_server(port).await; + drive_two_stream_disconnect_waves( + port, + "/stream", + &latch, + &mut iterations, + "stream-write-disconnect", + ) + .await; + assert_ping_survives(port, "stream-write disconnect").await; + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_disconnect_before_streaming_head_does_not_kill_the_loop() { + let _heavy_case = HEAVY_CASE_LOCK.lock().await; + let mut gate = spawn_counting_gate().await; + let mut iterations = spawn_iteration_counter().await; + let latch = WaveLatch::new("pre-streaming-head disconnect"); + let port = common::free_tcp_port(); + // Client disconnects *before* the streaming head is sent (no head read). The + // handler reaches the counting checkpoint and then parks on a marker. The test + // releases it only after every client socket is confirmed dropped, so + // `start streaming response` deterministically sees the disconnected request. + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + open url at "http://127.0.0.1:{counter_port}/tick" and stream response as iteration + close iteration + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{gate_port}/checkpoint" and stream response as checkpoint + close checkpoint + open url at "http://127.0.0.1:{gate_port}/ack" and stream response as acknowledgement + wait for next chunk from acknowledgement as acknowledged + close acknowledgement + repeat while file exists at "{release_path}": + wait for 1 milliseconds + end repeat + start streaming response to req with status 200 and content type "text/plain" as out + write line "late" to out + close out + end check + end check + end loop + "#, + gate_port = gate.port, + counter_port = iterations.port, + release_path = latch.wfl_path.as_str(), + ); + let server = start_proxy_server(code); + wait_for_server(port).await; + drive_two_gated_disconnect_waves( + port, + "/prehead", + &mut gate, + &latch, + &mut iterations, + "pre-streaming-head disconnect", + ) + .await; + assert_ping_survives(port, "pre-streaming-head disconnect").await; + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_repeated_wait_timeouts_do_not_kill_the_loop() { + let _heavy_case = HEAVY_CASE_LOCK.lock().await; + let mut counter = spawn_iteration_counter().await; + let port = common::free_tcp_port(); + // Finite `wait for request ... with timeout` that repeatedly expires with no + // client traffic must not trip the structural breaker. After many idle + // timeouts a real `/ping` must still be served. + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + open url at "http://127.0.0.1:{counter_port}/tick" and stream response as tick + close tick + wait for request comes in on srv as req with timeout 1 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + respond to req with "ok" + end check + end check + end loop + "#, + counter_port = counter.port, + ); + let server = start_proxy_server(code); + wait_for_server(port).await; + // The loop initially starts 256 handlers. It can start handler 512 only after + // consuming 256 finite request-wait expiries and refilling once more. A buggy + // structural classifier breaks while consuming expiry 256 and tops out at 511, + // so this is an observable threshold proof rather than an elapsed-time proxy. + wait_for_proven_handler_iterations(&mut counter, DISCONNECT_TOTAL, "finite-timeout").await; + assert_ping_survives(port, "repeated wait timeouts").await; + shutdown(port, server).await; +} diff --git a/tests/concurrent_main_loop_test.rs b/tests/concurrent_main_loop_test.rs new file mode 100644 index 00000000..9648b055 --- /dev/null +++ b/tests/concurrent_main_loop_test.rs @@ -0,0 +1,388 @@ +// Tests for `main loop concurrently:` (concurrent request handlers). +// +// Key properties: +// - `main loop concurrently:` parses (concurrent = true); plain `main loop:` +// stays serial (concurrent = false) — no silent upgrade. +// - Concurrent: a slow handler does NOT block a fast sibling. +// - Serial: a slow handler DOES block the next request (unchanged behavior). +// - A handler that errors is contained; the server keeps serving. +// +// Each server exposes a `/shutdown` path that closes the server and breaks the +// loop, so the test can stop the server thread deterministically and join it. + +use std::time::{Duration, Instant}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::Statement; + +mod common; + +fn parse_program(code: &str) -> Vec { + let tokens = lex_wfl_with_positions(code); + let mut parser = Parser::new(&tokens); + parser + .parse() + .unwrap_or_else(|e| panic!("Parse error: {e:?}")) + .statements +} + +#[test] +fn test_main_loop_concurrently_parses_as_concurrent() { + let stmts = parse_program("main loop concurrently:\n display \"x\"\nend loop"); + match &stmts[0] { + Statement::MainLoop { concurrent, .. } => assert!(*concurrent), + other => panic!("Expected MainLoop, got {other:?}"), + } +} + +#[test] +fn test_plain_main_loop_stays_serial() { + let stmts = parse_program("main loop:\n display \"x\"\nend loop"); + match &stmts[0] { + Statement::MainLoop { concurrent, .. } => { + assert!(!*concurrent, "plain main loop must remain serial") + } + other => panic!("Expected MainLoop, got {other:?}"), + } +} + +fn start_server_thread(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + let ast = parser.parse().expect("parse"); + let mut interpreter = Interpreter::new(); + // Surface an unexpected interpreter error as a thread panic so + // `shutdown` re-raises it instead of the test silently passing. + if let Err(errors) = interpreter.interpret(&ast).await { + panic!("server interpreter failed: {errors:?}"); + } + }); + }) +} + +/// Wait until the WFL server has actually bound `port` and is accepting +/// connections, rather than sleeping a fixed interval. A fixed sleep is flaky on +/// a loaded CI runner where binding can take longer than the guess, producing +/// spurious `Connection refused` failures. A bare TCP connect that drops +/// immediately is a safe readiness probe: warp accepts and closes it without +/// delivering an HTTP request to the handler. +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready in time"); +} + +fn server_code(port: u16, concurrently: bool) -> String { + let marker = if concurrently { " concurrently" } else { "" }; + format!( + r#" + listen on port {port} as srv + main loop{marker}: + wait for request comes in on srv as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + check if p is equal to "/slow": + wait for 500 milliseconds + respond to req with "slow" + otherwise: + respond to req with "fast" + end check + end check + end loop + "# + ) +} + +/// Send `/shutdown` so the server closes and its loop breaks, then join. +async fn shutdown(port: u16, server: std::thread::JoinHandle<()>) { + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/shutdown")) + .send() + .await; + // Re-raise a server-thread panic (or interpreter error) instead of dropping + // it, so a server-side failure fails the test loudly. + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(join_err) => panic!("server join task failed: {join_err}"), + } +} + +#[tokio::test] +async fn test_concurrent_slow_handler_does_not_block_fast() { + let port = common::free_tcp_port(); + let server = start_server_thread(server_code(port, true)); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + + // Kick off the slow request first and give it a moment to be dequeued. + let slow_url = format!("http://127.0.0.1:{port}/slow"); + let slow = + tokio::spawn(async move { reqwest::Client::new().get(&slow_url).send().await.unwrap() }); + tokio::time::sleep(Duration::from_millis(80)).await; + + // The fast request must complete promptly even though /slow is mid-handler. + let t0 = Instant::now(); + let fast = client + .get(format!("http://127.0.0.1:{port}/fast")) + .send() + .await + .expect("fast request failed"); + let fast_elapsed = t0.elapsed(); + let fast_body = fast.text().await.unwrap(); + + assert_eq!(fast_body, "fast"); + assert!( + fast_elapsed < Duration::from_millis(300), + "fast request was blocked behind the slow handler ({fast_elapsed:?})" + ); + + let slow_resp = slow.await.expect("slow request task panicked"); + assert_eq!(slow_resp.text().await.unwrap(), "slow"); + + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_concurrent_handler_error_does_not_kill_server() { + // A handler that errors mid-iteration (here: responding twice) must be + // contained — the concurrent loop keeps serving other requests. + let port = common::free_tcp_port(); + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + check if p is equal to "/boom": + respond to req with "boom-ok" + respond to req with "this second respond errors" + otherwise: + respond to req with "ok" + end check + end check + end loop + "# + ); + let server = start_server_thread(code); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + + // The erroring handler still delivered its first response. + let boom = client + .get(format!("http://127.0.0.1:{port}/boom")) + .send() + .await + .expect("boom request failed"); + assert_eq!(boom.text().await.unwrap(), "boom-ok"); + + // The server survived the caught error and keeps serving. + let ok = client + .get(format!("http://127.0.0.1:{port}/ok")) + .send() + .await + .expect("follow-up request failed"); + assert_eq!(ok.text().await.unwrap(), "ok"); + + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_concurrent_handlers_do_not_share_count_loop_state() { + // Per-handler run-state isolation (P1 #1). Two concurrent handlers each run + // a `count` loop that yields (via `wait for`) mid-iteration and then reads + // `count`. The interpreter's count-loop state (`current_count`, + // `in_count_loop`) is a single shared field; without per-poll isolation, one + // handler's `count` bleeds into the other across the yield. + // + // The two ranges are disjoint (1..5 vs 100..104), so any cross-contamination + // is unmistakable: with isolation each handler observes only its own range. + let port = common::free_tcp_port(); + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + check if p is equal to "/a": + store seen as "" + count from 1 to 5: + wait for 80 milliseconds + change seen to seen with count with "-" + end count + respond to req with seen + otherwise: + store seen as "" + count from 100 to 104: + wait for 80 milliseconds + change seen to seen with count with "-" + end count + respond to req with seen + end check + end check + end loop + "# + ); + let server = start_server_thread(code); + wait_for_server(port).await; + + let a_url = format!("http://127.0.0.1:{port}/a"); + let b_url = format!("http://127.0.0.1:{port}/b"); + // Fire both at once so their count loops interleave on the single thread. + let a = tokio::spawn(async move { + reqwest::Client::new() + .get(&a_url) + .send() + .await + .unwrap() + .text() + .await + .unwrap() + }); + let b = tokio::spawn(async move { + reqwest::Client::new() + .get(&b_url) + .send() + .await + .unwrap() + .text() + .await + .unwrap() + }); + + let a_body = a.await.expect("/a task panicked"); + let b_body = b.await.expect("/b task panicked"); + + assert_eq!( + a_body, "1-2-3-4-5-", + "/a handler observed a count from outside its own loop (shared count-loop state)" + ); + assert_eq!( + b_body, "100-101-102-103-104-", + "/b handler observed a count from outside its own loop (shared count-loop state)" + ); + + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_handler_that_never_responds_gets_immediate_500() { + // Lifecycle guarantee (P1 #3): a handler that dequeues a request and ends + // WITHOUT responding must resolve the client with 500 immediately — not leave + // it waiting out the request timeout. The `/drop` path does no `respond`; the + // handler simply ends, and the client must still get a prompt 500. + let port = common::free_tcp_port(); + let code = format!( + r#" + listen on port {port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + check if p is equal to "/drop": + store ignored as "handler returns without responding" + otherwise: + respond to req with "ok" + end check + end check + end loop + "# + ); + let server = start_server_thread(code); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + + // The un-answered request resolves promptly with 500 rather than hanging. + let t0 = Instant::now(); + let dropped = client + .get(format!("http://127.0.0.1:{port}/drop")) + .send() + .await + .expect("/drop request failed"); + let elapsed = t0.elapsed(); + assert_eq!( + dropped.status().as_u16(), + 500, + "a handler that never responds must yield 500" + ); + assert!( + elapsed < Duration::from_secs(5), + "500 should arrive immediately on handler exit, not after the request timeout ({elapsed:?})" + ); + + // The server survived and keeps serving. + let ok = client + .get(format!("http://127.0.0.1:{port}/ok")) + .send() + .await + .expect("follow-up request failed"); + assert_eq!(ok.text().await.unwrap(), "ok"); + + shutdown(port, server).await; +} + +#[tokio::test] +async fn test_serial_slow_handler_blocks_next() { + let port = common::free_tcp_port(); + let server = start_server_thread(server_code(port, false)); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + + let slow_url = format!("http://127.0.0.1:{port}/slow"); + let slow = + tokio::spawn(async move { reqwest::Client::new().get(&slow_url).send().await.unwrap() }); + tokio::time::sleep(Duration::from_millis(80)).await; + + // On the serial loop the fast request cannot be handled until the slow + // handler finishes, so it is delayed by roughly the slow handler's duration. + let t0 = Instant::now(); + let fast = client + .get(format!("http://127.0.0.1:{port}/fast")) + .send() + .await + .expect("fast request failed"); + let fast_elapsed = t0.elapsed(); + let fast_body = fast.text().await.unwrap(); + + assert_eq!(fast_body, "fast"); + assert!( + fast_elapsed > Duration::from_millis(300), + "serial main loop should have blocked the fast request behind the slow one ({fast_elapsed:?})" + ); + + let slow_resp = slow.await.expect("slow request task panicked"); + assert_eq!(slow_resp.text().await.unwrap(), "slow"); + + shutdown(port, server).await; +} diff --git a/tests/concurrent_prehead_prune_race_test.rs b/tests/concurrent_prehead_prune_race_test.rs new file mode 100644 index 00000000..7d99cb62 --- /dev/null +++ b/tests/concurrent_prehead_prune_race_test.rs @@ -0,0 +1,167 @@ +//! Real-socket regression (maintainer re-review, P1): a sibling handler's +//! `wait for request` global prune must NOT erase a parked handler's pre-head +//! cancellation signal. +//! +//! Handler A blocks opening a header-stalled upstream BEFORE `start streaming +//! response`, so its only disconnect signal is its pending request's oneshot. When +//! A's client disconnects, A's pending entry becomes closed — but any later +//! `wait for request` prunes ALL closed entries. If a sibling prunes A's entry +//! before A's ~20ms poll notices, A's owned id is then simply absent from the map; +//! treating "absent" as "still connected" left A parked until its read timeout. +//! +//! This drives the race: a continuous stream of pruning `/kick` requests runs while +//! A's client disconnects, so a prune reliably removes A's closed entry in the poll +//! gap. With the fix (absent owned id == disconnected), A is cancelled promptly and +//! the upstream closes well before the idle timeout; without it, A hangs to timeout. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// Upstream: accept ONE connection, read the request, then WITHHOLD the response +/// head. Signal when the proxy drops the connection (peer close => read 0/Err). +async fn spawn_header_withholding_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready"); +} + +#[tokio::test] +async fn test_sibling_prune_does_not_strand_a_pre_head_disconnect() { + let (upstream_port, mut upstream_closed) = spawn_header_withholding_upstream().await; + let proxy_port = common::free_tcp_port(); + + let code = format!( + r#" + listen on port {proxy_port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/kick": + respond to req with "ok" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{upstream_port}/" and stream response as up + start streaming response to req with status 200 and content type "text/plain" as down + wait for next chunk from up as c + close down + end check + end check + end loop + "# + ); + + // 4s idle timeout: with the fix the stranded handler is cancelled in ~20ms once + // its entry is pruned; without it, it hangs until this timeout. + let server = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 4, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + let _ = interp.interpret(&ast).await; + }); + }); + wait_for_server(proxy_port).await; + + // Continuous pruning traffic: every `/kick` runs `wait for request`, which prunes + // all closed pending entries. Keep it dense so a prune lands in A's poll gap. + let kick_stop = Arc::new(AtomicBool::new(false)); + let kick_stop2 = Arc::clone(&kick_stop); + let kicker = tokio::spawn(async move { + let client = reqwest::Client::new(); + while !kick_stop2.load(Ordering::Relaxed) { + let _ = client + .get(format!("http://127.0.0.1:{proxy_port}/kick")) + .timeout(Duration::from_secs(2)) + .send() + .await; + } + }); + + // Let the pruning traffic ramp up, then connect A, let it block opening the + // header-stalled upstream, and disconnect it while the prunes are flowing. + tokio::time::sleep(Duration::from_millis(200)).await; + { + let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", proxy_port)) + .await + .expect("connect A"); + sock.write_all(b"GET /proxy HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + .await + .expect("send A request"); + sock.flush().await.ok(); + tokio::time::sleep(Duration::from_millis(300)).await; + // `sock` drops -> A disconnects while pruning traffic flows. + } + + // A's upstream must close PROMPTLY (cancelled), not at the 4s idle timeout. + let start = Instant::now(); + tokio::time::timeout(Duration::from_secs(3), &mut upstream_closed) + .await + .expect("A's upstream was not cancelled after a sibling pruned its disconnected entry") + .expect("upstream close sender dropped"); + let elapsed = start.elapsed(); + assert!( + elapsed < Duration::from_secs(3), + "the stranded pre-head handler should be cancelled promptly once pruned, \ + not hang to the idle timeout; took {elapsed:?}" + ); + + kick_stop.store(true, Ordering::Relaxed); + let _ = kicker.await; + + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/shutdown")) + .timeout(Duration::from_secs(2)) + .send() + .await; + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("server join task failed: {e}"), + } +} diff --git a/tests/dropped_interpret_cleanup_test.rs b/tests/dropped_interpret_cleanup_test.rs new file mode 100644 index 00000000..f3171a35 --- /dev/null +++ b/tests/dropped_interpret_cleanup_test.rs @@ -0,0 +1,87 @@ +//! Real-socket regression for P1 (#4): a cancelled/dropped `interpret()` future +//! must still close the outbound stream handles the run opened. +//! +//! Handler-exit cleanup (concurrent `IsolatedHandler::drop`, serial-loop/program +//! cleanup sites) closes outbound handles on normal control-flow exits. But if the +//! whole `interpret()` future is DROPPED (an embedder cancels it) while a handle +//! sits idle in `IoClient.stream_handles` — opened, not currently inside a read — +//! none of those sites run, and (with the interpreter kept alive, e.g. a reused +//! REPL) the upstream request leaks until the interpreter itself is dropped. An +//! RAII guard tied to the run must drop those handles when the future unwinds. + +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// Upstream: send a chunked head immediately, then STALL. Signal when the proxy +/// drops the connection (a blocking read returns 0/Err at peer close). +async fn spawn_head_then_stall_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +#[tokio::test] +async fn test_dropped_interpret_future_closes_outbound_handle() { + let (port, mut upstream_closed) = spawn_head_then_stall_upstream().await; + + // Open an outbound stream, then sit in a long wait WITHOUT reading it — the + // handle is parked in `IoClient.stream_handles`, not held inside an in-flight + // read (dropping a read future would itself drop the handle and mask the bug). + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as up +wait for 5000 milliseconds"# + ); + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + + let mut interp = Interpreter::new(); + { + let fut = interp.interpret(&program); + tokio::pin!(fut); + // Drive the run long enough to open the stream and enter the wait, then + // let `fut` drop at the end of this scope (the interpret future is + // cancelled). The interpreter itself stays alive below. + let _ = tokio::time::timeout(Duration::from_millis(800), fut.as_mut()).await; + } + + // The dropped future must have released the outbound handle (RAII), so the + // upstream is cancelled and the mock observes its connection close — even + // though `interp` is still alive. + tokio::time::timeout(Duration::from_secs(3), &mut upstream_closed) + .await + .expect( + "upstream was not closed after the interpret() future was dropped — \ + the outbound handle leaked until interpreter teardown", + ) + .expect("upstream close sender dropped"); + + // Keep the interpreter alive until after the assertion, so the close was the + // RAII guard's doing and not the interpreter being torn down. + drop(interp); +} diff --git a/tests/dropped_interpret_server_cleanup_test.rs b/tests/dropped_interpret_server_cleanup_test.rs new file mode 100644 index 00000000..34b77276 --- /dev/null +++ b/tests/dropped_interpret_server_cleanup_test.rs @@ -0,0 +1,327 @@ +//! Real-socket regression (maintainer re-review, P1): when the `interpret()` future +//! is DROPPED after a handler starts a streaming response, the interpret-scoped +//! cleanup guard must close the server response stream (ending the client's body) +//! and 500 any unanswered request — even though the reusable `Interpreter` itself +//! stays alive. +//! +//! The tests use explicit drop/release channels. They first observe the exact +//! lifecycle checkpoint, drop only the `interpret()` future, assert the client +//! outcome while the interpreter remains alive, and release the interpreter +//! immediately afterward. No fixed multi-second sleep stands in for dequeue or +//! cleanup completion. + +use std::sync::Arc; +use std::time::Duration; +use tokio::io::AsyncReadExt; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready"); +} + +struct ControlledServer { + thread: std::thread::JoinHandle<()>, + drop_run: Option>, + dropped: tokio::sync::oneshot::Receiver<()>, + release_interpreter: Option>, +} + +/// Run `interpret()` until the test explicitly drops that future, then keep the +/// same interpreter alive until the client-side assertions finish. +fn start_controlled_server(code: String) -> ControlledServer { + let (drop_run, drop_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped) = tokio::sync::oneshot::channel(); + let (release_interpreter, release_rx) = tokio::sync::oneshot::channel(); + let thread = std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("server runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 60, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + { + let future = interp.interpret(&program); + tokio::pin!(future); + tokio::select! { + result = &mut future => { + panic!( + "interpret() returned before the test requested its drop: {result:?}" + ); + } + signal = drop_rx => { + signal.expect("drop-run controller disappeared"); + } + } + } + // The pinned future and its cleanup guard have now been dropped, while + // the reusable interpreter remains alive below. + let _ = dropped_tx.send(()); + // Dropping the sender during a test panic releases this immediately; + // the timeout is only a final leak backstop. + let _ = tokio::time::timeout(Duration::from_secs(10), release_rx).await; + drop(interp); + }); + }); + + ControlledServer { + thread, + drop_run: Some(drop_run), + dropped, + release_interpreter: Some(release_interpreter), + } +} + +async fn request_run_drop(server: &mut ControlledServer) { + server + .drop_run + .take() + .expect("drop signal is sent exactly once") + .send(()) + .expect("controlled server still waits for the drop signal"); + tokio::time::timeout(Duration::from_secs(3), &mut server.dropped) + .await + .expect("interpret() future was not dropped promptly") + .expect("controlled server ended before reporting the drop"); +} + +async fn finish_controlled_server(mut server: ControlledServer) { + let _ = server + .release_interpreter + .take() + .expect("interpreter release is sent exactly once") + .send(()); + tokio::time::timeout(Duration::from_secs(12), async { + while !server.thread.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("controlled server did not stop within 12 seconds"); + if let Err(panic) = server.thread.join() { + std::panic::resume_unwind(panic); + } +} + +/// A handler calls this endpoint only after it dequeues the real client request. +/// The mock reports that exact checkpoint and withholds its response, keeping the +/// handler's pending response parked until the test drops `interpret()`. +async fn spawn_dequeue_checkpoint() -> (u16, tokio::sync::oneshot::Receiver>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind dequeue checkpoint"); + let port = listener + .local_addr() + .expect("dequeue checkpoint address") + .port(); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + let setup = async { + let (mut socket, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept()) + .await + .map_err(|_| "timed out waiting for dequeue checkpoint".to_string())? + .map_err(|error| format!("accept dequeue checkpoint: {error}"))?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let mut head = Vec::new(); + let mut buffer = [0u8; 512]; + loop { + let count = tokio::time::timeout_at(deadline, socket.read(&mut buffer)) + .await + .map_err(|_| "timed out reading dequeue checkpoint".to_string())? + .map_err(|error| format!("read dequeue checkpoint: {error}"))?; + if count == 0 { + return Err("handler closed before sending the dequeue checkpoint".to_string()); + } + head.extend_from_slice(&buffer[..count]); + if head.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if head.len() > 16 * 1024 { + return Err("dequeue checkpoint head exceeded 16 KiB".to_string()); + } + } + let request = String::from_utf8_lossy(&head); + if !request.starts_with("GET /dequeued ") { + return Err(format!( + "unexpected dequeue checkpoint request: {:?}", + request.lines().next() + )); + } + Ok(socket) + } + .await; + + match setup { + Ok(mut socket) => { + let _ = ready_tx.send(Ok(())); + // Keep the outbound request parked. Dropping `interpret()` cancels + // it and closes this socket; no HTTP response is intentionally sent. + let _ = tokio::time::timeout(Duration::from_secs(10), async { + let mut byte = [0u8; 1]; + loop { + match socket.read(&mut byte).await { + Ok(0) | Err(_) => break, + Ok(_) => {} + } + } + }) + .await; + } + Err(error) => { + let _ = ready_tx.send(Err(error)); + } + } + }); + (port, ready_rx) +} + +#[tokio::test] +async fn test_dropped_run_closes_server_stream_while_interpreter_stays_alive() { + let port = common::free_tcp_port(); + + // Handler: start streaming, send a chunk, flush, then park. Receiving `hello` + // is the exact checkpoint that the server stream exists before the run is + // explicitly dropped. + let code = format!( + r#" + listen on port {port} as srv + main loop: + wait for request comes in on srv as req with timeout 60000 + start streaming response to req with status 200 and content type "text/plain" as out + write chunk "hello" to out + flush out + wait for 60000 milliseconds + end loop + "# + ); + + let mut server = start_controlled_server(code); + wait_for_server(port).await; + + let mut response = tokio::time::timeout( + Duration::from_secs(5), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/")) + .send(), + ) + .await + .expect("timed out waiting for streaming response") + .expect("streaming request failed"); + assert_eq!(response.status().as_u16(), 200); + + let mut body = Vec::new(); + tokio::time::timeout(Duration::from_secs(5), async { + while !String::from_utf8_lossy(&body).contains("hello") { + match response.chunk().await { + Ok(Some(bytes)) => body.extend_from_slice(&bytes), + Ok(None) => panic!("stream ended before the pre-drop chunk arrived"), + Err(error) => panic!("stream failed before the run was dropped: {error}"), + } + } + }) + .await + .expect("timed out waiting for the pre-drop streamed chunk"); + + request_run_drop(&mut server).await; + + // The interpreter is deliberately still alive here. Only the run's cleanup + // guard can close the body, and it must be a clean end rather than a transport + // failure accepted as equivalent. + tokio::time::timeout(Duration::from_secs(3), async { + loop { + match response.chunk().await { + Ok(Some(bytes)) => body.extend_from_slice(&bytes), + Ok(None) => break, + Err(error) => { + panic!("drop-guard stream cleanup must end with clean EOF: {error}") + } + } + } + }) + .await + .expect("client body did not end while the interpreter was kept alive"); + assert!( + String::from_utf8_lossy(&body).contains("hello"), + "the pre-drop streamed chunk disappeared: {:?}", + String::from_utf8_lossy(&body) + ); + + finish_controlled_server(server).await; +} + +#[tokio::test] +async fn test_dropped_run_answers_pending_request_with_500() { + let (checkpoint_port, checkpoint_ready) = spawn_dequeue_checkpoint().await; + let port = common::free_tcp_port(); + let code = format!( + r#" + listen on port {port} as srv + main loop: + wait for request comes in on srv as req with timeout 60000 + open url at "http://127.0.0.1:{checkpoint_port}/dequeued" and stream response as checkpoint + wait for 60000 milliseconds + end loop + "# + ); + + let mut server = start_controlled_server(code); + wait_for_server(port).await; + + let url = format!("http://127.0.0.1:{port}/"); + let request = tokio::spawn(async move { reqwest::Client::new().get(url).send().await }); + tokio::time::timeout(Duration::from_secs(5), checkpoint_ready) + .await + .expect("handler did not reach the post-dequeue checkpoint") + .expect("dequeue checkpoint task ended without a result") + .expect("dequeue checkpoint failed"); + + request_run_drop(&mut server).await; + + let response = tokio::time::timeout(Duration::from_secs(5), request) + .await + .expect("client hung waiting for the dropped-run 500") + .expect("pending request task panicked") + .expect("pending request failed"); + assert_eq!( + response.status().as_u16(), + 500, + "dropped run must answer the still-pending request with 500, got {}", + response.status() + ); + assert_eq!( + response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()), + Some("text/plain; charset=utf-8"), + "dropped-run cleanup must use the explicit plain-text 500 response" + ); + let body = tokio::time::timeout(Duration::from_secs(3), response.bytes()) + .await + .expect("timed out reading dropped-run cleanup response body") + .expect("read dropped-run cleanup response body"); + assert_eq!( + body.as_ref(), + b"Internal Server Error\n", + "dropped-run cleanup must resolve the pending request with its exact 500 body" + ); + + finish_controlled_server(server).await; +} diff --git a/tests/execute_file_test.rs b/tests/execute_file_test.rs index 20cb1afb..4e279628 100644 --- a/tests/execute_file_test.rs +++ b/tests/execute_file_test.rs @@ -4,6 +4,8 @@ // Executes another WFL file in-process with a nested interpreter, optionally // passing HTTP request context and capturing the child's display/print output. +mod common; + use std::collections::HashMap; use std::fs; use std::sync::Arc; @@ -497,7 +499,7 @@ async fn test_web_server_serves_executed_wfl_page() { ) .expect("Failed to write page file"); - let port: u16 = 58123; + let port = common::free_tcp_port(); let server_file = temp_dir.path().join("server.wfl"); let server_code = format!( concat!( diff --git a/tests/file_io_performance_test.rs b/tests/file_io_performance_test.rs index 57db989b..6b7289e4 100644 --- a/tests/file_io_performance_test.rs +++ b/tests/file_io_performance_test.rs @@ -200,27 +200,31 @@ mod file_io_performance_tests { #[tokio::test] async fn test_directory_listing_performance() { - let test_files: Vec = (0..30).map(|i| format!("dir_perf_{}.txt", i)).collect(); - let test_file_refs: Vec<&str> = test_files.iter().map(|s| s.as_str()).collect(); - cleanup_test_files(&test_file_refs); + let test_dir = tempfile::tempdir().expect("Failed to create directory-listing fixture"); + let test_files: Vec<_> = (0..30) + .map(|i| test_dir.path().join(format!("dir_perf_{}.txt", i))) + .collect(); // Create multiple files for directory listing for (i, file) in test_files.iter().enumerate() { fs::write(file, format!("Content for file {}", i)).expect("Failed to create test file"); } - let code = r#" + let fixture_path = test_dir.path().to_string_lossy().replace('\\', "/"); + let code = format!( + r#" // Test directory listing performance - wait for store all_files as list files in "." - wait for store txt_files as list files in "." with pattern "dir_perf_*.txt" - wait for store recursive_files as list files recursively in "." + wait for store all_files as list files in "{fixture_path}" + wait for store txt_files as list files in "{fixture_path}" with pattern "dir_perf_*.txt" + wait for store recursive_files as list files recursively in "{fixture_path}" display "Listed all files: " with length of all_files display "Listed TXT files: " with length of txt_files display "Listed recursive files: " with length of recursive_files - "#; + "# + ); - let result = execute_wfl_code_with_timing(code).await; + let result = execute_wfl_code_with_timing(&code).await; assert!( result.is_ok(), "Directory listing performance test failed: {:?}", @@ -234,8 +238,6 @@ mod file_io_performance_tests { "Directory listing took too long: {:?}", elapsed ); - - cleanup_test_files(&test_file_refs); } #[tokio::test] diff --git a/tests/flush_action_backcompat_test.rs b/tests/flush_action_backcompat_test.rs new file mode 100644 index 00000000..8319a0b8 --- /dev/null +++ b/tests/flush_action_backcompat_test.rs @@ -0,0 +1,416 @@ +//! Backward-compatibility regression (maintainer re-review, P1): a statement-initial +//! `flush ` must NOT hijack a pre-existing zero-argument action named +//! `flush `. +//! +//! Before `flush` became a streaming command, `flush cache` was an expression +//! statement that auto-invoked an action `flush cache`. The dispatcher now routes +//! merged `flush …` tokens to the streaming flush; it must still prefer a defined +//! action of that full name so an existing program keeps working. + +use std::fs; +use std::process::Command; +use tempfile::TempDir; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::typechecker::TypeChecker; + +fn run_src(src: &str) -> (String, Option) { + let dir = TempDir::new().expect("tempdir"); + let path = dir.path().join("main.wfl"); + fs::write(&path, src).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_wfl")) + .arg(&path) + .output() + .expect("failed to execute WFL"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + (combined, output.status.code()) +} + +#[test] +fn flush_calls_a_matching_zero_arg_action_instead_of_flushing_a_stream() { + // `flush cache` must invoke the action `flush cache`, printing CALLED — not try + // to flush a (nonexistent) stream `cache`. + let src = "define action called flush cache:\n\ + \x20\x20\x20\x20display \"CALLED\"\n\ + end action\n\ + \n\ + flush cache\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "program should exit cleanly; output was:\n{out}" + ); + assert!( + out.contains("CALLED"), + "the pre-existing `flush cache` action must be called; output was:\n{out}" + ); + assert!( + !out.to_lowercase().contains("stream"), + "`flush cache` must not be reinterpreted as a stream flush; output was:\n{out}" + ); +} + +#[test] +fn truly_bare_flush_still_calls_the_legacy_zero_argument_action() { + let src = "define action called flush:\n\ + \x20\x20\x20\x20display \"CALLED\"\n\ + end action\n\ + \n\ + flush\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "the exact bare `flush` action must remain callable; output was:\n{out}" + ); + assert!( + out.contains("CALLED"), + "the exact bare `flush` statement must auto-call its legacy action; output was:\n{out}" + ); + assert!( + !out.to_lowercase().contains("stream"), + "the exact bare `flush` statement must not become a stream operation; output was:\n{out}" + ); +} + +#[test] +fn parenthesized_flush_target_keeps_the_exact_zero_argument_action_fallback() { + let src = "define action called flush:\n\ + \x20\x20\x20\x20display \"PAREN_CALLED\"\n\ + end action\n\ + store ignored as 1\n\ + flush (ignored)\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "`flush (expr)` must keep the exact `flush` action fallback; output:\n{out}" + ); + assert!( + out.contains("PAREN_CALLED"), + "the exact zero-argument action must run; output:\n{out}" + ); +} + +#[test] +fn explicit_call_flush_target_keeps_the_exact_zero_argument_action_fallback() { + let src = "define action called flush:\n\ + \x20\x20\x20\x20display \"CALL_CALLED\"\n\ + end action\n\ + define action called acquire stream:\n\ + \x20\x20\x20\x20return 1\n\ + end action\n\ + flush call acquire stream\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "`flush call ...` must keep the exact `flush` action fallback; output:\n{out}" + ); + assert!( + out.contains("CALL_CALLED"), + "the exact zero-argument action must run; output:\n{out}" + ); +} + +#[test] +fn truly_bare_flush_still_evaluates_a_non_callable_legacy_variable() { + let src = "store flush as 1\n\ + flush\n\ + display flush\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "the exact bare `flush` variable must remain a valid expression statement; output:\n{out}" + ); + assert!( + out.contains('1'), + "the bare legacy variable must remain readable after evaluation; output:\n{out}" + ); + assert!( + !out.to_lowercase().contains("stream"), + "the exact bare `flush` variable must not become a stream operation; output:\n{out}" + ); +} + +#[test] +fn flush_without_a_matching_action_still_errors_as_a_stream_flush() { + // With no action `flush cache` and no stream `cache`, `flush cache` falls + // through to the stream interpretation and errors (rather than silently + // succeeding) — proving the action fallback is a preference, not a bypass. + let src = "flush cache\n"; + let (out, code) = run_src(src); + assert_ne!( + code, + Some(0), + "a bare `flush cache` with no target must error; output:\n{out}" + ); +} + +#[test] +fn flush_non_callable_full_name_binding_is_expression_statement() { + // Pre-streaming: `store flush cache as 1` then `flush cache` evaluated the + // variable and completed. Must not try to flush an undefined stream `cache` + // (issue #642). + let src = "store flush cache as 1\n\ + flush cache\n\ + display flush cache\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "non-callable full-name binding must keep working as an expression statement; output:\n{out}" + ); + assert!( + out.contains('1'), + "expected the bound value to still be readable; output:\n{out}" + ); + assert!( + !out.to_lowercase().contains("stream"), + "`flush cache` must not be reinterpreted as a stream flush; output:\n{out}" + ); +} + +#[test] +fn flush_parameterized_single_action_still_arity_errors() { + // A single parameterized `flush cache` action: old expression-statement + // auto-call with zero args produced an arity error — not a silent success + // and not a stream flush (issue #642 re-review). + let src = "define action called flush cache with parameters x:\n\ + \x20\x20\x20\x20display x\n\ + end action\n\ + \n\ + flush cache\n"; + let (out, code) = run_src(src); + assert_ne!( + code, + Some(0), + "parameterized flush cache with no args must arity-error; output:\n{out}" + ); + assert!( + out.to_lowercase().contains("argument") + || out.to_lowercase().contains("expected") + || out.contains("1"), + "expected an arity error, got:\n{out}" + ); +} + +#[test] +fn flush_overloaded_without_zero_arg_is_expression_not_stream() { + // True overload set with no zero-arg member: bare name evaluates as an + // overloaded value (expression statement), not a stream flush. + let src = "define action called flush cache with parameters x:\n\ + \x20\x20\x20\x20display x\n\ + end action\n\ + define action called flush cache with parameters x and y:\n\ + \x20\x20\x20\x20display x\n\ + end action\n\ + \n\ + flush cache\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "overloaded flush with no zero-arg overload must not error as a stream flush; output:\n{out}" + ); + assert!( + out.contains("OK"), + "program should reach the trailing display; output:\n{out}" + ); +} + +#[test] +fn flush_with_postfix_uses_legacy_expression_when_bound() { + // `flush cache[0]` must evaluate IndexAccess on Variable("flush cache"), not + // try to flush stream `cache[0]`. + let src = "store flush cache as [\"a\" and \"b\"]\n\ + flush cache[0]\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "flush cache[0] with a bound list must be an expression statement; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +fn typecheck_src(src: &str) -> Result<(), String> { + let tokens = lex_wfl_with_positions(src); + let program = Parser::new(&tokens).parse().expect("parse"); + TypeChecker::new() + .check_types(&program) + .map_err(|errors| format!("{errors:?}")) +} + +#[test] +fn flush_with_nested_postfix_uses_the_recursive_legacy_root() { + let src = "store flush cache as [[\"a\"]]\n\ + flush cache[0][0]\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "nested legacy postfix must resolve the full-name root; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +#[test] +fn flush_with_binary_expression_uses_the_legacy_binding() { + let src = "store flush cache as 1\n\ + flush cache plus 1\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "binary legacy expression must resolve the full-name root; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +#[test] +fn flush_with_of_call_uses_the_full_legacy_action_name() { + let src = "define action called flush cache with parameters value:\n\ + \x20\x20\x20\x20display value\n\ + end action\n\ + flush cache of 7\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "an `of` continuation must call the full legacy action name; output:\n{out}" + ); + assert!( + out.contains('7'), + "the full-name action should receive its argument; output:\n{out}" + ); +} + +#[test] +fn flush_with_post_of_index_preserves_the_legacy_action_result() { + let src = "define action called flush cache with parameters values:\n\ + \x20\x20\x20\x20return values\n\ + end action\n\ + store items as [7]\n\ + flush cache of (items)[0]\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "the indexed result of the full legacy action must remain an expression statement; output:\n{out}" + ); + assert!( + out.contains("OK"), + "the legacy post-`of` expression must complete; output:\n{out}" + ); + assert!( + !out.to_lowercase().contains("stream"), + "the legacy action result must not be reinterpreted as a stream target; output:\n{out}" + ); +} + +#[test] +fn flush_split_rewrite_keeps_the_original_legacy_binding() { + let src = "store flush cache as 1\n\ + flush cache split \"a,b\" by \",\"\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "split rewrite must still select the bound legacy expression; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +#[test] +fn flush_explicit_find_in_rewrite_keeps_the_original_legacy_binding() { + let src = "create pattern letter_a:\n\ + \x20\x20\x20\x20\"a\"\n\ + end pattern\n\ + store flush cache as 1\n\ + flush cache find letter_a in \"abc\"\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "explicit find-in rewrite must select the bound legacy expression; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +#[test] +fn flush_explicit_replace_in_rewrite_keeps_the_original_legacy_binding() { + let src = "create pattern letter_a:\n\ + \x20\x20\x20\x20\"a\"\n\ + end pattern\n\ + store flush cache as 1\n\ + flush cache replace letter_a with \"z\" in \"abc\"\n\ + display \"OK\"\n"; + let (out, code) = run_src(src); + assert_eq!( + code, + Some(0), + "explicit replace-in rewrite must select the bound legacy expression; output:\n{out}" + ); + assert!(out.contains("OK"), "expected OK; output:\n{out}"); +} + +#[test] +fn flush_direct_container_property_uses_the_legacy_expression_branch() { + let src = "create container Cache:\n\ + \x20\x20\x20\x20property flush cache: Number\n\ + \x20\x20\x20\x20action inspect:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20flush cache plus 1\n\ + \x20\x20\x20\x20end\n\ + end"; + assert!( + typecheck_src(src).is_ok(), + "a direct container property must select the legacy branch: {:?}", + typecheck_src(src).err() + ); +} + +#[test] +fn flush_inherited_container_property_uses_the_legacy_expression_branch() { + let src = "create container Base:\n\ + \x20\x20\x20\x20property flush cache: Number\n\ + end\n\ + create container Child extends Base:\n\ + \x20\x20\x20\x20action inspect:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20flush cache plus 1\n\ + \x20\x20\x20\x20end\n\ + end"; + assert!( + typecheck_src(src).is_ok(), + "an inherited container property must select the legacy branch: {:?}", + typecheck_src(src).err() + ); +} + +#[test] +fn flush_invalid_container_property_expression_is_statically_rejected() { + let src = "create container Cache:\n\ + \x20\x20\x20\x20property flush cache: Text\n\ + \x20\x20\x20\x20action inspect:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20flush cache minus 1\n\ + \x20\x20\x20\x20end\n\ + end"; + let errors = typecheck_src(src).expect_err("Text minus Number must be rejected"); + assert!( + !errors.contains("`flush` requires a response-stream handle"), + "the bound property must be checked as the legacy expression, got: {errors}" + ); +} diff --git a/tests/http_server_streaming_test.rs b/tests/http_server_streaming_test.rs new file mode 100644 index 00000000..e0fb8bcb --- /dev/null +++ b/tests/http_server_streaming_test.rs @@ -0,0 +1,461 @@ +// Tests for streamed server responses (item 3): +// start streaming response to [with status ] [and content type ] as +// write line to // frames a line (newline appended) +// write chunk to // raw bytes/text, verbatim +// flush +// close // ends the response body +// +// A WFL web server streams a response; a reqwest client reads it back. + +use std::time::Duration; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Statement}; + +mod common; + +// ----------------------------- parser tests ------------------------------ + +fn parse_single_statement(code: &str) -> Statement { + let tokens = lex_wfl_with_positions(code); + let mut parser = Parser::new(&tokens); + let program = parser + .parse() + .unwrap_or_else(|e| panic!("Parse error for {code:?}: {e:?}")); + assert_eq!( + program.statements.len(), + 1, + "Expected exactly one statement for {code:?}" + ); + program.statements.into_iter().next().unwrap() +} + +#[test] +fn test_start_streaming_response_parses() { + let stmt = parse_single_statement( + r#"start streaming response to req with status 200 and content type "application/x-ndjson" as out"#, + ); + match stmt { + Statement::StartStreamingResponseStatement { + status, + content_type, + variable_name, + .. + } => { + assert!(status.is_some()); + assert!(content_type.is_some()); + assert_eq!(variable_name, "out"); + } + other => panic!("Expected StartStreamingResponseStatement, got {other:?}"), + } +} + +#[test] +fn test_start_streaming_response_connective_before_as_parses() { + // A connective (`and`/`with`) directly before `as` is consumed as the + // end-of-clauses join, so `... with status 200 and as out` parses rather than + // failing at the `as` binding with a confusing "expected as, found and". + let stmt = + parse_single_statement(r#"start streaming response to req with status 200 and as out"#); + match stmt { + Statement::StartStreamingResponseStatement { + status, + variable_name, + .. + } => { + assert!(status.is_some()); + assert_eq!(variable_name, "out"); + } + other => panic!("Expected StartStreamingResponseStatement, got {other:?}"), + } +} + +#[test] +fn test_content_type_variable_binds_correct_name() { + // `content type ` where is a bare identifier: the lexer merges it + // into `type `, so the parser must split the marker off and bind the + // variable, not `type ` as one name. + let stmt = parse_single_statement( + r#"start streaming response to req with status 200 and content type ct as out"#, + ); + match stmt { + Statement::StartStreamingResponseStatement { content_type, .. } => match content_type { + Some(wfl::parser::ast::Expression::Variable(name, _, _)) => assert_eq!(name, "ct"), + other => panic!("Expected content type Variable(\"ct\"), got {other:?}"), + }, + other => panic!("Expected StartStreamingResponseStatement, got {other:?}"), + } +} + +#[test] +fn test_write_line_parses() { + let stmt = parse_single_statement(r#"write line payload to out"#); + match stmt { + Statement::StreamWriteStatement { is_line, .. } => assert!(is_line), + other => panic!("Expected StreamWriteStatement, got {other:?}"), + } +} + +#[test] +fn test_write_chunk_parses() { + let stmt = parse_single_statement(r#"write chunk payload to out"#); + match stmt { + Statement::StreamWriteStatement { is_line, .. } => assert!(!is_line), + other => panic!("Expected StreamWriteStatement, got {other:?}"), + } +} + +#[test] +fn test_flush_parses() { + let stmt = parse_single_statement("flush out"); + match stmt { + Statement::FlushStreamStatement { .. } => {} + other => panic!("Expected FlushStreamStatement, got {other:?}"), + } +} + +#[test] +fn test_flush_with_index_operand_parses() { + // `flush streams["a"]`: the lexer merges `flush streams`; the trailing `["a"]` + // must bind to the operand (one IndexAccess), not leave dangling tokens that + // split into a bogus second statement. + let tokens = lex_wfl_with_positions(r#"flush streams["a"]"#); + let program = Parser::new(&tokens).parse().expect("parse"); + assert_eq!( + program.statements.len(), + 1, + "`flush [idx]` must be one statement, got {:#?}", + program.statements + ); + match &program.statements[0] { + Statement::FlushStreamStatement { target, .. } => assert!( + matches!(target, wfl::parser::ast::Expression::IndexAccess { .. }), + "flush operand must be an IndexAccess, got {target:#?}" + ), + other => panic!("expected FlushStreamStatement, got {other:?}"), + } +} + +#[test] +fn test_flush_with_property_operand_parses() { + // `flush obj.out`: the lexer merges `flush obj`; the trailing `.out` must bind + // to the operand (a PropertyAccess), not split off. + let tokens = lex_wfl_with_positions("flush obj.out"); + let program = Parser::new(&tokens).parse().expect("parse"); + assert_eq!( + program.statements.len(), + 1, + "`flush .prop` must be one statement, got {:#?}", + program.statements + ); + match &program.statements[0] { + Statement::FlushStreamStatement { target, .. } => assert!( + matches!(target, wfl::parser::ast::Expression::PropertyAccess { .. }), + "flush operand must be a PropertyAccess, got {target:#?}" + ), + other => panic!("expected FlushStreamStatement, got {other:?}"), + } +} + +#[test] +fn test_unmerged_flush_targets_parse_as_single_flush_statements() { + let cases = [ + ("flush (out)", "variable"), + ("flush call acquire stream", "call"), + ("flush (streams)[0]", "index"), + ("flush (holder).stream", "property"), + ("flush (holder).method()", "method"), + ]; + + for (source, expected_shape) in cases { + let statement = parse_single_statement(source); + let target = match &statement { + Statement::FlushStreamStatement { target, .. } => target, + other => panic!("expected FlushStreamStatement for `{source}`, got {other:#?}"), + }; + let actual_shape = match target { + Expression::Variable(..) => "variable", + Expression::ActionCall { .. } => "call", + Expression::IndexAccess { .. } => "index", + Expression::PropertyAccess { .. } => "property", + Expression::MethodCall { .. } => "method", + other => panic!("unexpected flush target for `{source}`: {other:#?}"), + }; + assert_eq!( + actual_shape, expected_shape, + "wrong target shape for `{source}`: {target:#?}" + ); + } +} + +#[test] +fn test_write_bare_line_variable_to_file_still_parses() { + // Backward compat: `write to ` with a variable literally named + // `line`/`chunk` must NOT be intercepted as a stream write (regression). + for src in ["write line to out", "write chunk to out"] { + let stmt = parse_single_statement(src); + match stmt { + Statement::WriteToStatement { .. } => {} + other => panic!("Expected WriteToStatement for {src:?}, got {other:?}"), + } + } +} + +// ----------------------------- runtime tests ------------------------------ + +fn start_server_thread(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("Failed to create runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + let ast = parser.parse().expect("Failed to parse WFL code"); + let mut interpreter = Interpreter::new(); + // Surface an unexpected interpreter error as a thread panic, so the + // test's `join_server` re-raises it instead of silently passing. + if let Err(errors) = interpreter.interpret(&ast).await { + panic!("server interpreter failed: {errors:?}"); + } + }); + }) +} + +/// Join the server thread and re-raise its panic (if any) in the test thread, so +/// a server-side panic or interpreter error fails the test loudly rather than +/// being dropped. `JoinHandle::join`'s error is `Box` (no `Debug`), so +/// `resume_unwind` is the way to propagate it with its original message. +fn join_server(handle: std::thread::JoinHandle<()>) { + if let Err(panic) = handle.join() { + std::panic::resume_unwind(panic); + } +} + +/// Wait until the WFL server has bound `port` and is accepting connections, +/// instead of a fixed sleep that flakes on a loaded CI runner (spurious +/// `Connection refused` when binding takes longer than the guess). A bare TCP +/// connect that drops immediately is a safe readiness probe. +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready in time"); +} + +#[tokio::test] +async fn test_streamed_response_lines_and_headers() { + let port = common::free_tcp_port(); + let server_code = format!( + r#" + listen on port {port} as s + wait for request comes in on s as req with timeout 10000 + start streaming response to req with status 200 and content type "application/x-ndjson" as out + write line "alpha" to out + write line "beta" to out + flush out + write line "gamma" to out + close out + close server s + "# + ); + + let server_handle = start_server_thread(server_code); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://127.0.0.1:{port}/stream")) + .send() + .await + .expect("Failed to send request"); + + assert_eq!(response.status().as_u16(), 200); + let ct = response + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + assert_eq!(ct, "application/x-ndjson"); + + let body = response.text().await.expect("Failed to read body"); + assert_eq!(body, "alpha\nbeta\ngamma\n"); + + join_server(server_handle); +} + +#[tokio::test] +async fn test_post_of_index_operands_execute_for_content_write_and_flush() { + let port = common::free_tcp_port(); + let server_code = format!( + r#" + define action called choose with parameters values: + return values + end action + define action called cache with parameters values: + return values + end action + store types as ["text/plain"] + store chunks as ["post-of body"] + listen on port {port} as s + wait for request comes in on s as req with timeout 10000 + start streaming response to req with status 200 and content type choose of (types)[0] as out + store streams as [out] + write line choose of (chunks)[0] to out + flush cache of (streams)[0] + close out + close server s + "# + ); + + let server_handle = start_server_thread(server_code); + wait_for_server(port).await; + + let response = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/post-of")) + .send() + .await + .expect("request failed"); + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get("content-type") + .and_then(|value| value.to_str().ok()), + Some("text/plain"), + "the post-`of` index must select the content type returned by `choose`" + ); + assert_eq!( + response.text().await.expect("read response body"), + "post-of body\n", + "the indexed return value must reach the streamed write" + ); + + join_server(server_handle); +} + +#[tokio::test] +async fn test_write_after_close_does_not_reach_client() { + // Writing after `close out` is a catchable error and does NOT reach the + // client: the client sees only the bytes written before close. + let port = common::free_tcp_port(); + let server_code = format!( + r#" + listen on port {port} as s + wait for request comes in on s as req with timeout 10000 + start streaming response to req with status 200 and content type "text/plain" as out + write line "before" to out + close out + try: + write line "after" to out + catch: + display "write after close correctly errored" + end try + close server s + "# + ); + + let server_handle = start_server_thread(server_code); + wait_for_server(port).await; + + let response = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/x")) + .send() + .await + .expect("request failed"); + let body = response.text().await.unwrap(); + assert_eq!( + body, "before\n", + "writes after close must not reach the client" + ); + + join_server(server_handle); +} + +#[tokio::test] +async fn test_stream_auto_closes_when_handler_ends_without_close() { + // Lifecycle guarantee (spec item 5): a handler that starts a stream and ends + // WITHOUT `close out` must still finalize the client's body on the way out. + // Otherwise the sender lingers in the interpreter's stream table, the body is + // never terminated, and the client hangs forever (and the table leaks). + let port = common::free_tcp_port(); + let server_code = format!( + r#" + listen on port {port} as s + main loop: + wait for request comes in on s as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server s + break + otherwise: + start streaming response to req with status 200 and content type "text/plain" as out + write line "hello" to out + end check + end loop + "# + ); + + let server_handle = start_server_thread(server_code); + wait_for_server(port).await; + + let response = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/x")) + .send() + .await + .expect("request failed"); + assert_eq!(response.status().as_u16(), 200); + + // Reading the body must COMPLETE (the stream was auto-closed). If the handler + // leaked the stream, this read hangs — the timeout turns that into a failure + // instead of a stuck test. + let body = tokio::time::timeout(Duration::from_secs(5), response.text()) + .await + .expect("body did not finish — stream was not auto-closed when the handler ended") + .expect("failed to read body"); + assert_eq!(body, "hello\n"); + + // Stop the server. + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/shutdown")) + .send() + .await; + join_server(server_handle); +} + +#[tokio::test] +async fn test_streamed_response_write_chunk_verbatim() { + let port = common::free_tcp_port(); + let server_code = format!( + r#" + listen on port {port} as s + wait for request comes in on s as req with timeout 10000 + start streaming response to req with status 201 and content type "text/plain" as out + write chunk "one" to out + write chunk "two" to out + close out + close server s + "# + ); + + let server_handle = start_server_thread(server_code); + wait_for_server(port).await; + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://127.0.0.1:{port}/raw")) + .send() + .await + .expect("Failed to send request"); + + assert_eq!(response.status().as_u16(), 201); + let body = response.text().await.expect("Failed to read body"); + // write chunk does not append newlines. + assert_eq!(body, "onetwo"); + + join_server(server_handle); +} diff --git a/tests/http_stream_test.rs b/tests/http_stream_test.rs new file mode 100644 index 00000000..cf3e2f11 --- /dev/null +++ b/tests/http_stream_test.rs @@ -0,0 +1,321 @@ +// Tests for generic outbound response streaming: +// open url at "" [with ...] and stream response as upstream +// wait for next chunk from upstream as chunk -> Binary, or nothing at EOF +// wait for next line from upstream as line -> Text, or nothing at EOF +// close upstream -> cancels the upstream request +// +// A minimal local TCP server stands in for a real upstream (e.g. an LLM +// endpoint emitting newline-delimited JSON), so the tests are offline-safe. + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +use wfl::interpreter::Interpreter; +use wfl::interpreter::value::Value; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::Statement; + +// ----------------------------- parser tests ------------------------------ + +fn parse_single_statement(code: &str) -> Statement { + let tokens = lex_wfl_with_positions(code); + let mut parser = Parser::new(&tokens); + let program = parser + .parse() + .unwrap_or_else(|e| panic!("Parse error for {code:?}: {e:?}")); + assert_eq!( + program.statements.len(), + 1, + "Expected exactly one statement for {code:?}" + ); + program.statements.into_iter().next().unwrap() +} + +#[test] +fn test_stream_response_parses_to_http_stream_statement() { + let stmt = + parse_single_statement(r#"open url at "https://example.com" and stream response as up"#); + match stmt { + Statement::HttpStreamStatement { + method, + headers, + body, + variable_name, + .. + } => { + assert!(method.is_none()); + assert!(headers.is_none()); + assert!(body.is_none()); + assert_eq!(variable_name, "up"); + } + other => panic!("Expected HttpStreamStatement, got {other:?}"), + } +} + +#[test] +fn test_stream_response_with_method_headers_body() { + let stmt = parse_single_statement( + r#"open url at "https://api.example.com" with method "POST" and headers h and body b and stream response as up"#, + ); + match stmt { + Statement::HttpStreamStatement { + method, + headers, + body, + variable_name, + .. + } => { + assert!(method.is_some()); + assert!(headers.is_some()); + assert!(body.is_some()); + assert_eq!(variable_name, "up"); + } + other => panic!("Expected HttpStreamStatement, got {other:?}"), + } +} + +#[test] +fn test_wait_for_next_chunk_parses() { + let stmt = parse_single_statement("wait for next chunk from up as chunk"); + match stmt { + Statement::WaitForNextChunkStatement { variable_name, .. } => { + assert_eq!(variable_name, "chunk"); + } + other => panic!("Expected WaitForNextChunkStatement, got {other:?}"), + } +} + +#[test] +fn test_wait_for_next_line_parses() { + let stmt = parse_single_statement("wait for next line from up as line"); + match stmt { + Statement::WaitForNextLineStatement { variable_name, .. } => { + assert_eq!(variable_name, "line"); + } + other => panic!("Expected WaitForNextLineStatement, got {other:?}"), + } +} + +#[test] +fn test_wait_for_next_as_duration_variable_still_parses() { + // Backward compat: a variable literally named `next` in a duration wait must + // NOT be intercepted as `wait for next chunk|line` (regression). + let stmt = parse_single_statement("wait for next milliseconds"); + match stmt { + Statement::WaitForDurationStatement { unit, .. } => assert_eq!(unit, "milliseconds"), + other => panic!("Expected WaitForDurationStatement, got {other:?}"), + } +} + +// ----------------------------- runtime tests ------------------------------ + +/// Spawn a one-shot server that answers 200 with the given body, streamed with +/// an explicit Content-Length and `Connection: close`. +async fn spawn_body_server(body: &'static str) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut tmp = [0u8; 2048]; + // Drain the request head (single read is enough for a GET). + let _ = socket.read(&mut tmp).await; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/x-ndjson\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + socket.write_all(response.as_bytes()).await.unwrap(); + socket.shutdown().await.ok(); + }); + format!("http://{addr}") +} + +async fn run_wfl(code: &str) -> Interpreter { + let tokens = lex_wfl_with_positions(code); + let mut parser = Parser::new(&tokens); + let program = parser + .parse() + .unwrap_or_else(|e| panic!("Parse error: {e:?}")); + let mut interpreter = Interpreter::new(); + interpreter + .interpret(&program) + .await + .unwrap_or_else(|e| panic!("Runtime error: {e:?}")); + interpreter +} + +fn get_var(interpreter: &Interpreter, name: &str) -> Value { + interpreter + .global_env() + .borrow() + .get(name) + .unwrap_or_else(|| panic!("Variable '{name}' not found")) +} + +fn get_text(interpreter: &Interpreter, name: &str) -> String { + match get_var(interpreter, name) { + Value::Text(t) => t.to_string(), + other => panic!("Expected '{name}' to be text, got {other:?}"), + } +} + +#[tokio::test] +async fn test_stream_exposes_status_and_headers_immediately() { + let url = spawn_body_server("alpha\nbeta\n").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + store s as up["status"] + store ok as up["ok"] + store ct as up["headers"]["content-type"] + close up + "# + ); + let interpreter = run_wfl(&code).await; + + match get_var(&interpreter, "s") { + Value::Number(n) => assert_eq!(n, 200.0), + other => panic!("Expected numeric status, got {other:?}"), + } + match get_var(&interpreter, "ok") { + Value::Bool(b) => assert!(b), + other => panic!("Expected boolean ok, got {other:?}"), + } + assert_eq!(get_text(&interpreter, "ct"), "application/x-ndjson"); +} + +#[tokio::test] +async fn test_next_line_yields_lines_then_nothing_at_eof() { + let url = spawn_body_server("alpha\nbeta\ngamma\n").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + wait for next line from up as line1 + wait for next line from up as line2 + wait for next line from up as line3 + wait for next line from up as line4 + "# + ); + let interpreter = run_wfl(&code).await; + + assert_eq!(get_text(&interpreter, "line1"), "alpha"); + assert_eq!(get_text(&interpreter, "line2"), "beta"); + assert_eq!(get_text(&interpreter, "line3"), "gamma"); + // Clean EOF binds `nothing` (Null). + match get_var(&interpreter, "line4") { + Value::Null => {} + other => panic!("Expected nothing at EOF, got {other:?}"), + } +} + +#[tokio::test] +async fn test_next_line_returns_final_unterminated_line() { + // No trailing newline: the last line is still delivered before EOF. + let url = spawn_body_server("one\ntwo").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + wait for next line from up as a + wait for next line from up as b + wait for next line from up as c + store closed_after_eof as no + try: + wait for next line from up as d + catch: + store closed_after_eof as yes + end try + "# + ); + let interpreter = run_wfl(&code).await; + assert_eq!(get_text(&interpreter, "a"), "one"); + assert_eq!(get_text(&interpreter, "b"), "two"); + match get_var(&interpreter, "c") { + Value::Null => {} + other => panic!("Expected nothing at EOF, got {other:?}"), + } + assert!( + matches!(get_var(&interpreter, "closed_after_eof"), Value::Bool(true)), + "the one EOF result must consume the exhausted handle; a later read is catchably closed" + ); +} + +#[tokio::test] +async fn test_next_line_reusing_same_variable_in_one_scope() { + // Regression: reading successive lines into the SAME `as line` variable in + // one scope must work (define errors on an existing binding, so this needs + // define_or_replace — matching `wait for request ... as req`). + let url = spawn_body_server("a\nb\n").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + wait for next line from up as line + store first as line + wait for next line from up as line + store second as line + "# + ); + let interpreter = run_wfl(&code).await; + assert_eq!(get_text(&interpreter, "first"), "a"); + assert_eq!(get_text(&interpreter, "second"), "b"); +} + +#[tokio::test] +async fn test_next_line_count_loop_reusing_same_variable() { + // The common streaming pattern: a count loop reusing `as line`. + let url = spawn_body_server("a\nb\nc\n").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + store collected as "" + count from 1 to 100: + wait for next line from up as line + check if line is nothing: + break + otherwise: + change collected to collected with line + end check + end count + "# + ); + let interpreter = run_wfl(&code).await; + assert_eq!(get_text(&interpreter, "collected"), "abc"); +} + +#[tokio::test] +async fn test_next_chunk_yields_binary_then_nothing() { + let url = spawn_body_server("raw-bytes-payload").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + wait for next chunk from up as chunk1 + "# + ); + let interpreter = run_wfl(&code).await; + + match get_var(&interpreter, "chunk1") { + Value::Binary(b) => assert!(!b.is_empty(), "first chunk should carry bytes"), + other => panic!("Expected binary chunk, got {other:?}"), + } +} + +#[tokio::test] +async fn test_reading_from_closed_stream_is_error() { + let url = spawn_body_server("x\n").await; + let code = format!( + r#" + open url at "{url}" and stream response as up + close up + wait for next line from up as line + "# + ); + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + let program = parser.parse().unwrap(); + let mut interpreter = Interpreter::new(); + let result = interpreter.interpret(&program).await; + assert!( + result.is_err(), + "reading from a closed stream handle should be a catchable error" + ); +} diff --git a/tests/open_file_local_type_test.rs b/tests/open_file_local_type_test.rs new file mode 100644 index 00000000..0fad03b0 --- /dev/null +++ b/tests/open_file_local_type_test.rs @@ -0,0 +1,122 @@ +//! Regression coverage for local `open file ... as ...` bindings that analyzer +//! body scopes do not retain for the type-checker pass. +//! +//! These type-checker tests do not claim that runtime file bindings can shadow +//! an outer variable: `Environment::define` rejects parent-scope collisions. + +use std::fs; +use std::process::Command; +use tempfile::TempDir; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::typechecker::TypeChecker; + +fn typecheck(source: &str) -> Result<(), String> { + let program = Parser::new(&lex_wfl_with_positions(source)) + .parse() + .expect("parse"); + TypeChecker::new() + .check_types(&program) + .map_err(|errors| format!("{errors:?}")) +} + +#[test] +fn fresh_local_file_handles_are_concrete_in_action_loop_and_method_scopes() { + let sources = [ + ( + "action", + "define action called dump:\n\ + \x20\x20\x20\x20open file at \"unused.txt\" for writing as out\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to out\n\ + \x20\x20\x20\x20close out\n\ + end action\n", + ), + ( + "main loop", + "main loop:\n\ + \x20\x20\x20\x20open file at \"unused.txt\" for writing as out\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to out\n\ + \x20\x20\x20\x20close out\n\ + \x20\x20\x20\x20break\n\ + end loop\n", + ), + ( + "container method", + "create container Writer:\n\ + \x20\x20\x20\x20action dump:\n\ + \x20\x20\x20\x20\x20\x20\x20\x20open file at \"unused.txt\" for writing as out\n\ + \x20\x20\x20\x20\x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20\x20\x20\x20\x20write line value to out\n\ + \x20\x20\x20\x20\x20\x20\x20\x20close out\n\ + \x20\x20\x20\x20end\n\ + end\n", + ), + ]; + + for (scope, source) in sources { + assert!( + typecheck(source).is_ok(), + "a fresh File handle in {scope} must select only the classic write \ + reading; errors: {:?}\nsource:\n{source}", + typecheck(source).err() + ); + } +} + +#[test] +fn reconstructed_local_file_type_does_not_retype_an_outer_visible_binding() { + // The type checker must reconstruct `out` as File while checking the loop, + // then expose the original outer Text binding after leaving that scope. + let source = "store out as \"outer.txt\"\n\ + main loop:\n\ + \x20\x20\x20\x20open file at \"inner.txt\" for writing as out\n\ + \x20\x20\x20\x20close out\n\ + \x20\x20\x20\x20break\n\ + end loop\n\ + close out\n"; + let errors = + typecheck(source).expect_err("the outer Text binding must remain Text after the loop"); + assert!( + errors.contains("file or stream handle") || errors.contains("File"), + "expected the outer Text/handle diagnostic, got: {errors}" + ); +} + +#[test] +fn ambiguous_write_uses_the_classic_branch_for_a_real_open_file_handle() { + let temp = TempDir::new().expect("tempdir"); + let program_path = temp.path().join("program.wfl"); + let output_path = temp.path().join("actual-output.txt"); + let wfl_output_path = output_path.to_string_lossy().replace('\\', "/"); + let source = format!( + "define action called dump:\n\ + \x20\x20\x20\x20open file at \"{wfl_output_path}\" for writing as out\n\ + \x20\x20\x20\x20store line value as \"classic\"\n\ + \x20\x20\x20\x20write line value to out\n\ + \x20\x20\x20\x20close out\n\ + end action\n\ + call dump\n" + ); + fs::write(&program_path, source).expect("write WFL program"); + + let output = Command::new(env!("CARGO_BIN_EXE_wfl")) + .arg(&program_path) + .output() + .expect("run WFL program"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.status.success(), + "a real opened File handle must select the classic write branch; output:\n{combined}" + ); + assert_eq!( + fs::read_to_string(&output_path).expect("read output file"), + "classic", + "the classic fallback content must be written to the opened handle" + ); +} diff --git a/tests/outbound_stream_absolute_lifetime_test.rs b/tests/outbound_stream_absolute_lifetime_test.rs new file mode 100644 index 00000000..8cdb4f3d --- /dev/null +++ b/tests/outbound_stream_absolute_lifetime_test.rs @@ -0,0 +1,103 @@ +//! Real-socket regression for P1 (#3): `outbound_stream_max_seconds` must be a +//! TRUE absolute lifetime — enforced even when a read is served from bytes that +//! were already buffered locally by an earlier read. +//! +//! `stream_pull` already fails a network read once the absolute deadline has +//! elapsed, so an empty-buffer read after the deadline errors correctly. But +//! `next_line`/`next_chunk` serve buffered bytes BEFORE consulting the deadline, +//! so a proxy that pulled a multi-line chunk could keep draining that buffer long +//! after the stream's absolute lifetime expired. This proves a buffered read +//! taken past the deadline now fails (and the upstream is dropped), instead of +//! succeeding. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// Upstream: send a chunked head + ONE body chunk framing two newline-terminated +/// lines ("line1\nline2\n"), then STALL (keep the socket open, send nothing more). +/// Signal on the returned receiver when the proxy drops the upstream connection. +async fn spawn_two_lines_then_stall_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + // One chunk carrying both lines: 0xC = 12 bytes = "line1\nline2\n". + let _ = sock.write_all(b"C\r\nline1\nline2\n\r\n").await; + let _ = sock.flush().await; + // Stall: never send more, never close. Detect the proxy dropping the + // upstream (its handle expired) via a blocking read returning 0/Err. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +#[tokio::test] +async fn test_buffered_read_after_absolute_deadline_expires() { + let (port, mut upstream_closed) = spawn_two_lines_then_stall_upstream().await; + + // Read the first line (which buffers "line2"), sleep past the 1s absolute + // stream lifetime, then read again. The second line comes from the local + // buffer — it must NOT be served after the absolute deadline. + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as s +wait for next line from s as a +wait for 1500 milliseconds +wait for next line from s as b"# + ); + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + + // Idle/run timeout 10s; absolute stream lifetime 1s. + let config = WflConfig { + timeout_seconds: 10, + outbound_stream_max_seconds: 1, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + + let start = Instant::now(); + let result = interp.interpret(&program).await; + let elapsed = start.elapsed(); + + assert!( + result.is_err(), + "a buffered `wait for next line` taken ~1.5s after opening (past the 1s \ + absolute stream lifetime) must fail, not be served from the local buffer" + ); + // It must fail at the buffered read (~1.5s in), not hang out to the 10s + // run timeout or the mock's 30s stall. + assert!( + elapsed < Duration::from_secs(5), + "the expired buffered read should fail promptly (took {elapsed:?})" + ); + + // Expiring the handle must drop the upstream (cancel the request), so the + // mock observes its connection close. + tokio::time::timeout(Duration::from_secs(5), &mut upstream_closed) + .await + .expect("upstream was not closed after the stream's absolute lifetime expired") + .expect("upstream close sender dropped"); +} diff --git a/tests/outbound_stream_deadline_test.rs b/tests/outbound_stream_deadline_test.rs new file mode 100644 index 00000000..68df8559 --- /dev/null +++ b/tests/outbound_stream_deadline_test.rs @@ -0,0 +1,84 @@ +//! Real-socket regression for P1: `outbound_stream_max_seconds` must bound an +//! ACTIVE body read, not be overridden by the broader run/budget duration. +//! +//! A mock upstream sends the response head immediately and then stalls (never +//! sends a body chunk). With `timeout_seconds = 10` but +//! `outbound_stream_max_seconds = 1`, a `wait for next chunk` must fail in about +//! one second (the absolute stream deadline), not wait out the ten-second run +//! timeout. Before the fix, `run_http_with_budget` derived its timeout purely +//! from the budget/run duration and discarded the stream's shorter +//! `min(idle, remaining_total)`, so the read waited ~10s. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// Bind an ephemeral port and serve exactly one connection: read the request, +/// send a chunked-encoding response head, then hold the socket open WITHOUT +/// sending any body chunk (a head-then-stall upstream). +async fn spawn_head_then_stall_upstream() -> u16 { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + // Consume the request head so the client's write completes. + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + // Stall: keep the connection open but never send a body chunk. + tokio::time::sleep(Duration::from_secs(30)).await; + drop(sock); + } + }); + port +} + +#[tokio::test] +async fn test_outbound_stream_absolute_deadline_bounds_active_read() { + let port = spawn_head_then_stall_upstream().await; + + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as s +wait for next chunk from s as c"# + ); + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + + // Run/idle timeout 10s, absolute stream lifetime 1s. + let config = WflConfig { + timeout_seconds: 10, + outbound_stream_max_seconds: 1, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + + let start = Instant::now(); + let result = interp.interpret(&program).await; + let elapsed = start.elapsed(); + + assert!( + result.is_err(), + "a stalled stream read must fail, not hang or succeed" + ); + assert!( + elapsed < Duration::from_secs(4), + "`wait for next chunk` must fail near the 1s absolute stream deadline, \ + not the 10s run timeout (took {elapsed:?})" + ); + // And it must not fail instantly either — the head arrived and the 1s clock + // had to elapse. + assert!( + elapsed >= Duration::from_millis(500), + "the read failed too early to be the ~1s absolute deadline (took {elapsed:?})" + ); +} diff --git a/tests/outbound_stream_disconnect_test.rs b/tests/outbound_stream_disconnect_test.rs new file mode 100644 index 00000000..40e2bb1b --- /dev/null +++ b/tests/outbound_stream_disconnect_test.rs @@ -0,0 +1,157 @@ +//! Real-socket regression for P1: a downstream (browser) disconnect cancels a +//! handler BLOCKED in an upstream `wait for next chunk`, closing the upstream TCP +//! connection and recovering the handler — instead of the handler hanging until +//! the absolute stream deadline. +//! +//! Topology: mock upstream (sends one chunk, then stalls) <- WFL proxy server -> +//! reqwest client. The client reads the first proxied chunk, then disconnects +//! while the handler is blocked reading the (stalled) upstream. The mock detects +//! its own connection closing (a blocking read returns 0 at peer close) only if +//! the handler's blocked upstream read was actually cancelled. + +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// Upstream: send a chunked head + one body chunk, then STALL (send nothing +/// more, so the proxy's next read blocks). Detect the proxy dropping the +/// connection via a blocking read that returns 0 at peer close. +async fn spawn_one_chunk_then_stall_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.write_all(b"5\r\nhello\r\n").await; + let _ = sock.flush().await; + // Stall: send nothing more. Block on read; when the proxy drops the + // upstream (its blocked read cancelled by the client disconnect), the + // peer close surfaces here as Ok(0) / Err. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} // unexpected client->server data; ignore + } + } + } + }); + (port, rx) +} + +fn start_proxy_server(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + if let Err(errors) = interp.interpret(&ast).await { + panic!("proxy interpreter failed: {errors:?}"); + } + }); + }) +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("proxy server on {addr} did not become ready"); +} + +#[tokio::test] +async fn test_downstream_disconnect_cancels_blocked_upstream_read() { + let (upstream_port, mut upstream_disconnect) = spawn_one_chunk_then_stall_upstream().await; + + let proxy_port = common::free_tcp_port(); + // The handler proxies: read chunks from upstream and write them downstream. + // After the first chunk it blocks on the stalled upstream. `outbound_stream_max_seconds` + // is the default (300s), so ONLY a disconnect can cancel that blocked read + // within the test window. + let code = format!( + r#" + listen on port {proxy_port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 20000 + store p as req["path"] + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{upstream_port}/" and stream response as up + start streaming response to req with status 200 and content type "text/plain" as down + count from 1 to 100: + wait for next chunk from up as c + check if c is nothing: + break + otherwise: + write chunk c to down + end check + end count + end check + end loop + "# + ); + let server = start_proxy_server(code); + wait_for_server(proxy_port).await; + + // Client: read the first proxied chunk, then DISCONNECT (drop the response) + // while the handler is blocked reading the stalled upstream. + { + let resp = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/proxy")) + .send() + .await + .expect("proxy request failed"); + assert_eq!(resp.status().as_u16(), 200); + let mut resp = resp; + let first = resp.chunk().await.expect("read first chunk"); + assert_eq!( + first.as_deref(), + Some(&b"hello"[..]), + "expected the first proxied chunk" + ); + // Drop `resp` here -> client disconnects. + } + + // The mock upstream must observe ITS connection close promptly — proving the + // handler's blocked upstream read was cancelled by the disconnect (not left + // hanging until the absolute deadline). + tokio::time::timeout(Duration::from_secs(5), &mut upstream_disconnect) + .await + .expect( + "upstream was not closed after the client disconnected — blocked read not cancelled", + ) + .expect("upstream disconnect sender dropped"); + + // Stop the proxy server. + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/shutdown")) + .send() + .await; + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("server join task failed: {e}"), + } +} diff --git a/tests/outbound_stream_head_disconnect_test.rs b/tests/outbound_stream_head_disconnect_test.rs new file mode 100644 index 00000000..0c922bc4 --- /dev/null +++ b/tests/outbound_stream_head_disconnect_test.rs @@ -0,0 +1,149 @@ +//! Real-socket regression for P1 (#1): a downstream (browser) disconnect must +//! cancel a proxy handler blocked in the UPSTREAM HEAD phase (`open url ... and +//! stream response`), before any `start streaming response`, not only a blocked +//! body read. +//! +//! Topology: an upstream that WITHHOLDS its response head <- WFL concurrent proxy +//! -> a client that connects and disconnects while the handler is blocked opening +//! the upstream. The upstream must observe its connection close promptly (the +//! handler cancelled the head open because the client went away), and an unrelated +//! `/ping` must still be served throughout. + +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// Upstream: accept, read the request, then WITHHOLD the response head (send +/// nothing). Signal when the proxy drops the connection (peer close => read 0/Err). +async fn spawn_header_withholding_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + // Withhold the response head entirely; just wait for the proxy to drop + // the connection when its client disconnects. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +fn start_proxy_server(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + if let Err(errors) = interp.interpret(&ast).await { + panic!("proxy interpreter failed: {errors:?}"); + } + }); + }) +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("proxy server on {addr} did not become ready"); +} + +#[tokio::test] +async fn test_disconnect_cancels_blocked_upstream_head_open() { + let (upstream_port, mut upstream_closed) = spawn_header_withholding_upstream().await; + let proxy_port = common::free_tcp_port(); + + let code = format!( + r#" + listen on port {proxy_port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{upstream_port}/" and stream response as up + start streaming response to req with status 200 and content type "text/plain" as down + wait for next chunk from up as c + close down + end check + end check + end loop + "# + ); + let server = start_proxy_server(code); + wait_for_server(proxy_port).await; + + // Client: connect, send the request, then DISCONNECT while the handler is + // blocked opening the (header-withholding) upstream. + { + let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", proxy_port)) + .await + .expect("connect to proxy"); + sock.write_all(b"GET /proxy HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + .await + .expect("send request"); + sock.flush().await.ok(); + // Give the handler a moment to dequeue and reach the blocked head open, + // then drop the socket to disconnect. + tokio::time::sleep(Duration::from_millis(300)).await; + // `sock` drops here -> client disconnects. + } + + // The upstream must observe its connection close promptly — the blocked head + // open was cancelled by the disconnect, not left to wait out the idle timeout. + tokio::time::timeout(Duration::from_secs(4), &mut upstream_closed) + .await + .expect("upstream head open was not cancelled after the client disconnected") + .expect("upstream close sender dropped"); + + // The concurrent loop stayed alive: an unrelated request is still served. + let ping = tokio::time::timeout( + Duration::from_secs(5), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/ping")) + .send(), + ) + .await + .expect("/ping timed out") + .expect("/ping failed"); + assert_eq!(ping.status().as_u16(), 200); + assert_eq!(ping.text().await.unwrap(), "pong"); + + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/shutdown")) + .send() + .await; + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("server join task failed: {e}"), + } +} diff --git a/tests/outbound_stream_open_expiry_test.rs b/tests/outbound_stream_open_expiry_test.rs new file mode 100644 index 00000000..eb787b2a --- /dev/null +++ b/tests/outbound_stream_open_expiry_test.rs @@ -0,0 +1,159 @@ +//! Real-socket regression (maintainer re-review, P1): `outbound_stream_max_seconds` +//! must be a TRUE absolute lifetime enforced in real time — even when the handler +//! NEVER reads the opened stream. +//! +//! The deadline was previously consulted only on the next read, so a handler that +//! opened an outbound stream and then parked (or did other work) without reading +//! kept the upstream connection alive indefinitely past the cap — contradicting the +//! documented "can never live past this hard cap". This proves the upstream is +//! dropped at the cap with NO read performed, well before the program ends. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// Upstream: send a valid response head, then STALL (never send more, never close). +/// Signal on the returned receiver when the proxy drops the upstream connection. +async fn spawn_head_then_stall_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + // Stall: never send a body, never close. Detect the proxy dropping the + // upstream (its handle reaped) via a blocking read returning 0/Err. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +#[tokio::test] +async fn test_opened_but_unread_stream_expires_at_the_absolute_cap() { + let (port, mut upstream_closed) = spawn_head_then_stall_upstream().await; + + // Open the stream and then just WAIT — never read a chunk/line. With a 1s + // absolute cap the upstream must be dropped ~1s in, long before the 6s wait + // (and the program-end cleanup that would otherwise mask a missing reaper). + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as s +wait for 6000 milliseconds"# + ); + + // Capture whether setup (open) succeeded and the program completed. + let (result_tx, result_rx) = std::sync::mpsc::channel::>(); + let client = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("client runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 30, + outbound_stream_max_seconds: 1, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + let result = interp.interpret(&program).await; + let summary = match result { + Ok(_) => Ok(()), + Err(errs) => Err(format!("{errs:?}")), + }; + let _ = result_tx.send(summary); + }); + }); + + let start = Instant::now(); + tokio::time::timeout(Duration::from_secs(4), &mut upstream_closed) + .await + .expect("the opened-but-unread upstream was not dropped at the absolute cap") + .expect("upstream close sender dropped"); + let elapsed = start.elapsed(); + + // Lower bound: reaper should not fire instantly (setup must succeed and the + // 1s cap must be waited out). Upper bound: well before the 6s program park + // that would mask a missing reaper (issue #642 R3). + assert!( + elapsed >= Duration::from_millis(700), + "upstream should be reaped near the 1s absolute cap, not instantly; took {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "the upstream should be reaped at the ~1s absolute cap, not at program end; \ + took {elapsed:?}" + ); + + let summary = result_rx + .recv_timeout(Duration::from_secs(8)) + .expect("interpreter should finish after the program wait"); + // Opening an unread stream is successful setup; the program parks 6s and + // completes cleanly (the reaper only drops the upstream handle — it does not + // fail an unread stream by itself). + assert!( + summary.is_ok(), + "opened-but-unread stream program should complete after setup; got: {summary:?}" + ); + + match tokio::task::spawn_blocking(move || client.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("client join task failed: {e}"), + } +} + +#[tokio::test] +async fn test_read_after_unread_stream_expiry_reports_typed_timeout() { + let (port, upstream_closed) = spawn_head_then_stall_upstream().await; + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as s +wait for 1200 milliseconds +wait for next chunk from s as chunk"# + ); + + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 30, + outbound_stream_max_seconds: 1, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + let errors = interp + .interpret(&program) + .await + .expect_err("reading a stream after its hard deadline must fail"); + let message = format!("{errors:?}"); + + assert!( + message.contains("kind: Timeout"), + "expired stream must preserve ErrorKind::Timeout, got: {message}" + ); + assert!( + !message.to_lowercase().contains("unknown or already-closed"), + "expired stream must not degrade to an unknown-handle error: {message}" + ); + tokio::time::timeout(Duration::from_secs(2), upstream_closed) + .await + .expect("expired unread stream should drop its upstream") + .expect("upstream close sender dropped"); +} diff --git a/tests/outbound_stream_ownership_test.rs b/tests/outbound_stream_ownership_test.rs new file mode 100644 index 00000000..48476777 --- /dev/null +++ b/tests/outbound_stream_ownership_test.rs @@ -0,0 +1,78 @@ +//! Real-socket regression for P1: an outbound stream is handler-OWNED — when the +//! run/handler ends with the stream still open (no explicit `close`), the handle +//! is dropped, cancelling the in-flight upstream request. Otherwise an abandoned +//! proxy read leaks the upstream connection until the whole interpreter tears +//! down. +//! +//! The mock upstream streams one chunk and then keeps trying to write; when the +//! client drops the connection its write fails and it signals a oneshot. The WFL +//! program reads one chunk and ends WITHOUT `close`, while the interpreter is +//! still alive — so a leak would keep the connection open and the test times out. + +use std::sync::Arc; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// One-connection upstream: send a chunked head + one body chunk, then keep +/// writing keepalive chunks. When the client has disconnected, a write fails and +/// we signal via the oneshot. +async fn spawn_streaming_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.write_all(b"5\r\nhello\r\n").await; // one chunk: "hello" + let _ = sock.flush().await; + + // Keep sending; the first failed write means the client dropped. + for _ in 0..100 { + tokio::time::sleep(Duration::from_millis(100)).await; + if sock.write_all(b"1\r\nx\r\n").await.is_err() || sock.flush().await.is_err() { + let _ = tx.send(()); + return; + } + } + } + }); + (port, rx) +} + +#[tokio::test] +async fn test_outbound_stream_closed_when_run_ends_without_close() { + let (port, disconnect_rx) = spawn_streaming_upstream().await; + + // Reads one chunk, then the program ENDS without `close s`. + let code = format!( + r#"open url at "http://127.0.0.1:{port}/" and stream response as s +wait for next chunk from s as first"# + ); + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + + let mut interp = Interpreter::with_config(Arc::new(WflConfig::default())); + interp.interpret(&program).await.expect("interpret"); + + // The interpreter is still alive here, so only handler-exit cleanup (not the + // interpreter's own teardown) can have dropped the outbound handle. The mock + // therefore only sees its client disconnect if that cleanup cancelled the + // upstream — a leak would keep the connection open and this times out. + tokio::time::timeout(Duration::from_secs(5), disconnect_rx) + .await + .expect("upstream not disconnected after the run ended — outbound handle leaked") + .expect("disconnect sender dropped unexpectedly"); + + drop(interp); +} diff --git a/tests/outbound_stream_reaper_race_test.rs b/tests/outbound_stream_reaper_race_test.rs new file mode 100644 index 00000000..10498c8a --- /dev/null +++ b/tests/outbound_stream_reaper_race_test.rs @@ -0,0 +1,200 @@ +//! Real-socket regression (issue #642 P1): the absolute-lifetime reaper and an +//! active body read must share one atomic lifecycle. +//! +//! If the reaper only removes a parked handle, a read that took the handle out +//! for the await can win the read/timeout race and reinsert the expired handle — +//! leaving a live upstream past the documented real-time hard cap. With the fix: +//! the reaper marks a shared slot expired; `put_stream` refuses reinsertion; the +//! next/current outcome surfaces a typed Timeout and the upstream is dropped. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +/// Upstream: send a valid response head, then stall (never body, never close). +/// Signal when the proxy drops the upstream connection. +async fn spawn_head_then_stall_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +#[tokio::test] +async fn test_active_read_near_deadline_surfaces_timeout_and_drops_upstream() { + let (port, mut upstream_closed) = spawn_head_then_stall_upstream().await; + + // Cap is 1s. Immediately start a body read that will park on the stalled + // upstream; the reaper must still expire the slot and cancel the read as a + // Timeout (not "unknown/already closed"), dropping the upstream ~at the cap. + let code = format!( + r#" + open url at "http://127.0.0.1:{port}/" and stream response as s + wait for next chunk from s as c + display c + "# + ); + + // Send only a serializable error summary — `Value` is not `Send`. + let (result_tx, result_rx) = std::sync::mpsc::channel::>(); + let client = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("client runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 30, // idle timeout long enough that absolute cap wins + outbound_stream_max_seconds: 1, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + let result = interp.interpret(&program).await; + let summary = match result { + Ok(_) => Ok(()), + Err(errs) => Err(format!("{errs:?}")), + }; + let _ = result_tx.send(summary); + }); + }); + + let start = Instant::now(); + tokio::time::timeout(Duration::from_secs(4), &mut upstream_closed) + .await + .expect("upstream was not dropped near the absolute cap during an active read") + .expect("upstream close sender dropped"); + let elapsed = start.elapsed(); + assert!( + elapsed >= Duration::from_millis(700), + "the 1s hard lifetime must not fire as an unrelated immediate error; took {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(3), + "upstream should drop near the 1s absolute cap, not the 30s idle timeout; took {elapsed:?}" + ); + + let result = result_rx + .recv_timeout(Duration::from_secs(5)) + .expect("interpreter should finish after the absolute-cap timeout"); + match tokio::task::spawn_blocking(move || client.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("client join task failed: {e}"), + } + + let msg = result.expect_err("active read past absolute cap must fail (Timeout), not succeed"); + assert!( + msg.contains("kind: Timeout"), + "absolute-cap cancellation must preserve ErrorKind::Timeout, got: {msg}" + ); + assert!( + !msg.to_lowercase().contains("closed"), + "expired slot must surface Timeout, not a closed-stream error; got: {msg}" + ); +} + +#[tokio::test] +async fn test_rapid_open_close_completes_against_stalled_upstreams() { + // Open and immediately close many outbound streams against a real (stalling) + // upstream. This real-socket smoke test proves close does not wait for the hard + // lifetime. Sleeping-task accounting itself is asserted by the retained-runtime + // unit test in `interpreter::tests`, where runtime shutdown cannot hide a leak. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind"); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + break; + }; + tokio::spawn(async move { + let mut buf = [0u8; 512]; + let _ = sock.read(&mut buf).await; + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + // Stall until the proxy drops us on close. + let mut b = [0u8; 64]; + loop { + match sock.read(&mut b).await { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + } + }); + } + }); + + // Cap is large (60s); explicit close must still complete promptly for all 40 + // sequential real upstreams. + let mut lines = String::new(); + for i in 0..40 { + lines.push_str(&format!( + "open url at \"http://127.0.0.1:{port}/s{i}\" and stream response as s{i}\n\ + close s{i}\n" + )); + } + + let code = lines; + let start = Instant::now(); + let client = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 10, + outbound_stream_max_seconds: 60, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + interp + .interpret(&program) + .await + .expect("rapid open/close must succeed"); + }); + }); + tokio::time::timeout(Duration::from_secs(20), async { + while !client.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("rapid open/close client did not finish within 20 seconds"); + if let Err(panic) = client.join() { + std::panic::resume_unwind(panic); + } + let elapsed = start.elapsed(); + // Should finish in well under the 60s cap (and under a few seconds of network). + assert!( + elapsed < Duration::from_secs(20), + "rapid open/close should finish without waiting for the 60s hard cap; took {elapsed:?}" + ); +} diff --git a/tests/property_index_access_test.rs b/tests/property_index_access_test.rs new file mode 100644 index 00000000..34a65488 --- /dev/null +++ b/tests/property_index_access_test.rs @@ -0,0 +1,176 @@ +//! Regression (P1 #5): a bracket index immediately after a `.property` (or +//! `.method(...)`) access must bind to that property value, not split off into a +//! separate bogus list-literal statement. +//! +//! Before the fix, `store ct as obj.headers["content-type"]` parsed as TWO +//! statements — `store ct as obj.headers` (a `PropertyAccess`) followed by a +//! standalone `["content-type"]` list literal — silently dropping the lookup so +//! `ct` was bound to the whole `headers` map. This proves both the AST shape +//! (one `IndexAccess` over the `PropertyAccess`) and the runtime value. + +use std::fs; +use std::process::Command; +use tempfile::TempDir; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Literal, Statement}; + +fn parse(src: &str) -> wfl::parser::ast::Program { + let tokens = lex_wfl_with_positions(src); + Parser::new(&tokens).parse().expect("parse should succeed") +} + +fn wfl_exe() -> &'static str { + env!("CARGO_BIN_EXE_wfl") +} + +/// Run inline WFL source, returning (stdout+stderr, exit code). +fn run_src(src: &str) -> (String, Option) { + let dir = TempDir::new().expect("tempdir"); + let path = dir.path().join("main.wfl"); + fs::write(&path, src).unwrap(); + let output = Command::new(wfl_exe()) + .arg(&path) + .output() + .expect("failed to execute WFL"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + drop(dir); + (combined, output.status.code()) +} + +#[test] +fn property_then_index_is_one_index_access_over_property_access() { + // A single statement: `store ct as obj.headers["content-type"]`. + let program = parse("store ct as obj.headers[\"content-type\"]\n"); + assert_eq!( + program.statements.len(), + 1, + "the property-then-index expression must not split into extra statements; got {:#?}", + program.statements + ); + + let value = match &program.statements[0] { + Statement::VariableDeclaration { name, value, .. } => { + assert_eq!(name, "ct"); + value + } + other => panic!("expected a VariableDeclaration, got {other:#?}"), + }; + + // Outer node is IndexAccess["content-type"] whose collection is the + // PropertyAccess obj.headers. + match value { + Expression::IndexAccess { + collection, index, .. + } => { + match index.as_ref() { + Expression::Literal(Literal::String(key), ..) => { + assert_eq!( + key.as_ref(), + "content-type", + "index key must be the bracket string" + ) + } + other => panic!("expected a string index, got {other:#?}"), + } + match collection.as_ref() { + Expression::PropertyAccess { + object, property, .. + } => { + assert_eq!(property, "headers"); + assert!( + matches!(object.as_ref(), Expression::Variable(n, ..) if n == "obj"), + "property-access object must be the variable `obj`, got {object:#?}" + ); + } + other => panic!( + "index collection must be the PropertyAccess obj.headers, got {other:#?}" + ), + } + } + other => panic!("expected IndexAccess over PropertyAccess, got {other:#?}"), + } +} + +#[test] +fn chained_property_index_nests_left_to_right() { + // `grid.rows[0][1]` -> IndexAccess( IndexAccess( PropertyAccess(grid.rows), 0 ), 1 ) + let program = parse("store cell as grid.rows[0][1]\n"); + assert_eq!(program.statements.len(), 1, "must be one statement"); + let value = match &program.statements[0] { + Statement::VariableDeclaration { value, .. } => value, + other => panic!("expected VariableDeclaration, got {other:#?}"), + }; + // Outermost index is [1]. + let inner = match value { + Expression::IndexAccess { collection, .. } => collection, + other => panic!("expected outer IndexAccess, got {other:#?}"), + }; + // Next index is [0] over the property access. + match inner.as_ref() { + Expression::IndexAccess { collection, .. } => { + assert!( + matches!(collection.as_ref(), Expression::PropertyAccess { property, .. } if property == "rows"), + "innermost collection must be grid.rows, got {collection:#?}" + ); + } + other => panic!("expected inner IndexAccess, got {other:#?}"), + } +} + +#[test] +fn property_index_runtime_value_is_the_indexed_field_not_the_whole_map() { + // Unambiguous: the correct index result ("BBB") differs from the property + // map, so a split (ct = the headers map) fails the equality check. + let src = "create map inner:\n\ + \x20 \"a\" is \"AAA\"\n\ + \x20 \"b\" is \"BBB\"\n\ + end map\n\ + create map outer:\n\ + \x20 \"headers\" is inner\n\ + end map\n\ + store ct as outer.headers[\"b\"]\n\ + check if ct is equal to \"BBB\":\n\ + \x20 display \"INDEX_OK\"\n\ + otherwise:\n\ + \x20 display \"INDEX_WRONG\"\n\ + end check\n"; + let (out, code) = run_src(src); + assert_eq!(code, Some(0), "program should exit 0: {out}"); + assert!( + out.contains("INDEX_OK"), + "`outer.headers[\"b\"]` must yield the indexed value \"BBB\", not the whole map: {out}" + ); + assert!( + !out.contains("INDEX_WRONG"), + "the property-then-index lookup returned the wrong value: {out}" + ); +} + +#[test] +fn outbound_stream_header_index_parses_as_single_statement() { + // The canonical proxy pattern from the docs: + // `store ct as upstream.headers["content-type"]`. It must be one statement + // (an IndexAccess over the PropertyAccess), not a split that drops the key. + let program = parse( + "open url at \"http://example.com\" and stream response as upstream\n\ + store ct as upstream.headers[\"content-type\"]\n", + ); + assert_eq!( + program.statements.len(), + 2, + "expected exactly the open + store statements, no split list literal; got {:#?}", + program.statements + ); + match &program.statements[1] { + Statement::VariableDeclaration { value, .. } => assert!( + matches!(value, Expression::IndexAccess { .. }), + "the header lookup must be an IndexAccess, got {value:#?}" + ), + other => panic!("expected the store statement, got {other:#?}"), + } +} diff --git a/tests/response_expression_disconnect_runtime_test.rs b/tests/response_expression_disconnect_runtime_test.rs new file mode 100644 index 00000000..29b09775 --- /dev/null +++ b/tests/response_expression_disconnect_runtime_test.rs @@ -0,0 +1,455 @@ +//! Real-socket coverage for response-expression cancellation (issue #642). +//! +//! A buffered response body and each fallible streaming-head operand (status, +//! content type, and headers) are evaluated before the response commits. If the +//! browser disconnects during one of those evaluations, the handler must cancel +//! the evaluation, release its real outbound stream, and leave the concurrent +//! server able to serve unrelated work. +//! +//! Each case uses two real upstream sockets. The data upstream sends one chunk +//! and then withholds the rest of its body. After consuming that first chunk, the +//! WFL action opens a checkpoint stream, consumes its marker, explicitly closes +//! it, and only then blocks on the data upstream again. The test waits for the +//! checkpoint socket to close before dropping the browser socket, so the +//! synchronization is event-controlled rather than based on a handler sleep. + +use std::time::Duration; + +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::oneshot; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +const PROBE_DEADLINE: Duration = Duration::from_secs(10); + +struct EvaluationProbe { + data_port: u16, + checkpoint_port: u16, + checkpoint_passed: oneshot::Receiver>, + data_closed: oneshot::Receiver>, +} + +async fn read_http_head(socket: &mut tokio::net::TcpStream) -> Result, String> { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + while !request.ends_with(b"\r\n\r\n") { + let count = socket + .read(&mut byte) + .await + .map_err(|error| format!("read request head: {error}"))?; + if count == 0 { + return Err("peer closed before sending a complete request head".to_string()); + } + request.push(byte[0]); + if request.len() > 16 * 1024 { + return Err("request head exceeded 16 KiB".to_string()); + } + } + Ok(request) +} + +async fn wait_for_peer_close(socket: &mut tokio::net::TcpStream) { + let mut byte = [0u8; 1]; + loop { + match socket.read(&mut byte).await { + Ok(0) | Err(_) => return, + Ok(_) => {} + } + } +} + +async fn spawn_evaluation_probe(label: &'static str) -> EvaluationProbe { + let data_listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .unwrap_or_else(|error| panic!("{label}: bind data upstream: {error}")); + let data_port = data_listener + .local_addr() + .unwrap_or_else(|error| panic!("{label}: inspect data upstream address: {error}")) + .port(); + let (data_closed_tx, data_closed) = oneshot::channel(); + tokio::spawn(async move { + let result = async { + let (mut socket, _) = data_listener + .accept() + .await + .map_err(|error| format!("{label}: accept data upstream: {error}"))?; + read_http_head(&mut socket).await?; + socket + .write_all( + b"HTTP/1.1 200 OK\r\n\ + Content-Type: application/octet-stream\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: keep-alive\r\n\r\n\ + 5\r\nready\r\n", + ) + .await + .map_err(|error| format!("{label}: write data marker: {error}"))?; + socket + .flush() + .await + .map_err(|error| format!("{label}: flush data marker: {error}"))?; + wait_for_peer_close(&mut socket).await; + Ok(()) + } + .await; + let _ = data_closed_tx.send(result); + }); + + let checkpoint_listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .unwrap_or_else(|error| panic!("{label}: bind checkpoint upstream: {error}")); + let checkpoint_port = checkpoint_listener + .local_addr() + .unwrap_or_else(|error| panic!("{label}: inspect checkpoint address: {error}")) + .port(); + let (checkpoint_passed_tx, checkpoint_passed) = oneshot::channel(); + tokio::spawn(async move { + let result = async { + let (mut socket, _) = checkpoint_listener + .accept() + .await + .map_err(|error| format!("{label}: accept checkpoint: {error}"))?; + read_http_head(&mut socket).await?; + // Deliberately omit the terminating zero-length chunk. The WFL + // action reads this marker and then closes the still-live stream. + // Observing that close proves the action passed the checkpoint. + socket + .write_all( + b"HTTP/1.1 200 OK\r\n\ + Content-Type: application/octet-stream\r\n\ + Transfer-Encoding: chunked\r\n\ + Connection: keep-alive\r\n\r\n\ + 1\r\nA\r\n", + ) + .await + .map_err(|error| format!("{label}: write checkpoint marker: {error}"))?; + socket + .flush() + .await + .map_err(|error| format!("{label}: flush checkpoint marker: {error}"))?; + wait_for_peer_close(&mut socket).await; + Ok(()) + } + .await; + let _ = checkpoint_passed_tx.send(result); + }); + + EvaluationProbe { + data_port, + checkpoint_port, + checkpoint_passed, + data_closed, + } +} + +struct ProxyServer { + thread: Option>, + abort: Option>, +} + +impl Drop for ProxyServer { + fn drop(&mut self) { + if let Some(abort) = self.abort.take() { + let _ = abort.send(()); + } + } +} + +fn start_proxy_server(code: String) -> ProxyServer { + let (abort, abort_rx) = oneshot::channel(); + let thread = std::thread::Builder::new() + .name("response-expression-disconnect-proxy".to_string()) + .stack_size(wfl::INTERPRETER_STACK_SIZE) + .spawn(move || { + let runtime = tokio::runtime::Runtime::new().expect("create proxy runtime"); + runtime.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens) + .parse() + .unwrap_or_else(|errors| panic!("parse proxy program: {errors:?}")); + let mut interpreter = Interpreter::new(); + tokio::select! { + result = interpreter.interpret(&ast) => { + if let Err(errors) = result { + panic!("proxy interpreter failed: {errors:?}"); + } + } + _ = abort_rx => { + // Test cleanup drops the interpreter and all live listeners. + } + } + }); + }) + .expect("spawn proxy interpreter thread"); + ProxyServer { + thread: Some(thread), + abort: Some(abort), + } +} + +async fn wait_for_server(port: u16) { + let address = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&address).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("proxy server on {address} did not become ready"); +} + +async fn assert_ping_survives(port: u16, context: &str) { + let response = tokio::time::timeout( + PROBE_DEADLINE, + reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/ping")) + .send(), + ) + .await + .unwrap_or_else(|_| panic!("{context}: /ping timed out after expression cancellation")) + .unwrap_or_else(|error| panic!("{context}: /ping failed: {error}")); + assert_eq!( + response.status().as_u16(), + 200, + "{context}: server returned a non-success /ping status" + ); + assert_eq!( + response + .text() + .await + .unwrap_or_else(|error| panic!("{context}: read /ping body: {error}")), + "pong", + "{context}: server returned the wrong /ping body" + ); +} + +async fn disconnect_at_checkpoint( + proxy_port: u16, + path: &'static str, + label: &'static str, + mut probe: EvaluationProbe, +) { + let mut browser = tokio::net::TcpStream::connect(("127.0.0.1", proxy_port)) + .await + .unwrap_or_else(|error| panic!("{label}: connect browser socket: {error}")); + browser + .write_all( + format!( + "GET {path} HTTP/1.1\r\n\ + Host: 127.0.0.1\r\n\ + Connection: close\r\n\r\n" + ) + .as_bytes(), + ) + .await + .unwrap_or_else(|error| panic!("{label}: send browser request: {error}")); + browser + .flush() + .await + .unwrap_or_else(|error| panic!("{label}: flush browser request: {error}")); + + tokio::time::timeout(PROBE_DEADLINE, &mut probe.checkpoint_passed) + .await + .unwrap_or_else(|_| { + panic!("{label}: response expression never passed its upstream checkpoint") + }) + .unwrap_or_else(|_| panic!("{label}: checkpoint task ended without a result")) + .unwrap_or_else(|error| panic!("{error}")); + + assert!( + matches!( + probe.data_closed.try_recv(), + Err(oneshot::error::TryRecvError::Empty) + ), + "{label}: data upstream closed before the browser disconnected" + ); + + // The action has consumed its first real upstream chunk, passed and closed + // its marker stream, and is now blocked reading the unfinished data body. + // Dropping this socket is the only release signal for that evaluation. + drop(browser); + + tokio::time::timeout(PROBE_DEADLINE, &mut probe.data_closed) + .await + .unwrap_or_else(|_| { + panic!("{label}: stalled upstream remained open after browser disconnect") + }) + .unwrap_or_else(|_| panic!("{label}: data upstream task ended without a result")) + .unwrap_or_else(|error| panic!("{error}")); + + assert_ping_survives(proxy_port, label).await; +} + +async fn shutdown_proxy(port: u16, mut server: ProxyServer) { + let _ = tokio::time::timeout( + PROBE_DEADLINE, + reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/shutdown")) + .send(), + ) + .await; + + let graceful = tokio::time::timeout(PROBE_DEADLINE, async { + while !server + .thread + .as_ref() + .expect("proxy thread exists until shutdown") + .is_finished() + { + tokio::task::yield_now().await; + } + }) + .await + .is_ok(); + if !graceful { + let _ = server + .abort + .take() + .expect("proxy abort signal is sent at most once") + .send(()); + } + + let thread = server + .thread + .take() + .expect("proxy thread is joined at most once"); + match tokio::task::spawn_blocking(move || thread.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(error) => panic!("proxy join task failed: {error}"), + } +} + +fn stalled_action( + name: &str, + prefix: &str, + probe: &EvaluationProbe, + return_statements: &str, +) -> String { + format!( + r#" +define action called {name}: + open url at "http://127.0.0.1:{data_port}/data" and stream response as {prefix}_source + wait for next chunk from {prefix}_source as {prefix}_ready + open url at "http://127.0.0.1:{checkpoint_port}/checkpoint" and stream response as {prefix}_checkpoint + wait for next chunk from {prefix}_checkpoint as {prefix}_acknowledged + close {prefix}_checkpoint + wait for next chunk from {prefix}_source as {prefix}_blocked +{return_statements} +end action +"#, + data_port = probe.data_port, + checkpoint_port = probe.checkpoint_port, + ) +} + +#[tokio::test] +async fn response_expression_disconnects_cancel_upstreams_and_preserve_server_liveness() { + let buffered = spawn_evaluation_probe("buffered response content").await; + let stream_status = spawn_evaluation_probe("streaming response status").await; + let stream_content_type = spawn_evaluation_probe("streaming response content type").await; + let stream_headers = spawn_evaluation_probe("streaming response headers").await; + let proxy_port = common::free_tcp_port(); + + let actions = [ + stalled_action( + "stalled_buffered_content", + "buffered_value", + &buffered, + " return \"late body\"", + ), + stalled_action( + "stalled_stream_status", + "stream_status", + &stream_status, + " return 201", + ), + stalled_action( + "stalled_stream_content_type", + "stream_content_type", + &stream_content_type, + " return \"text/plain\"", + ), + stalled_action( + "stalled_stream_headers", + "stream_headers", + &stream_headers, + " create map delayed_headers:\n \"X-Probe\" is \"late\"\n end map\n return delayed_headers", + ), + ] + .join("\n"); + + let program = format!( + r#" +{actions} +listen on port {proxy_port} as srv +main loop concurrently: + wait for request comes in on srv as req with timeout 30000 + store request_path as req["path"] + check if request_path is equal to "/ping": + respond to req with "pong" + otherwise: + check if request_path is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + check if request_path is equal to "/buffered": + respond to req with call stalled_buffered_content + otherwise: + check if request_path is equal to "/stream-status": + start streaming response to req with status call stalled_stream_status as out + close out + otherwise: + check if request_path is equal to "/stream-content-type": + start streaming response to req with status 200 and content type call stalled_stream_content_type as out + close out + otherwise: + start streaming response to req with status 200 and content type "text/plain" and headers call stalled_stream_headers as out + close out + end check + end check + end check + end check + end check +end loop +"# + ); + + let server = start_proxy_server(program); + wait_for_server(proxy_port).await; + + disconnect_at_checkpoint( + proxy_port, + "/buffered", + "buffered response content", + buffered, + ) + .await; + disconnect_at_checkpoint( + proxy_port, + "/stream-status", + "streaming response status", + stream_status, + ) + .await; + disconnect_at_checkpoint( + proxy_port, + "/stream-content-type", + "streaming response content type", + stream_content_type, + ) + .await; + disconnect_at_checkpoint( + proxy_port, + "/stream-headers", + "streaming response headers", + stream_headers, + ) + .await; + + shutdown_proxy(proxy_port, server).await; +} diff --git a/tests/response_stream_backpressure_test.rs b/tests/response_stream_backpressure_test.rs new file mode 100644 index 00000000..08cb7302 --- /dev/null +++ b/tests/response_stream_backpressure_test.rs @@ -0,0 +1,298 @@ +//! Real-socket regressions (maintainer re-review, P1): +//! +//! 1. A backpressured response-stream `write` must be BOUNDED: once the 64-slot +//! channel fills, a client that stays connected but stops reading would pin the +//! handler forever (`main loop` is deadline-exempt). It must instead time out at +//! `web_server_response_timeout_seconds` and error, releasing the handler. +//! +//! 2. Streaming must actually STREAM: the head and an early chunk must be visible on +//! the wire BEFORE the body completes/closes — not buffered until the end. + +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::config::WflConfig; +use wfl::interpreter::error::ErrorKind; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("server on {addr} did not become ready"); +} + +async fn read_response_head(sock: &mut tokio::net::TcpStream) -> Vec { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + let mut head = Vec::new(); + let mut buf = [0u8; 512]; + loop { + let n = tokio::time::timeout_at(deadline, sock.read(&mut buf)) + .await + .expect("timed out waiting for streaming response head") + .expect("failed to read streaming response head"); + assert_ne!(n, 0, "connection closed before the complete response head"); + head.extend_from_slice(&buf[..n]); + if head.windows(4).any(|window| window == b"\r\n\r\n") { + return head; + } + assert!( + head.len() <= 16 * 1024, + "streaming response head exceeded 16 KiB" + ); + } +} + +async fn join_server(server: std::thread::JoinHandle<()>) { + tokio::time::timeout(Duration::from_secs(10), async { + while !server.is_finished() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("server thread did not stop within 10 seconds"); + if let Err(panic) = server.join() { + std::panic::resume_unwind(panic); + } +} + +#[tokio::test] +async fn test_backpressured_write_to_a_non_reading_client_is_bounded() { + let port = common::free_tcp_port(); + + // Handler streams FAR more data than any OS send buffer can hold to a + // connected-but-non-reading client: the socket buffer fills, then the 64-slot + // channel fills, and the next `write` parks on backpressure. With a 2s response + // timeout the parked write must fail (not hang forever); the serial main loop + // propagates that error, so `interpret()` RETURNS instead of pinning the handler. + // The payload is grown by doubling to ~40 KB so a few hundred chunks overflow + // the buffer regardless of its autotuned size; the byte ceiling is raised so the + // write blocks (not a budget rejection) first. + let code = format!( + r#" + listen on port {port} as srv + main loop: + wait for request comes in on srv as req with timeout 60000 + store payload as "0123456789" + count from 1 to 12: + store payload as payload with payload + end count + start streaming response to req with status 200 and content type "text/plain" as out + count from 1 to 100000: + write chunk payload to out + end count + close out + break + end loop + "# + ); + + // Reduce the result to Send error fields before crossing the server-thread + // boundary. Keeping the typed kind separate prevents a broad text assertion + // from accepting an unrelated cancellation or pre-head failure. + let (done_tx, done_rx) = + tokio::sync::oneshot::channel::<(Duration, Result<(), Vec<(ErrorKind, String)>>)>(); + let server = std::thread::spawn(move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("server runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let config = WflConfig { + timeout_seconds: 60, + web_server_response_timeout_seconds: 2, + web_server_max_response_size: 512 * 1024 * 1024, + ..WflConfig::default() + }; + let mut interp = Interpreter::with_config(Arc::new(config)); + let start = Instant::now(); + let result = interp.interpret(&program).await; + let summary = match result { + Ok(_) => Ok(()), + Err(errs) => Err(errs + .into_iter() + .map(|error| (error.kind, error.message)) + .collect()), + }; + let _ = done_tx.send((start.elapsed(), summary)); + }); + }); + + wait_for_server(port).await; + + // Connect and confirm the 200 streaming head first. Only then stop reading and + // hold the socket open. This excludes a pending-request 504, a pre-head + // cancellation, or any other early exit from false-greening the stall test. + let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .expect("connect"); + sock.write_all(b"GET / HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + .await + .expect("send request"); + sock.flush().await.expect("flush request"); + let head = read_response_head(&mut sock).await; + let head_text = String::from_utf8_lossy(&head); + assert!( + head_text.starts_with("HTTP/1.1 200"), + "expected a successful streaming response head, got {head_text:?}" + ); + assert!( + head_text + .to_ascii_lowercase() + .contains("content-type: text/plain"), + "expected the streaming content type in the response head, got {head_text:?}" + ); + // `interpret()` must return once the stalled write times out (~2s). If the write + // were unbounded it would pin the handler and this never fires. + let (elapsed, summary) = tokio::time::timeout(Duration::from_secs(12), done_rx) + .await + .expect("interpret() never returned — the backpressured write pinned the handler forever") + .expect("done sender dropped"); + // Exact typed contract: only the bounded backpressure branch is acceptable. + // A Cancelled disconnect, a generic write error, or any other Timeout is not. + let errors = summary.expect_err( + "backpressured write to a non-reading client must error (write timeout), not succeed", + ); + assert_eq!( + errors.len(), + 1, + "expected exactly one backpressure error, got {errors:?}" + ); + assert_eq!( + errors[0].0, + ErrorKind::Timeout, + "backpressured connected client must produce ErrorKind::Timeout, got {errors:?}" + ); + assert_eq!( + errors[0].1, + "Cannot write to response stream: the client stopped reading (write timed out)", + "backpressure must report the exact stalled-client diagnostic" + ); + // The exact kind/message above identifies the backpressure timeout branch. + // Measure its lower bound from interpreter start: measuring from when the test + // thread happens to observe the head can false-fail if the server had already + // started the write timer before that observation. + assert!( + elapsed >= Duration::from_millis(1500), + "the configured 2s write timeout fired implausibly early; took only {elapsed:?}" + ); + assert!( + elapsed < Duration::from_secs(9), + "the stalled write should time out at ~2s (web_server_response_timeout_seconds), \ + took {elapsed:?}" + ); + + drop(sock); // keep the client connected until the assertion above + join_server(server).await; +} + +#[tokio::test] +async fn test_early_chunk_is_visible_before_the_body_completes() { + let port = common::free_tcp_port(); + + // Handler sends an early chunk + flush, WAITS 2s, then sends a late chunk and + // closes. A streaming client must SEE the early chunk well before the late one — + // proving head/first-chunk visibility before body completion, not buffer-to-end. + let code = format!( + r#" + listen on port {port} as srv + main loop: + wait for request comes in on srv as req with timeout 30000 + start streaming response to req with status 200 and content type "text/plain" as out + write chunk "EARLY" to out + flush out + wait for 2000 milliseconds + write chunk "LATE" to out + close out + break + end loop + "# + ); + + let (done_tx, done_rx) = + tokio::sync::oneshot::channel::>>(); + let server = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("server runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + let result = match interp.interpret(&program).await { + Ok(_) => Ok(()), + Err(errors) => Err(errors + .into_iter() + .map(|error| (error.kind, error.message)) + .collect()), + }; + let _ = done_tx.send(result); + }); + }); + + wait_for_server(port).await; + + let mut resp = reqwest::Client::new() + .get(format!("http://127.0.0.1:{port}/")) + .send() + .await + .expect("request send"); + assert_eq!(resp.status().as_u16(), 200); + + let start = Instant::now(); + let mut early_at = None; + let mut late_at = None; + let mut acc = String::new(); + tokio::time::timeout(Duration::from_secs(6), async { + loop { + match resp.chunk().await { + Ok(Some(bytes)) => { + acc.push_str(&String::from_utf8_lossy(&bytes)); + if early_at.is_none() && acc.contains("EARLY") { + early_at = Some(start.elapsed()); + } + if late_at.is_none() && acc.contains("LATE") { + late_at = Some(start.elapsed()); + } + } + Ok(None) => break, + Err(error) => { + panic!("stream transport failed instead of ending cleanly: {error}") + } + } + } + }) + .await + .expect("streaming body stalled"); + + let early = early_at.expect("the EARLY chunk was never received"); + let late = late_at.expect("the LATE chunk was never received"); + // The early chunk must arrive well before the late one — proving it was flushed + // to the wire while the body was still open, not buffered until close. + assert!( + early < Duration::from_millis(1500), + "the EARLY chunk should be visible almost immediately, arrived at {early:?}" + ); + assert!( + late - early > Duration::from_millis(1000), + "the LATE chunk should arrive ~2s after EARLY (early={early:?}, late={late:?}); \ + a small gap means the body was buffered to completion instead of streamed" + ); + + let interpreter_result = tokio::time::timeout(Duration::from_secs(3), done_rx) + .await + .expect("interpreter did not finish after the streaming body closed") + .expect("interpreter result sender dropped"); + interpreter_result.unwrap_or_else(|errors| { + panic!("streaming visibility program must finish successfully, got {errors:?}") + }); + join_server(server).await; +} diff --git a/tests/stream_handle_type_test.rs b/tests/stream_handle_type_test.rs new file mode 100644 index 00000000..7420060c --- /dev/null +++ b/tests/stream_handle_type_test.rs @@ -0,0 +1,233 @@ +//! Type-contract coverage for stream handles (maintainer review). +//! +//! Stream handles bound by `start streaming response ... as ` and +//! `... stream response as ` are map-shaped objects. `close ` +//! must accept them — before this fix the type checker only accepted a `File` +//! object, so a valid `close out` / `close upstream` produced a spurious +//! "Expected a File object" diagnostic. A concrete scalar must still be rejected. + +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Statement}; +use wfl::typechecker::TypeChecker; + +fn typecheck(code: &str) -> Result<(), String> { + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + TypeChecker::new() + .check_types(&program) + .map_err(|e| format!("{e:?}")) +} + +#[test] +fn test_close_server_response_stream_handle_typechecks() { + // `out` is a streaming-response handle; `close out` must type-check cleanly. + let code = "listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 and content type \"text/plain\" as out\n\ + write line \"hi\" to out\n\ + close out"; + assert!( + typecheck(code).is_ok(), + "closing a streaming-response handle must not be flagged as a non-File: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_close_outbound_stream_handle_typechecks() { + // `upstream` is an outbound streaming handle; `close upstream` must be clean. + let code = "open url at \"http://example.com\" and stream response as upstream\n\ + close upstream"; + assert!( + typecheck(code).is_ok(), + "closing an outbound stream handle must not be flagged as a non-File: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_close_non_file_custom_handle_is_rejected() { + // Only a File custom type is closeable via `close`. Other custom handles + // (a database connection here) are NOT — the interpreter cannot close them, + // so accepting `close db` would be a false negative. + let code = "open database at \"sqlite::memory:\" as db\nclose db"; + let errors = typecheck(code).expect_err("closing a database handle must be a type error"); + assert!( + errors.contains("file or stream handle") || errors.contains("File"), + "expected a close-operand type error for a database handle, got: {errors}" + ); +} + +#[test] +fn test_outbound_stream_handle_fields_are_indexable() { + // The outbound handle now has a distinct `Custom("HttpStream")` type; reading + // its fields by index must still type-check (the field type is runtime-known). + // Mirrors the docs example: a direct field and a nested header lookup. + let code = "open url at \"http://example.com\" and stream response as resp\n\ + store code as resp[\"status\"]\n\ + store ct as resp[\"headers\"][\"content-type\"]\n\ + close resp"; + assert!( + typecheck(code).is_ok(), + "indexing a stream handle's fields must still type-check: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_outbound_stream_handle_dot_access_typechecks() { + // The canonical docs use dot access (`upstream.status`, + // `upstream.headers["content-type"]`); the distinct handle type must support + // it, not just bracket notation. + let code = "open url at \"http://example.com\" and stream response as upstream\n\ + display upstream.status\n\ + store ct as upstream.headers[\"content-type\"]\n\ + close upstream"; + assert!( + typecheck(code).is_ok(), + "dot access on a stream handle must type-check: {:?}", + typecheck(code).err() + ); + + // Type-checking alone is a false green here: even when + // `upstream.headers["content-type"]` mis-parses into two statements + // (`store ct as upstream.headers` + a stray `["content-type"]` list literal) + // both halves type-check. Assert the *structure*: the header lookup binds as + // one IndexAccess over the PropertyAccess, so the key is not silently dropped. + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let ct_stmt = program + .statements + .iter() + .find_map(|s| match s { + Statement::VariableDeclaration { name, value, .. } if name == "ct" => Some(value), + _ => None, + }) + .expect("a `store ct as ...` statement"); + assert!( + matches!(ct_stmt, Expression::IndexAccess { .. }), + "`upstream.headers[\"content-type\"]` must bind `ct` to an IndexAccess over the \ + PropertyAccess (the lookup must not split off), got {ct_stmt:#?}" + ); +} + +#[test] +fn test_stream_handle_numeric_index_is_rejected() { + // Runtime object indexing requires a text field name; a numeric key must be a + // static type error (it was, back when the handle was Map). + let code = "open url at \"http://example.com\" and stream response as resp\n\ + store x as resp[5]\n\ + close resp"; + let errors = typecheck(code).expect_err("a numeric stream-handle key must be a type error"); + assert!( + errors.contains("field name must be text") || errors.contains("must be text"), + "expected a text-key type error, got: {errors}" + ); +} + +#[test] +fn test_wait_for_next_from_non_stream_is_rejected() { + // `wait for next chunk|line from ` requires an outbound stream handle; + // reading one from a concrete number is a static type error, not deferred. + for verb in ["chunk", "line"] { + let code = format!("store n as 5\nwait for next {verb} from n as c"); + let errors = + typecheck(&code).expect_err("reading a stream from a number must be a type error"); + assert!( + errors.contains("stream"), + "expected a stream-source type error for `wait for next {verb}`, got: {errors}" + ); + } +} + +#[test] +fn test_wait_for_next_from_http_stream_is_ok() { + // The valid form (reading from an outbound stream handle) must type-check. + let code = "open url at \"http://example.com\" and stream response as up\n\ + wait for next chunk from up as c\n\ + close up"; + assert!( + typecheck(code).is_ok(), + "reading from an outbound stream handle must type-check: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_flush_non_stream_is_rejected() { + // `flush ` requires a server response-stream handle. + let code = "store n as 5\nflush n"; + let errors = typecheck(code).expect_err("flushing a number must be a type error"); + assert!( + errors.contains("stream"), + "expected a stream-target type error for `flush`, got: {errors}" + ); +} + +#[test] +fn test_write_line_to_non_stream_is_rejected() { + // An UNAMBIGUOUS stream write (literal value => no classic file-write fallback) + // to a concrete number cannot be a file write either, so it is a type error. + let code = "store n as 5\nwrite line \"x\" to n"; + let errors = typecheck(code).expect_err("writing to a number must be a type error"); + assert!( + errors.contains("stream"), + "expected a stream-target type error for `write line`, got: {errors}" + ); +} + +#[test] +fn test_write_and_flush_response_stream_is_ok() { + // The valid server-streaming path must type-check cleanly. + let code = "listen on port 8080 as s\n\ + wait for request comes in on s as req\n\ + start streaming response to req with status 200 and content type \"text/plain\" as out\n\ + write line \"hi\" to out\n\ + flush out\n\ + close out"; + assert!( + typecheck(code).is_ok(), + "writing/flushing a response-stream handle must type-check: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_write_line_variable_to_file_path_is_ok_when_classic_lead_is_defined() { + // The AMBIGUOUS merged form (`write line ... to `) carries a + // classic file-write fallback whose legacy variable is the full merged name + // (`line payload`). A concrete text path selects that branch, so its actual + // lead must be defined; the speculative stream lead (`payload`) need not be. + let code = "store line payload as \"data\"\n\ + write line payload to \"/tmp/out.txt\""; + assert!( + typecheck(code).is_ok(), + "an ambiguous file write must accept its defined classic lead: {:?}", + typecheck(code).err() + ); +} + +#[test] +fn test_close_ordinary_map_is_rejected() { + // A plain user map is NOT closeable — only file/stream handles are. This is + // the tightening that a distinct handle type buys over accepting any `Map`. + let code = "create map m:\n \"k\" is \"v\"\nend map\nclose m"; + let errors = typecheck(code).expect_err("closing an ordinary map must be a type error"); + assert!( + errors.contains("file or stream handle") || errors.contains("File"), + "expected a close-operand type error for a plain map, got: {errors}" + ); +} + +#[test] +fn test_close_scalar_is_still_rejected() { + // A concrete non-handle value is still a type error — the fix widens `close` + // to file/stream handles, it does not make `close` accept anything. + let code = "store n as 5\nclose n"; + let errors = typecheck(code).expect_err("closing a number must be a type error"); + assert!( + errors.contains("file or stream handle") || errors.contains("File"), + "expected a close-operand type error, got: {errors}" + ); +} diff --git a/tests/transpiler_test.rs b/tests/transpiler_test.rs index de042f21..bedcb188 100644 --- a/tests/transpiler_test.rs +++ b/tests/transpiler_test.rs @@ -67,6 +67,24 @@ fn test_display_statement() { assert_contains(&js, r#"WFL.display("Hello, World!");"#); } +#[test] +fn test_ambiguous_write_line_uses_classic_file_fallback() { + let source = "store line note as \"hello\"\nwrite line note to \"f.txt\""; + let js = transpile_wfl(source) + .expect("an ambiguous write with a classic file fallback must transpile"); + assert_contains(&js, "WFL.file.write(\"f.txt\".path, line_note);"); +} + +#[test] +fn test_unambiguous_stream_write_still_fails_to_transpile() { + let error = transpile_wfl("write line \"hello\" to out") + .expect_err("a genuine response-stream write has no JavaScript translation"); + assert!( + error.contains("Streaming HTTP statements are not supported"), + "expected the streaming-specific transpiler error, got: {error}" + ); +} + #[test] fn test_if_statement() { let source = r#" @@ -680,3 +698,25 @@ fn test_describe_and_test_descriptions_are_javascript_string_literals() { assert_contains(&js, "describe(\"quoted \\\"suite\\\"\", function()"); assert_contains(&js, "it(\"quoted \\\"case\\\"\", function()"); } + +#[test] +fn test_main_loop_concurrently_fails_to_transpile() { + // `main loop concurrently:` has no serial JavaScript translation; it must + // error rather than silently emit a serial loop. + let source = "main loop concurrently:\n display \"x\"\nend loop"; + let result = transpile_wfl(source); + // Assert the specific transpiler rejection, not merely any error — so a + // future parse failure can't masquerade as the intended rejection. + let error = result.expect_err("main loop concurrently should fail to transpile"); + assert!( + error.contains("not supported in JavaScript transpilation"), + "expected the unsupported-transpilation error, got: {error}" + ); + + // Plain `main loop` still transpiles. + let serial = transpile_wfl("main loop:\n display \"x\"\nend loop"); + assert!( + serial.is_ok(), + "plain main loop should transpile: {serial:?}" + ); +} diff --git a/tests/typechecker_response_stream_join_test.rs b/tests/typechecker_response_stream_join_test.rs new file mode 100644 index 00000000..dadd9330 --- /dev/null +++ b/tests/typechecker_response_stream_join_test.rs @@ -0,0 +1,211 @@ +//! Regression coverage for conservative type-state joins across conditional +//! control flow involving response-stream and file-handle bindings. + +use std::sync::Arc; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, FileOpenMode, Literal, Operator, Program, Statement}; +use wfl::typechecker::TypeChecker; + +fn parse(source: &str) -> Program { + Parser::new(&lex_wfl_with_positions(source)) + .parse() + .expect("parse") +} + +fn typecheck(program: &Program) -> Result<(), String> { + TypeChecker::new() + .check_types(program) + .map_err(|errors| format!("{errors:?}")) +} + +fn bool_literal(value: bool) -> Expression { + Expression::Literal(Literal::Boolean(value), 2, 1) +} + +fn text_literal(value: &str) -> Expression { + Expression::Literal(Literal::String(Arc::from(value)), 2, 1) +} + +fn stream_binding() -> Statement { + Statement::StartStreamingResponseStatement { + request: text_literal("request"), + status: Some(Expression::Literal(Literal::Integer(200), 2, 1)), + content_type: None, + headers: None, + variable_name: "out".to_string(), + line: 2, + column: 1, + } +} + +fn invalid_stream_lead_with_valid_file_fallback() -> Statement { + Statement::StreamWriteStatement { + value: Expression::BinaryOperation { + left: Box::new(Expression::Literal(Literal::Integer(10), 2, 1)), + operator: Operator::Minus, + right: Box::new(text_literal("not a number")), + line: 2, + column: 1, + }, + target: Expression::Variable("out".to_string(), 2, 1), + is_line: true, + fallback_content: Some(Box::new(text_literal("valid file text"))), + line: 2, + column: 1, + } +} + +fn open_out_file() -> Statement { + Statement::OpenFileStatement { + path: text_literal("unused.txt"), + variable_name: "out".to_string(), + mode: FileOpenMode::Write, + line: 1, + column: 1, + } +} + +fn ambiguous_file_write_program(control: Statement) -> Program { + let mut program = parse( + "open file at \"unused.txt\" for writing as out\n\ + store value as 10\n\ + store line value as \"text\"\n\ + store n as 1\n\ + write line value minus n to out\n", + ); + program.statements.insert(1, control); + program +} + +#[test] +fn maybe_skipped_stream_bindings_require_both_write_readings_to_be_valid() { + let controls = [ + ( + "if", + Statement::IfStatement { + condition: bool_literal(false), + then_block: vec![stream_binding()], + else_block: None, + line: 2, + column: 1, + }, + ), + ( + "single-line if", + Statement::SingleLineIf { + condition: bool_literal(false), + then_stmt: Box::new(stream_binding()), + else_stmt: None, + line: 2, + column: 1, + }, + ), + ( + "while", + Statement::WhileLoop { + condition: bool_literal(false), + body: vec![stream_binding()], + line: 2, + column: 1, + }, + ), + ]; + + for (label, control) in controls { + let errors = typecheck(&ambiguous_file_write_program(control)) + .expect_err("File or ResponseStream must conservatively validate both write readings"); + assert!( + errors.contains("Cannot perform Minus operation"), + "{label} must retain the possible outer File path and reject the \ + Text/Number classic fallback; got: {errors}" + ); + } +} + +#[test] +fn while_loop_rechecks_stream_lead_after_tail_response_stream_rebind() { + let program = Program { + statements: vec![ + open_out_file(), + Statement::WhileLoop { + condition: bool_literal(true), + body: vec![ + invalid_stream_lead_with_valid_file_fallback(), + stream_binding(), + ], + line: 2, + column: 1, + }, + ], + }; + + let errors = typecheck(&program) + .expect_err("the loop backedge must recheck the body under ResponseStream or File"); + assert!( + errors.contains("Cannot perform Minus operation"), + "the first iteration has a valid File fallback, but a later iteration must reject \ + the Number/Text stream lead after the tail ResponseStream rebind; got: {errors}" + ); +} + +#[test] +fn repeat_while_loop_rechecks_stream_lead_after_tail_response_stream_rebind() { + let program = Program { + statements: vec![ + open_out_file(), + Statement::RepeatWhileLoop { + condition: bool_literal(true), + body: vec![ + invalid_stream_lead_with_valid_file_fallback(), + stream_binding(), + ], + line: 2, + column: 1, + }, + ], + }; + + let errors = typecheck(&program) + .expect_err("the repeat-loop backedge must recheck the body under ResponseStream or File"); + assert!( + errors.contains("Cannot perform Minus operation"), + "the first iteration has a valid File fallback, but a later iteration must reject \ + the Number/Text stream lead after the tail ResponseStream rebind; got: {errors}" + ); +} + +#[test] +fn two_concrete_branch_types_join_instead_of_taking_the_last_checked_branch() { + let program = Program { + statements: vec![ + Statement::IfStatement { + condition: bool_literal(true), + then_block: vec![stream_binding()], + else_block: Some(vec![Statement::OpenFileStatement { + path: text_literal("unused.txt"), + variable_name: "out".to_string(), + mode: FileOpenMode::Write, + line: 3, + column: 1, + }]), + line: 1, + column: 1, + }, + Statement::FlushStreamStatement { + target: Expression::Variable("out".to_string(), 5, 1), + legacy_binding: None, + action_fallback: None, + line: 5, + column: 1, + }, + ], + }; + + assert!( + typecheck(&program).is_ok(), + "ResponseStream or File must join to a gradual type instead of treating \ + the last checked File branch as definite; errors: {:?}", + typecheck(&program).err() + ); +} diff --git a/tests/typechecker_response_stream_scope_test.rs b/tests/typechecker_response_stream_scope_test.rs new file mode 100644 index 00000000..62a4587c --- /dev/null +++ b/tests/typechecker_response_stream_scope_test.rs @@ -0,0 +1,202 @@ +//! Regression coverage for response-stream symbols reconstructed in type-checker +//! child scopes after analyzer body scopes have been discarded. +//! +//! These tests cover type visibility and restoration during checking. Runtime +//! shadows response-stream names inside the corresponding child environments. + +use std::sync::Arc; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Literal, Operator, Program, Statement, WsHandlerEvent}; +use wfl::typechecker::TypeChecker; + +fn typecheck(code: &str) -> Result<(), String> { + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + TypeChecker::new() + .check_types(&program) + .map_err(|errors| format!("{errors:?}")) +} + +fn typecheck_program(program: &Program) -> Result<(), String> { + TypeChecker::new() + .check_types(program) + .map_err(|errors| format!("{errors:?}")) +} + +fn text_literal(value: &str) -> Expression { + Expression::Literal(Literal::String(Arc::from(value)), 1, 1) +} + +fn stream_binding() -> Statement { + Statement::StartStreamingResponseStatement { + request: text_literal("request"), + status: Some(Expression::Literal(Literal::Integer(200), 2, 1)), + content_type: None, + headers: None, + variable_name: "out".to_string(), + line: 2, + column: 1, + } +} + +fn outer_number_binding() -> Statement { + Statement::VariableDeclaration { + name: "out".to_string(), + value: Expression::Literal(Literal::Integer(10), 1, 1), + is_constant: false, + line: 1, + column: 1, + } +} + +fn subtract_from_outer_out() -> Statement { + Statement::DisplayStatement { + value: Expression::BinaryOperation { + left: Box::new(Expression::Variable("out".to_string(), 4, 1)), + operator: Operator::Minus, + right: Box::new(Expression::Literal(Literal::Integer(1), 4, 1)), + line: 4, + column: 1, + }, + line: 4, + column: 1, + } +} + +#[test] +fn response_stream_bindings_do_not_escape_typechecker_child_scopes() { + let scoped_blocks = [ + "repeat while false:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + end repeat\n", + "try:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + when error:\n\ + \x20\x20\x20\x20display \"ignored\"\n\ + end try\n", + "count from 1 to 1:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + end count\n", + ]; + + for scoped_block in scoped_blocks { + let source = format!( + "open file at \"unused.txt\" for writing as out\n\ + {scoped_block}\ + store value as \"wrong stream type\"\n\ + store line value as 10\n\ + store n as 1\n\ + write line value minus n to out\n" + ); + assert!( + typecheck(&source).is_ok(), + "a response stream reconstructed in a type-checker child scope must not replace \ + the outer File type; source:\n{source}\nerrors: {:?}", + typecheck(&source).err() + ); + } +} + +#[test] +fn response_stream_bindings_are_local_while_outer_text_remains_visible_afterward() { + let scoped_blocks = [ + "repeat while false:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + \x20\x20\x20\x20flush out\n\ + end repeat\n", + "try:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + \x20\x20\x20\x20flush out\n\ + when error:\n\ + \x20\x20\x20\x20display \"ignored\"\n\ + end try\n", + "count from 1 to 1:\n\ + \x20\x20\x20\x20start streaming response to \"request\" with status 200 as out\n\ + \x20\x20\x20\x20flush out\n\ + end count\n", + ]; + + for scoped_block in scoped_blocks { + let source = format!( + "store out as \"outer text\"\n\ + {scoped_block}\ + store invalid as out minus 1\n" + ); + let errors = typecheck(&source) + .expect_err("the outer Text binding must remain Text after the child scope"); + assert!( + errors.contains("Cannot perform Minus operation"), + "expected the restored outer Text subtraction error; source:\n{source}\nerrors: {errors}" + ); + assert!( + !errors.contains("`flush` requires a response-stream handle"), + "the local binding must be visible as ResponseStream while checking its child scope; \ + source:\n{source}\nerrors: {errors}" + ); + } +} + +#[test] +fn default_count_binding_does_not_retype_an_outer_count_variable() { + let source = "store count as \"outside\"\n\ + count from 1 to 1:\n\ + \x20\x20\x20\x20display count\n\ + end count\n\ + store invalid as count minus 1\n"; + let errors = + typecheck(source).expect_err("the outer Text `count minus 1` must remain a type error"); + assert!( + errors.contains("Cannot perform Minus operation"), + "expected the outer Text/Number subtraction error, got: {errors}" + ); +} + +#[test] +fn event_handler_body_types_do_not_leak_after_registration() { + let program = Program { + statements: vec![ + outer_number_binding(), + Statement::EventHandler { + event_source: text_literal("source"), + event_name: "changed".to_string(), + handler_body: vec![stream_binding()], + line: 2, + column: 1, + }, + subtract_from_outer_out(), + ], + }; + + assert!( + typecheck_program(&program).is_ok(), + "a deferred event body runs in a fresh runtime child and must not retype outer `out`; \ + errors: {:?}", + typecheck_program(&program).err() + ); +} + +#[test] +fn websocket_handler_body_types_do_not_leak_after_registration() { + let program = Program { + statements: vec![ + outer_number_binding(), + Statement::WebSocketHandlerStatement { + event: WsHandlerEvent::Connect, + server: text_literal("WebSocketServer::127.0.0.1:0"), + binding: "conn".to_string(), + body: vec![stream_binding()], + line: 2, + column: 1, + }, + subtract_from_outer_out(), + ], + }; + + assert!( + typecheck_program(&program).is_ok(), + "a deferred WebSocket body runs in a fresh runtime child and must not retype outer \ + `out`; errors: {:?}", + typecheck_program(&program).err() + ); +} diff --git a/tests/typechecker_try_finally_join_test.rs b/tests/typechecker_try_finally_join_test.rs new file mode 100644 index 00000000..fee6fefe --- /dev/null +++ b/tests/typechecker_try_finally_join_test.rs @@ -0,0 +1,255 @@ +//! Regression coverage for type-state joins from `try` success/error endpoints +//! into `finally`, while keeping `when` error aliases clause-local. + +use std::sync::Arc; +use wfl::analyzer::{Analyzer, Symbol, SymbolKind}; +use wfl::parser::ast::{ + ErrorType, Expression, FileOpenMode, Literal, Operator, Program, Statement, Type, WhenClause, +}; +use wfl::typechecker::TypeChecker; + +fn text_literal(value: &str) -> Expression { + Expression::Literal(Literal::String(Arc::from(value)), 1, 1) +} + +fn stream_binding() -> Statement { + Statement::StartStreamingResponseStatement { + request: text_literal("request"), + status: Some(Expression::Literal(Literal::Integer(200), 3, 1)), + content_type: None, + headers: None, + variable_name: "out".to_string(), + line: 3, + column: 1, + } +} + +fn display_text(value: &str, line: usize) -> Statement { + Statement::DisplayStatement { + value: text_literal(value), + line, + column: 1, + } +} + +fn flush_out() -> Statement { + Statement::FlushStreamStatement { + target: Expression::Variable("out".to_string(), 5, 1), + legacy_binding: None, + action_fallback: None, + line: 5, + column: 1, + } +} + +fn subtract_one(name: &str, line: usize) -> Statement { + Statement::DisplayStatement { + value: Expression::BinaryOperation { + left: Box::new(Expression::Variable(name.to_string(), line, 1)), + operator: Operator::Minus, + right: Box::new(Expression::Literal(Literal::Integer(1), line, 1)), + line, + column: 1, + }, + line, + column: 1, + } +} + +fn store_text(name: &str, value: &str, line: usize) -> Statement { + Statement::VariableDeclaration { + name: name.to_string(), + value: text_literal(value), + is_constant: false, + line, + column: 1, + } +} + +fn display_variable(name: &str, line: usize) -> Statement { + Statement::DisplayStatement { + value: Expression::Variable(name.to_string(), line, 1), + line, + column: 1, + } +} + +#[test] +fn handler_response_stream_state_is_joined_before_finally() { + let program = Program { + statements: vec![ + Statement::OpenFileStatement { + path: text_literal("unused.txt"), + variable_name: "out".to_string(), + mode: FileOpenMode::Write, + line: 1, + column: 1, + }, + Statement::TryStatement { + body: vec![display_text("success", 2)], + when_clauses: vec![WhenClause { + error_type: ErrorType::General, + error_name: "caught".to_string(), + body: vec![stream_binding()], + }], + otherwise_block: None, + finally_block: Some(vec![flush_out()]), + line: 2, + column: 1, + }, + ], + }; + + assert!( + TypeChecker::new().check_types(&program).is_ok(), + "finally must see the gradual join of the successful File path and the handler's \ + ResponseStream path; errors: {:?}", + TypeChecker::new().check_types(&program).err() + ); +} + +#[test] +fn handler_error_aliases_remain_clause_local_before_finally() { + let mut analyzer = Analyzer::new(); + for name in ["caught", "error_message"] { + analyzer + .define_symbol(Symbol { + name: name.to_string(), + kind: SymbolKind::Variable { mutable: true }, + symbol_type: Some(Type::Number), + line: 1, + column: 1, + }) + .expect("define outer Number binding"); + } + + let program = Program { + statements: vec![Statement::TryStatement { + body: vec![display_text("success", 2)], + when_clauses: vec![WhenClause { + error_type: ErrorType::General, + error_name: "caught".to_string(), + body: vec![ + Statement::DisplayStatement { + value: Expression::Variable("caught".to_string(), 3, 1), + line: 3, + column: 1, + }, + Statement::DisplayStatement { + value: Expression::Variable("error_message".to_string(), 3, 1), + line: 3, + column: 1, + }, + ], + }], + otherwise_block: None, + finally_block: Some(vec![ + subtract_one("caught", 5), + subtract_one("error_message", 6), + ]), + line: 2, + column: 1, + }], + }; + + let result = TypeChecker::with_analyzer(analyzer).check_types(&program); + assert!( + result.is_ok(), + "finally must resolve the outer Number bindings, not clause-local Text aliases; \ + got: {:?}", + result.err() + ); +} + +#[test] +fn handler_created_binding_is_semantically_visible_in_finally() { + let program = Program { + statements: vec![Statement::TryStatement { + body: vec![display_text("success", 2)], + when_clauses: vec![WhenClause { + error_type: ErrorType::FileNotFound, + error_name: "caught".to_string(), + body: vec![store_text("cleanup_message", "handled", 3)], + }], + otherwise_block: None, + finally_block: Some(vec![display_variable("cleanup_message", 5)]), + line: 1, + column: 1, + }], + }; + + assert!( + TypeChecker::new().check_types(&program).is_ok(), + "the analyzer and checker must preserve an ordinary handler binding in the shared \ + runtime try scope until finally; errors: {:?}", + TypeChecker::new().check_types(&program).err() + ); +} + +#[test] +fn otherwise_created_binding_is_semantically_visible_in_finally() { + let program = Program { + statements: vec![Statement::TryStatement { + body: vec![display_text("success", 2)], + when_clauses: vec![], + otherwise_block: Some(vec![store_text("cleanup_message", "otherwise", 3)]), + finally_block: Some(vec![display_variable("cleanup_message", 5)]), + line: 1, + column: 1, + }], + }; + + assert!( + TypeChecker::new().check_types(&program).is_ok(), + "the analyzer and checker must preserve an ordinary otherwise binding in the shared \ + runtime try scope until finally; errors: {:?}", + TypeChecker::new().check_types(&program).err() + ); +} + +#[test] +fn full_pipeline_error_alias_is_clause_local() { + let program = Program { + statements: vec![ + Statement::VariableDeclaration { + name: "caught".to_string(), + value: Expression::Literal(Literal::Integer(10), 1, 1), + is_constant: false, + line: 1, + column: 1, + }, + Statement::VariableDeclaration { + name: "error_message".to_string(), + value: Expression::Literal(Literal::Integer(20), 1, 1), + is_constant: false, + line: 1, + column: 1, + }, + Statement::TryStatement { + body: vec![display_text("success", 2)], + when_clauses: vec![WhenClause { + error_type: ErrorType::General, + error_name: "caught".to_string(), + body: vec![ + display_variable("caught", 3), + display_variable("error_message", 3), + ], + }], + otherwise_block: None, + finally_block: Some(vec![ + subtract_one("caught", 5), + subtract_one("error_message", 6), + ]), + line: 2, + column: 1, + }, + ], + }; + + assert!( + TypeChecker::new().check_types(&program).is_ok(), + "the implicit Text error aliases must shadow only inside their clause, and finally \ + must resolve the outer Numbers; errors: {:?}", + TypeChecker::new().check_types(&program).err() + ); +} diff --git a/tests/wait_line_pre_response_disconnect_test.rs b/tests/wait_line_pre_response_disconnect_test.rs new file mode 100644 index 00000000..9a68dbdd --- /dev/null +++ b/tests/wait_line_pre_response_disconnect_test.rs @@ -0,0 +1,156 @@ +//! Real-socket regression (maintainer re-review, P1): a downstream disconnect must +//! cancel a proxy handler blocked in `wait for next LINE` BEFORE it has called +//! `start streaming response` — exactly like `wait for next chunk`. +//! +//! The chunk read raced the combined pre-response/downstream disconnect signal; the +//! line read only watched the (not-yet-existing) downstream stream, so a client that +//! went away while the handler was blocked reading an upstream line was ignored until +//! the read timeout, occupying the upstream socket and the handler. Topology: an +//! upstream that sends a head then withholds all body lines <- WFL concurrent proxy +//! -> a client that connects and disconnects during the blocked line read. + +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use wfl::Interpreter; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; + +mod common; + +/// Upstream: send a valid chunked head, then WITHHOLD all body bytes (no line ever +/// arrives). Signal when the proxy drops the connection (peer close => read 0/Err). +async fn spawn_head_then_no_lines_upstream() -> (u16, tokio::sync::oneshot::Receiver<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock upstream"); + let port = listener.local_addr().unwrap().port(); + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + if let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 1024]; + let _ = sock.read(&mut buf).await; // request head + let head = "HTTP/1.1 200 OK\r\n\ + Content-Type: text/plain\r\n\ + Transfer-Encoding: chunked\r\n\r\n"; + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.flush().await; + // Withhold all body lines; wait for the proxy to drop the connection. + loop { + match sock.read(&mut buf).await { + Ok(0) | Err(_) => { + let _ = tx.send(()); + return; + } + Ok(_) => {} + } + } + } + }); + (port, rx) +} + +fn start_proxy_server(code: String) -> std::thread::JoinHandle<()> { + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + rt.block_on(async { + let tokens = lex_wfl_with_positions(&code); + let ast = Parser::new(&tokens).parse().expect("parse"); + let mut interp = Interpreter::new(); + if let Err(errors) = interp.interpret(&ast).await { + panic!("proxy interpreter failed: {errors:?}"); + } + }); + }) +} + +async fn wait_for_server(port: u16) { + let addr = format!("127.0.0.1:{port}"); + for _ in 0..300 { + if tokio::net::TcpStream::connect(&addr).await.is_ok() { + return; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + panic!("proxy server on {addr} did not become ready"); +} + +#[tokio::test] +async fn test_disconnect_cancels_blocked_pre_response_line_read() { + let (upstream_port, mut upstream_closed) = spawn_head_then_no_lines_upstream().await; + let proxy_port = common::free_tcp_port(); + + // The handler opens the upstream and blocks in `wait for next line` BEFORE + // `start streaming response` — so only the pending-request disconnect signal can + // cancel it. The client disconnects during that blocked read. + let code = format!( + r#" + listen on port {proxy_port} as srv + main loop concurrently: + wait for request comes in on srv as req with timeout 30000 + store p as req["path"] + check if p is equal to "/ping": + respond to req with "pong" + otherwise: + check if p is equal to "/shutdown": + respond to req with "bye" + close server srv + break + otherwise: + open url at "http://127.0.0.1:{upstream_port}/" and stream response as up + wait for next line from up as ln + start streaming response to req with status 200 and content type "text/plain" as down + close down + end check + end check + end loop + "# + ); + let server = start_proxy_server(code); + wait_for_server(proxy_port).await; + + { + let mut sock = tokio::net::TcpStream::connect(("127.0.0.1", proxy_port)) + .await + .expect("connect to proxy"); + sock.write_all(b"GET /proxy HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n") + .await + .expect("send request"); + sock.flush().await.ok(); + // Let the handler dequeue, open the upstream, and block in the line read, + // then drop the socket to disconnect. + tokio::time::sleep(Duration::from_millis(400)).await; + // `sock` drops here -> client disconnects. + } + + // The upstream must observe its connection close promptly — the blocked line read + // was cancelled by the disconnect, not left to wait out the idle timeout. + tokio::time::timeout(Duration::from_secs(4), &mut upstream_closed) + .await + .expect( + "the blocked pre-response line read was not cancelled after the client disconnected", + ) + .expect("upstream close sender dropped"); + + // The concurrent loop stayed alive. + let ping = tokio::time::timeout( + Duration::from_secs(5), + reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/ping")) + .send(), + ) + .await + .expect("/ping timed out") + .expect("/ping failed"); + assert_eq!(ping.status().as_u16(), 200); + assert_eq!(ping.text().await.unwrap(), "pong"); + + let _ = reqwest::Client::new() + .get(format!("http://127.0.0.1:{proxy_port}/shutdown")) + .send() + .await; + match tokio::task::spawn_blocking(move || server.join()).await { + Ok(Ok(())) => {} + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(e) => panic!("server join task failed: {e}"), + } +} diff --git a/tests/write_line_backcompat_test.rs b/tests/write_line_backcompat_test.rs new file mode 100644 index 00000000..35bc2b5d --- /dev/null +++ b/tests/write_line_backcompat_test.rs @@ -0,0 +1,439 @@ +// Backward-compatibility for `write line to `. +// +// WFL allows space-separated identifiers, so `write line payload to out` could +// mean either the classic file write of a variable literally named "line +// payload", or the new streaming form (`write line to `). The +// merged form must not silently break the pre-existing file write: the runtime +// picks the interpretation from the target type, and static analysis must not +// reject the file-write reading (nor falsely warn the variable is unused). + +use wfl::Interpreter; +use wfl::analyzer::Analyzer; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Statement}; + +fn parse(code: &str) -> Vec { + let tokens = lex_wfl_with_positions(code); + let mut parser = Parser::new(&tokens); + parser + .parse() + .unwrap_or_else(|e| panic!("parse error: {e:?}")) + .statements +} + +#[test] +fn test_write_line_multiword_variable_parses_with_fallback() { + // The ambiguous merged form carries a classic-file-write fallback so the + // interpreter can disambiguate on the target type at runtime. + let stmt = &parse(r#"write line payload to out"#)[0]; + match stmt { + Statement::StreamWriteStatement { + fallback_content, + is_line, + .. + } => { + assert!(*is_line); + assert!( + fallback_content.is_some(), + "merged `write line ` must keep a file-write fallback" + ); + } + other => panic!("expected StreamWriteStatement, got {other:?}"), + } + + // The unambiguous literal form (never a valid classic file write) has none. + let stmt = &parse(r#"write line "x" to out"#)[0]; + match stmt { + Statement::StreamWriteStatement { + fallback_content, .. + } => assert!( + fallback_content.is_none(), + "literal-valued stream write needs no fallback" + ), + other => panic!("expected StreamWriteStatement, got {other:?}"), + } + + // `write chunk` has the same ambiguity and fallback contract as `write line`. + let stmt = &parse(r#"write chunk payload to out"#)[0]; + match stmt { + Statement::StreamWriteStatement { + fallback_content, + is_line, + .. + } => { + assert!(!*is_line, "write chunk must not be a line write"); + assert!( + fallback_content.is_some(), + "merged `write chunk ` must keep a file-write fallback" + ); + } + other => panic!("expected StreamWriteStatement, got {other:?}"), + } + let stmt = &parse(r#"write chunk "x" to out"#)[0]; + match stmt { + Statement::StreamWriteStatement { + fallback_content, .. + } => assert!( + fallback_content.is_none(), + "literal-valued chunk write needs no fallback" + ), + other => panic!("expected StreamWriteStatement, got {other:?}"), + } +} + +#[test] +fn test_write_line_with_continuation_parses_for_both_readings() { + // `write line payload with "!" to out`: the value must absorb the `with` + // continuation for the stream reading, and the classic file-write fallback + // must mirror it (leading variable `line payload`, same continuation) — not + // truncate at the bare variable and fail at `with`. + let stmt = &parse(r#"write line payload with "!" to out"#)[0]; + match stmt { + Statement::StreamWriteStatement { + value, + fallback_content, + .. + } => { + // Stream reading: Concatenation(Variable("payload"), "!"). + match value { + Expression::Concatenation { left, .. } => match &**left { + Expression::Variable(name, ..) => assert_eq!(name, "payload"), + other => panic!("stream value left should be Variable(payload), got {other:?}"), + }, + other => panic!("stream value should be a Concatenation, got {other:?}"), + } + // Classic file-write fallback: Concatenation(Variable("line payload"), "!"). + let fb = fallback_content + .as_ref() + .expect("merged `write line with ...` keeps a file-write fallback"); + match &**fb { + Expression::Concatenation { left, .. } => match &**left { + Expression::Variable(name, ..) => assert_eq!(name, "line payload"), + other => { + panic!("fallback left should be Variable(line payload), got {other:?}") + } + }, + other => panic!("fallback should mirror the continuation, got {other:?}"), + } + } + other => panic!("expected StreamWriteStatement, got {other:?}"), + } +} + +#[test] +fn test_write_line_variable_with_continuation_to_file_preserves_concatenation() { + // A pre-existing classic file write with a `with` continuation on a variable + // literally named `line note`. Before continuation parsing this failed to + // parse (the interception expected `to` right after the variable); it must + // now write the concatenated value to the file. + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("wfl_write_line_continuation.txt"); + let path_str = path.to_string_lossy().replace('\\', "/"); + + let code = format!( + r#"store line note as "kept" +write line note with "!" to "{path_str}""# + ); + + let program = { + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + parser.parse().expect("parse") + }; + + let mut analyzer = Analyzer::new(); + analyzer + .analyze(&program) + .expect("analysis must accept `write line with ... to `"); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let mut interp = Interpreter::new(); + interp.interpret(&program).await.expect("interpret"); + }); + + let contents = std::fs::read_to_string(&path).expect("output file should exist"); + assert_eq!( + contents, "kept!", + "the concatenated value (variable `line note` with \"!\") must reach the file" + ); +} + +#[test] +fn test_write_line_builtin_named_variable_with_continuation_preserves_concatenation() { + // Regression (maintainer review): the stream reading of `length with "!"` + // desugars to an ActionCall because `length` is a builtin. The classic + // file-write reading must still be the variable `line length` concatenated + // with "!" — parsed INDEPENDENTLY (cursor rewind), not derived from the + // stream AST, which would drop the `with "!"` and write only "kept". + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("wfl_write_line_builtin_named.txt"); + let path_str = path.to_string_lossy().replace('\\', "/"); + + let code = format!( + r#"store line length as "kept" +write line length with "!" to "{path_str}""# + ); + + let program = { + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + parser.parse().expect("parse") + }; + + let mut analyzer = Analyzer::new(); + analyzer + .analyze(&program) + .expect("analysis must accept the builtin-named continuation file write"); + + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let mut interp = Interpreter::new(); + interp.interpret(&program).await.expect("interpret"); + }); + + let contents = std::fs::read_to_string(&path).expect("output file should exist"); + assert_eq!( + contents, "kept!", + "the classic file write must keep the `with \"!\"` continuation even though `length` is a builtin" + ); +} + +#[test] +fn test_write_multiword_line_variable_to_file_still_works() { + // A pre-existing program: a variable literally named `line note` written to a + // file path. Must analyze cleanly (no undefined-variable error, no spurious + // unused warning) and, at runtime, write the VARIABLE'S value to the file — + // not stream-write the token "note". + // Unique temp dir per invocation so parallel/sharded runs cannot collide or + // delete each other's output; `TempDir` cleans up on drop (even on panic). + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("wfl_write_line_backcompat.txt"); + let path_str = path.to_string_lossy().replace('\\', "/"); + + let code = format!( + r#"store line note as "kept across versions" +write line note to "{path_str}""# + ); + + let program = { + let tokens = lex_wfl_with_positions(&code); + let mut parser = Parser::new(&tokens); + parser.parse().expect("parse") + }; + + // Static analysis must accept the file-write reading. + let mut analyzer = Analyzer::new(); + analyzer + .analyze(&program) + .expect("semantic analysis must accept `write line to `"); + + // Runtime writes the variable's value to the file. + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let mut interp = Interpreter::new(); + interp.interpret(&program).await.expect("interpret"); + }); + + let contents = std::fs::read_to_string(&path).expect("output file should exist"); + assert_eq!( + contents, "kept across versions", + "the variable `line note` must be written to the file, not the token `note`" + ); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn test_ambiguous_write_line_flags_undefined_in_continuation() { + // The continuation (everything right of the ambiguous lead) is shared by both + // readings, so a genuinely undefined variable there must still be caught even + // though the leading operand itself is ambiguous. + let code = "listen on port 8080 as srv\nstore payload as \"x\"\nwrite line payload with missing_suffix to srv"; + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut analyzer = Analyzer::new(); + let errors = analyzer + .analyze(&program) + .expect_err("`missing_suffix` in the continuation is undefined"); + assert!( + errors + .iter() + .any(|e| e.message.contains("missing_suffix") && e.message.contains("not defined")), + "expected an undefined-variable error naming `missing_suffix`, got: {errors:?}" + ); +} + +#[test] +fn test_ambiguous_write_line_accepts_valid_classic_with_continuation() { + // A valid pre-existing program: a variable literally named `line path`, + // written with a continuation to a file. The split stream name `path` is + // undefined, but the classic file-write reading resolves — analysis must NOT + // reject it (no false positive from the ambiguous split). + let code = "store line path as \"/tmp/x\"\nstore suffix as \"!\"\nwrite line path with suffix to \"/tmp/out\""; + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut analyzer = Analyzer::new(); + assert!( + analyzer.analyze(&program).is_ok(), + "a valid classic `write line with ... to ` must not be rejected: {:?}", + analyzer.get_errors() + ); +} + +#[test] +fn test_ambiguous_write_line_accepts_desugared_classic_writes() { + // Regression (maintainer review): the analyzer must NOT reject a valid classic + // file write whose value desugars to a call/comparison/pattern, where the + // ambiguous lead is not the plain leftmost leaf. Only the multiword `line …` + // variable is defined; the split stream name is not — and yet each of these + // is a valid pre-existing program that must analyze cleanly. + let cases = [ + "store line path as \"/api\"\nwrite line path starts with \"/\" to \"/tmp/out\"", + "store line score as 3\nwrite line score is between 1 and 5 to \"/tmp/out\"", + "store line subject as \"abc\"\nwrite line subject matches pattern \"a\" to \"/tmp/out\"", + // `write chunk` shares the same ambiguity. + "store chunk path as \"/api\"\nwrite chunk path starts with \"/\" to \"/tmp/out\"", + ]; + for code in cases { + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut analyzer = Analyzer::new(); + assert!( + analyzer.analyze(&program).is_ok(), + "a valid classic desugared write must not be rejected.\n code: {code}\n errors: {:?}", + analyzer.get_errors() + ); + } +} + +#[test] +fn test_ambiguous_write_line_drops_span_mismatched_fallback() { + // Regression (maintainer review): the classic file-write fallback is only a + // valid alternate reading when it consumes the SAME continuation span as the + // stream reading. `write line min with a: 1 and b: 2 to `: the stream + // reading is the builtin call `min` with named args `a`/`b` (consuming through + // `b: 2`), but the classic reading of the multiword variable `line min` can + // only parse `line min with a` before the `:` — a shorter, partial span. + // Keeping that partial parse as the fallback corrupts a file write, so it must + // be dropped (fallback = None) rather than retained just because it parsed. + let stmt = &parse("write line min with a: 1 and b: 2 to f")[0]; + match stmt { + Statement::StreamWriteStatement { + value, + fallback_content, + .. + } => { + assert!( + matches!(value, Expression::ActionCall { .. }), + "the stream reading should consume the whole named-argument call, got {value:?}" + ); + assert!( + fallback_content.is_none(), + "a partial (span-mismatched) classic fallback must be dropped, got {fallback_content:?}" + ); + } + other => panic!("expected StreamWriteStatement, got {other:?}"), + } +} + +#[test] +fn test_span_mismatched_write_line_to_file_does_not_corrupt() { + // The runtime counterpart: with a span-mismatched fallback dropped, writing + // the ambiguous `min(...)` form to a FILE target is a clean error instead of + // silently writing the corrupt partial concatenation. `line min` and `a` are + // defined so the OLD (buggy) fallback would have evaluated and written + // "CORRUPT..." to the file; the fix must prevent that. + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("wfl_write_line_span_mismatch.txt"); + let path_str = path.to_string_lossy().replace('\\', "/"); + + let code = format!( + "store line min as \"CORRUPT\"\n\ + store a as \"SUFFIX\"\n\ + write line min with a: 1 and b: 2 to \"{path_str}\"" + ); + let tokens = lex_wfl_with_positions(&code); + let program = Parser::new(&tokens).parse().expect("parse"); + + let rt = tokio::runtime::Runtime::new().unwrap(); + let result = rt.block_on(async { + let mut interp = Interpreter::new(); + interp.interpret(&program).await + }); + + let wrote_corrupt = std::fs::read_to_string(&path) + .map(|c| c.contains("CORRUPT")) + .unwrap_or(false); + assert!( + !wrote_corrupt, + "the span-mismatched fallback corrupted the file write: {:?}", + std::fs::read_to_string(&path) + ); + assert!( + result.is_err(), + "writing the ambiguous `min(...)` form to a file must be a clean error, not a silent corrupt write" + ); +} + +#[test] +fn test_ambiguous_write_line_flags_undefined_in_desugared_continuation() { + // The continuation of a DESUGARED (operator) ambiguous value is shared by both + // readings, so an undefined variable there must still be flagged even though + // the lead itself is target-dependent. `value` is defined; `missing_suffix` is + // not — and it lives in the `plus` continuation, not at the ambiguous lead. + let code = "listen on port 8080 as srv\n\ + store value as 1\n\ + write line value plus missing_suffix to srv"; + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut analyzer = Analyzer::new(); + let errors = analyzer + .analyze(&program) + .expect_err("`missing_suffix` in the operator continuation is undefined"); + assert!( + errors + .iter() + .any(|e| e.message.contains("missing_suffix") && e.message.contains("not defined")), + "expected an undefined-variable error naming `missing_suffix`, got: {errors:?}" + ); +} + +#[test] +fn test_ambiguous_write_line_operator_continuation_all_defined_ok() { + // A valid classic file write whose value desugars to an operator expression + // with a fully-defined continuation must NOT be rejected: the split stream + // lead `value` is undefined, but the classic reading (`line value`) and the + // shared continuation (`addend`) both resolve. + let code = "store line value as 3\n\ + store addend as 4\n\ + write line value plus addend to \"/tmp/out\""; + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + let mut analyzer = Analyzer::new(); + assert!( + analyzer.analyze(&program).is_ok(), + "a valid operator-continuation classic write must not be rejected: {:?}", + analyzer.get_errors() + ); +} + +#[test] +fn test_ambiguous_write_line_still_flags_when_neither_candidate_defined() { + // The ambiguous form defers definedness to runtime, but a genuine typo where + // NEITHER reading resolves (`payload` as a stream value, nor `line payload` + // as a file-write variable) must still be caught by static analysis. + let code = "listen on port 8080 as srv\nwrite line payload to srv"; + let tokens = lex_wfl_with_positions(code); + let program = Parser::new(&tokens).parse().expect("parse"); + + let mut analyzer = Analyzer::new(); + let result = analyzer.analyze(&program); + let errors = result.expect_err("neither `payload` nor `line payload` is defined"); + assert!( + errors + .iter() + .any(|e| e.message.contains("line payload") && e.message.contains("not defined")), + "expected an undefined-variable error naming `line payload`, got: {errors:?}" + ); +} diff --git a/tests/write_web_postfix_test.rs b/tests/write_web_postfix_test.rs new file mode 100644 index 00000000..6b776a88 --- /dev/null +++ b/tests/write_web_postfix_test.rs @@ -0,0 +1,1119 @@ +//! Regression (maintainer re-review, P1): postfix accessors on `write line|chunk` +//! operands and on merged `content type` / `headers` clause operands must compose +//! onto the operand instead of dangling after the statement. +//! +//! The lexer merges the command word with the following identifier and leaves any +//! `[...]` index / `.field` property accessors as separate tokens, so +//! `write line chunks[0] to out`, `write line upstream.status to out`, +//! `headers upstream.headers`, and `content type upstream.headers["content-type"]` +//! previously left those accessors to dangle (a parse error or a wrong split). + +use std::fs; +use std::process::Command; +use tempfile::TempDir; +use wfl::lexer::lex_wfl_with_positions; +use wfl::parser::Parser; +use wfl::parser::ast::{Expression, Statement}; + +fn parse(src: &str) -> wfl::parser::ast::Program { + let tokens = lex_wfl_with_positions(src); + Parser::new(&tokens).parse().expect("parse should succeed") +} + +fn stream_write_value(stmt: &Statement) -> &Expression { + match stmt { + Statement::StreamWriteStatement { value, .. } => value, + other => panic!("expected a StreamWriteStatement, got {other:#?}"), + } +} + +fn stream_write_fallback(stmt: &Statement) -> &Expression { + match stmt { + Statement::StreamWriteStatement { + fallback_content: Some(fallback), + .. + } => fallback, + other => panic!("expected an ambiguous StreamWriteStatement fallback, got {other:#?}"), + } +} + +fn expression_shape(expr: &Expression) -> &'static str { + match expr { + Expression::IndexAccess { .. } => "index", + Expression::PropertyAccess { .. } => "property", + Expression::MethodCall { .. } => "method", + Expression::FunctionCall { .. } => "of-call", + Expression::BinaryOperation { .. } => "operator", + other => panic!("unexpected operand shape in parity matrix: {other:#?}"), + } +} + +fn streaming_clause_operand<'a>(stmt: &'a Statement, clause: &str) -> &'a Expression { + match stmt { + Statement::StartStreamingResponseStatement { + content_type, + headers, + .. + } => match clause { + "content type" => content_type.as_ref().expect("content type operand"), + "headers" => headers.as_ref().expect("headers operand"), + other => panic!("unknown test clause {other}"), + }, + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } +} + +fn streaming_status_and_headers(stmt: &Statement) -> (&Expression, &Expression) { + match stmt { + Statement::StartStreamingResponseStatement { + status: Some(status), + headers: Some(headers), + .. + } => (status, headers), + other => { + panic!("expected a streaming response with status and headers operands, got {other:#?}") + } + } +} + +#[test] +fn streaming_status_clause_accepts_full_expressions_without_swallowing_headers() { + let cases = [ + ( + "start streaming response to req with status base plus 1 and headers h as out\n", + "operator", + ), + ( + "start streaming response to req with headers h and status codes at i as out\n", + "index", + ), + ( + "start streaming response to req with status reply.code and headers h as out\n", + "property", + ), + ( + "start streaming response to req with status choose of req and headers h as out\n", + "of-call", + ), + ]; + + for (source, expected_shape) in cases { + let program = parse(source); + assert_eq!( + program.statements.len(), + 1, + "`{source}` must remain one statement; got {:#?}", + program.statements + ); + let (status, headers) = streaming_status_and_headers(&program.statements[0]); + assert_eq!( + expression_shape(status), + expected_shape, + "status operand in `{source}`" + ); + assert!( + matches!(headers, Expression::Variable(name, ..) if name == "h"), + "headers must remain a separate clause in `{source}`; got {headers:#?}" + ); + } +} + +#[test] +fn streaming_status_clause_accepts_a_builtin_call_without_swallowing_headers() { + let program = parse( + "start streaming response to req with status abs of requested_status and headers h as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + + let (status, headers) = streaming_status_and_headers(&program.statements[0]); + assert!( + matches!( + status, + Expression::FunctionCall { + function, + arguments, + .. + } if matches!(function.as_ref(), Expression::Variable(name, ..) if name == "abs") + && matches!( + arguments.as_slice(), + [wfl::parser::ast::Argument { + value: Expression::Variable(name, ..), + .. + }] if name == "requested_status" + ) + ), + "the builtin status operand must remain `abs of requested_status`; got {status:#?}" + ); + assert!( + matches!(headers, Expression::Variable(name, ..) if name == "h"), + "headers must remain a separate response clause; got {headers:#?}" + ); +} + +#[test] +fn write_and_streaming_clauses_share_the_full_expression_suffix_grammar() { + let cases = [ + ("values[0]", "index"), + ("object.field", "property"), + ("object.method()", "method"), + ("values at 0", "index"), + ("values 0", "index"), + ("convert of values", "of-call"), + ("values plus 1", "operator"), + ]; + + for (operand, expected) in cases { + let write = parse(&format!("write line {operand} to out\n")); + assert_eq!( + write.statements.len(), + 1, + "write operand `{operand}` split into extra statements: {:#?}", + write.statements + ); + assert_eq!( + expression_shape(stream_write_value(&write.statements[0])), + expected, + "write operand `{operand}`" + ); + + for clause in ["content type", "headers"] { + let program = parse(&format!( + "start streaming response to req with status 200 and {clause} {operand} as out\n" + )); + assert_eq!( + program.statements.len(), + 1, + "{clause} operand `{operand}` split into extra statements: {:#?}", + program.statements + ); + assert_eq!( + expression_shape(streaming_clause_operand(&program.statements[0], clause)), + expected, + "{clause} operand `{operand}`" + ); + } + } +} + +#[test] +fn unmerged_content_type_literal_keeps_operator_continuation() { + let program = parse( + "start streaming response to req with status 200 and content type \"text/\" with subtype as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + let content_type = streaming_clause_operand(&program.statements[0], "content type"); + assert!( + matches!(content_type, Expression::Concatenation { .. }), + "unmerged literal operand must retain `with subtype`, got {content_type:#?}" + ); +} + +#[test] +fn write_line_indexed_operand_composes_into_one_index_access() { + let program = parse("write line chunks[0] to out\n"); + assert_eq!( + program.statements.len(), + 1, + "the indexed write operand must not split into extra statements; got {:#?}", + program.statements + ); + assert!( + matches!( + stream_write_value(&program.statements[0]), + Expression::IndexAccess { .. } + ), + "the write value must be an IndexAccess, got {:#?}", + stream_write_value(&program.statements[0]) + ); +} + +#[test] +fn write_line_property_operand_composes_into_one_property_access() { + let program = parse("write line upstream.status to out\n"); + assert_eq!( + program.statements.len(), + 1, + "the property write operand must not split into extra statements; got {:#?}", + program.statements + ); + assert!( + matches!( + stream_write_value(&program.statements[0]), + Expression::PropertyAccess { .. } + ), + "the write value must be a PropertyAccess, got {:#?}", + stream_write_value(&program.statements[0]) + ); +} + +#[test] +fn streaming_response_headers_clause_composes_postfix() { + // `headers upstream.headers` — the operand must bind the `.headers` access. + let program = parse( + "start streaming response to req with status 200 and headers upstream.headers as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::StartStreamingResponseStatement { headers, .. } => { + assert!( + matches!(headers, Some(Expression::PropertyAccess { .. })), + "the headers operand must be a PropertyAccess, got {headers:#?}" + ); + } + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } +} + +#[test] +fn streaming_response_content_type_clause_composes_property_then_index() { + // `content type upstream.headers["content-type"]` — property then index. + let program = parse( + "start streaming response to req with status 200 and content type upstream.headers[\"content-type\"] as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::StartStreamingResponseStatement { content_type, .. } => { + assert!( + matches!(content_type, Some(Expression::IndexAccess { .. })), + "the content type operand must be an IndexAccess (over a PropertyAccess), \ + got {content_type:#?}" + ); + } + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } +} + +#[test] +fn write_line_at_indexing_parses() { + // Classic `write line values at 0 to "/tmp/out"` must parse (issue #642). + let program = parse( + "store line values as [\"first\" and \"second\"]\n\ + write line values at 0 to \"/tmp/out\"\n", + ); + assert!( + program.statements.len() >= 2, + "expected store + write, got {:#?}", + program.statements + ); + let write = program + .statements + .iter() + .find(|s| matches!(s, Statement::StreamWriteStatement { .. })) + .expect("write statement"); + // Stream reading value is IndexAccess over Variable("values"). + assert!( + matches!(stream_write_value(write), Expression::IndexAccess { .. }), + "write value must be IndexAccess for `at` indexing, got {:#?}", + stream_write_value(write) + ); + assert!( + matches!( + stream_write_fallback(write), + Expression::IndexAccess { collection, index, .. } + if matches!(collection.as_ref(), Expression::Variable(name, ..) if name == "line values") + && matches!(index.as_ref(), Expression::Literal(wfl::parser::ast::Literal::Integer(0), ..)) + ), + "classic fallback must index the full `line values` binding, got {:#?}", + stream_write_fallback(write) + ); +} + +#[test] +fn write_line_direct_integer_indexing_parses() { + // Classic `write line values 0 to "/tmp/out"` must parse (issue #642). + let program = parse("write line values 0 to \"/tmp/out\"\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + assert!( + matches!( + stream_write_value(&program.statements[0]), + Expression::IndexAccess { .. } + ), + "write value must be IndexAccess for direct integer indexing, got {:#?}", + stream_write_value(&program.statements[0]) + ); + assert!( + matches!( + stream_write_fallback(&program.statements[0]), + Expression::IndexAccess { collection, index, .. } + if matches!(collection.as_ref(), Expression::Variable(name, ..) if name == "line values") + && matches!(index.as_ref(), Expression::Literal(wfl::parser::ast::Literal::Integer(0), ..)) + ), + "classic fallback must retain the full merged binding for direct indexing, got {:#?}", + stream_write_fallback(&program.statements[0]) + ); +} + +#[test] +fn streaming_response_content_type_of_call_parses() { + // `content type mime_type of path` — `of` continuation on the merged clause + // operand must compose like an ordinary expression (issue #642). + let program = parse( + "start streaming response to req with status 200 and content type mime_type of path as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::StartStreamingResponseStatement { content_type, .. } => { + assert!( + matches!(content_type, Some(Expression::FunctionCall { .. })), + "content type operand must be a FunctionCall (`mime_type of path`), got {content_type:#?}" + ); + } + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } +} + +#[test] +fn streaming_response_content_type_then_headers_both_orders() { + // Clause connectives must not be swallowed as Boolean AND (both orders). + let a = parse( + "start streaming response to req with status 200 and content type ct and headers h as out\n", + ); + match &a.statements[0] { + Statement::StartStreamingResponseStatement { + content_type, + headers, + .. + } => { + assert!(content_type.is_some(), "content type must bind"); + assert!( + headers.is_some(), + "headers must bind (not swallowed by content type)" + ); + } + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } + let b = parse( + "start streaming response to req with status 200 and headers h and content type ct as out\n", + ); + match &b.statements[0] { + Statement::StartStreamingResponseStatement { + content_type, + headers, + .. + } => { + assert!(headers.is_some(), "headers must bind"); + assert!( + content_type.is_some(), + "content type must bind (not swallowed by headers)" + ); + } + other => panic!("expected StartStreamingResponseStatement, got {other:#?}"), + } +} + +#[test] +fn streaming_clause_boundary_survives_nested_concatenation_rhs_in_both_orders() { + let cases = [ + ( + "start streaming response to req with content type \"text/\" with subtype and headers h as out\n", + "content type", + ), + ( + "start streaming response to req with headers base_headers with extra and content type ct as out\n", + "headers", + ), + ]; + + for (source, nested_clause) in cases { + let program = parse(source); + assert_eq!( + program.statements.len(), + 1, + "`{source}` must remain one statement; got {:#?}", + program.statements + ); + let statement = &program.statements[0]; + assert!( + matches!( + streaming_clause_operand(statement, nested_clause), + Expression::Concatenation { .. } + ), + "{nested_clause} must retain its complete concatenation operand; got {statement:#?}" + ); + assert!( + matches!( + statement, + Statement::StartStreamingResponseStatement { + content_type: Some(_), + headers: Some(_), + .. + } + ), + "the following response clause must not be swallowed into the concatenation RHS; \ + got {statement:#?}" + ); + } +} + +#[test] +fn type_prefixed_identifier_is_content_not_a_response_clause() { + let source = "start streaming response to req with content type \"application/\" with type suffix and headers h as out\n"; + let program = parse(source); + assert_eq!( + program.statements.len(), + 1, + "`{source}` must remain one statement; got {:#?}", + program.statements + ); + match &program.statements[0] { + Statement::StartStreamingResponseStatement { + content_type: Some(Expression::Concatenation { right, .. }), + headers: Some(Expression::Variable(headers, ..)), + .. + } => { + assert!( + matches!(right.as_ref(), Expression::Variable(name, ..) if name == "type suffix"), + "`type suffix` must remain the concatenation RHS, got {right:#?}" + ); + assert_eq!( + headers, "h", + "the following headers clause must remain separate" + ); + } + other => panic!( + "expected concatenated content type plus a separate headers clause, got {other:#?}" + ), + } +} + +#[test] +fn streaming_clause_boundary_survives_at_index_expression_in_both_orders() { + let cases = [ + ( + "start streaming response to req with content type media_types at kind and headers h as out\n", + "content type", + ), + ( + "start streaming response to req with headers header_sets at kind and content type ct as out\n", + "headers", + ), + ]; + + for (source, indexed_clause) in cases { + let program = parse(source); + assert_eq!( + program.statements.len(), + 1, + "`{source}` must remain one statement; got {:#?}", + program.statements + ); + let statement = &program.statements[0]; + assert!( + matches!( + streaming_clause_operand(statement, indexed_clause), + Expression::IndexAccess { .. } + ), + "{indexed_clause} must retain its complete `at` index operand; got {statement:#?}" + ); + assert!( + matches!( + statement, + Statement::StartStreamingResponseStatement { + content_type: Some(_), + headers: Some(_), + .. + } + ), + "the following response clause must not be swallowed into the `at` index; \ + got {statement:#?}" + ); + } +} + +#[test] +fn streaming_clause_boundary_propagates_through_recursive_operand_forms() { + let cases = [ + ( + "start streaming response to req with content type lookup of media_types at kind and headers h as out\n", + "of-call", + ), + ( + "start streaming response to req with content type touppercase with media_types at kind and headers h as out\n", + "builtin-call", + ), + ( + "start streaming response to req with content type not media_types at kind and headers h as out\n", + "unary", + ), + ]; + + for (source, operand_kind) in cases { + let program = parse(source); + assert_eq!( + program.statements.len(), + 1, + "`{source}` must remain one statement; got {:#?}", + program.statements + ); + let statement = &program.statements[0]; + let content_type = streaming_clause_operand(statement, "content type"); + let has_expected_shape = match operand_kind { + "of-call" => matches!(content_type, Expression::FunctionCall { .. }), + "builtin-call" => matches!(content_type, Expression::ActionCall { .. }), + "unary" => matches!(content_type, Expression::UnaryOperation { .. }), + _ => unreachable!("unknown recursive operand kind"), + }; + assert!( + has_expected_shape, + "{operand_kind} operand must retain its complete AST; got {content_type:#?}" + ); + assert!( + matches!( + statement, + Statement::StartStreamingResponseStatement { + content_type: Some(_), + headers: Some(_), + .. + } + ), + "the following headers clause must survive {operand_kind} recursion; got {statement:#?}" + ); + } +} + +#[test] +fn streaming_clause_boundary_propagates_through_explicit_call_arguments() { + let program = parse( + "start streaming response to req with content type call render with value and headers h as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + + let statement = &program.statements[0]; + let content_type = streaming_clause_operand(statement, "content type"); + assert!( + matches!( + content_type, + Expression::ActionCall { arguments, .. } + if arguments.len() == 1 + && matches!(&arguments[0].value, Expression::Variable(name, ..) if name == "value") + ), + "the following headers clause must not become another explicit-call argument; \ + got {content_type:#?}" + ); + assert!( + matches!( + statement, + Statement::StartStreamingResponseStatement { + headers: Some(Expression::Variable(name, ..)), + .. + } if name == "h" + ), + "the headers clause must remain outside the explicit call; got {statement:#?}" + ); +} + +#[test] +fn streaming_clause_boundary_propagates_through_at_index_under_file_exists() { + let program = parse( + "start streaming response to req with content type file exists at paths at kind and headers h as out\n", + ); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + + let statement = &program.statements[0]; + let content_type = streaming_clause_operand(statement, "content type"); + assert!( + matches!( + content_type, + Expression::FileExists { path, .. } + if matches!( + path.as_ref(), + Expression::IndexAccess { collection, index, .. } + if matches!(collection.as_ref(), Expression::Variable(name, ..) if name == "paths") + && matches!(index.as_ref(), Expression::Variable(name, ..) if name == "kind") + ) + ), + "the wrapper's nested `at` index must stop before the next response clause; \ + got {content_type:#?}" + ); + assert!( + matches!( + statement, + Statement::StartStreamingResponseStatement { + headers: Some(Expression::Variable(name, ..)), + .. + } if name == "h" + ), + "the headers clause must survive recursion through `file exists at`; got {statement:#?}" + ); +} + +fn assert_post_of_index(expr: &Expression, expected_function: &str, expected_argument: &str) { + match expr { + Expression::IndexAccess { + collection, index, .. + } => { + assert!( + matches!( + index.as_ref(), + Expression::Literal(wfl::parser::ast::Literal::Integer(0), ..) + ), + "post-call index must be integer zero, got {index:#?}" + ); + match collection.as_ref() { + Expression::FunctionCall { + function, + arguments, + .. + } => { + assert_eq!( + leftmost_variable(function), + Some(expected_function), + "unexpected function root in {expr:#?}" + ); + assert!( + matches!( + arguments.as_slice(), + [wfl::parser::ast::Argument { + value: Expression::Variable(name, ..), + .. + }] if name == expected_argument + ), + "the parenthesized argument must stay inside the call, got {arguments:#?}" + ); + } + other => panic!("index must wrap an `of` FunctionCall, got {other:#?}"), + } + } + other => panic!("expected postfix index after an `of` call, got {other:#?}"), + } +} + +fn post_of_index_shape(expr: &Expression) -> (String, String, i64) { + match expr { + Expression::IndexAccess { + collection, index, .. + } => { + let index = match index.as_ref() { + Expression::Literal(wfl::parser::ast::Literal::Integer(index), ..) => *index, + other => panic!("expected an integer post-call index, got {other:#?}"), + }; + match collection.as_ref() { + Expression::FunctionCall { + function, + arguments, + .. + } => { + let function = leftmost_variable(function).unwrap_or_else(|| { + panic!("expected a variable call root, got {function:#?}") + }); + let argument = match arguments.as_slice() { + [ + wfl::parser::ast::Argument { + value: Expression::Variable(name, ..), + .. + }, + ] => name, + other => panic!("expected one variable call argument, got {other:#?}"), + }; + (function.to_string(), argument.to_string(), index) + } + other => panic!("expected the index to wrap an `of` call, got {other:#?}"), + } + } + other => panic!("expected a post-`of` index expression, got {other:#?}"), + } +} + +fn initializer_expression(source: &str) -> Expression { + let program = parse(&format!("store parity result as {source}\n")); + match &program.statements[0] { + Statement::VariableDeclaration { value, .. } => value.clone(), + other => panic!("expected a variable initializer, got {other:#?}"), + } +} + +#[test] +fn seeded_operands_resume_postfix_parsing_after_of_calls() { + let write = parse("write line choose of (chunks)[0] to out\n"); + assert_eq!(write.statements.len(), 1, "got {:#?}", write.statements); + assert_post_of_index(stream_write_value(&write.statements[0]), "choose", "chunks"); + assert_post_of_index( + stream_write_fallback(&write.statements[0]), + "line choose", + "chunks", + ); + + let streaming = parse( + "start streaming response to req with content type choose of (types)[0] and headers h as out\n", + ); + assert_eq!( + streaming.statements.len(), + 1, + "got {:#?}", + streaming.statements + ); + assert_post_of_index( + streaming_clause_operand(&streaming.statements[0], "content type"), + "choose", + "types", + ); + assert!( + matches!( + &streaming.statements[0], + Statement::StartStreamingResponseStatement { + headers: Some(Expression::Variable(name, ..)), + .. + } if name == "h" + ), + "headers must remain outside the indexed call; got {:#?}", + streaming.statements[0] + ); + + let flush = parse("flush cache of (items)[0]\n"); + assert_eq!(flush.statements.len(), 1, "got {:#?}", flush.statements); + match &flush.statements[0] { + Statement::FlushStreamStatement { + target, + action_fallback: Some(fallback), + .. + } => { + assert_post_of_index(target, "cache", "items"); + assert_post_of_index(fallback, "flush cache", "items"); + } + other => panic!("expected ambiguous FlushStreamStatement, got {other:#?}"), + } +} + +#[test] +fn post_of_operands_match_the_ordinary_expression_ast_shape() { + let write = parse("write line choose of (chunks)[0] to out\n"); + let ordinary_write = initializer_expression("choose of (chunks)[0]"); + assert_eq!( + post_of_index_shape(stream_write_value(&write.statements[0])), + post_of_index_shape(&ordinary_write), + "the seeded write operand must compose exactly like an ordinary expression" + ); + let ordinary_classic_write = initializer_expression("line choose of (chunks)[0]"); + assert_eq!( + post_of_index_shape(stream_write_fallback(&write.statements[0])), + post_of_index_shape(&ordinary_classic_write), + "the classic write fallback must preserve the ordinary post-`of` AST" + ); + + let streaming = + parse("start streaming response to req with content type choose of (types)[0] as out\n"); + let ordinary_content_type = initializer_expression("choose of (types)[0]"); + assert_eq!( + post_of_index_shape(streaming_clause_operand( + &streaming.statements[0], + "content type", + )), + post_of_index_shape(&ordinary_content_type), + "the content-type operand must compose exactly like an ordinary expression" + ); + + let flush = parse("flush cache of (items)[0]\n"); + let (target, fallback) = match &flush.statements[0] { + Statement::FlushStreamStatement { + target, + action_fallback: Some(fallback), + .. + } => (target, fallback), + other => panic!("expected an ambiguous FlushStreamStatement, got {other:#?}"), + }; + let ordinary_flush_target = initializer_expression("cache of (items)[0]"); + assert_eq!( + post_of_index_shape(target), + post_of_index_shape(&ordinary_flush_target), + "the streaming flush target must preserve the ordinary post-`of` AST" + ); + let ordinary_legacy_flush = initializer_expression("flush cache of (items)[0]"); + assert_eq!( + post_of_index_shape(fallback), + post_of_index_shape(&ordinary_legacy_flush), + "the legacy flush fallback must preserve the ordinary post-`of` AST" + ); +} + +#[test] +fn write_line_of_call_argument_absorbs_arithmetic() { + // `double of n minus 1` must parse as `double of (n minus 1)` — the same + // precedence as an ordinary expression — not `(double of n) minus 1`. + let program = parse("write line double of n minus 1 to out\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match stream_write_value(&program.statements[0]) { + Expression::FunctionCall { arguments, .. } => { + assert_eq!( + arguments.len(), + 1, + "the `of` call should take one argument, got {arguments:#?}" + ); + assert!( + matches!(arguments[0].value, Expression::BinaryOperation { .. }), + "the `of` argument must absorb `minus 1` (double of (n minus 1)), got {:#?}", + arguments[0].value + ); + } + other => panic!("expected the value to be an `of` FunctionCall, got {other:#?}"), + } +} + +#[test] +fn write_line_method_call_operand_composes() { + // `obj.method()` must compose into a MethodCall, not leave `()` dangling. + let program = parse("write line obj.method() to out\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + assert!( + matches!( + stream_write_value(&program.statements[0]), + Expression::MethodCall { .. } + ), + "the write value must be a MethodCall, got {:#?}", + stream_write_value(&program.statements[0]) + ); +} + +#[test] +fn flush_method_call_operand_composes() { + // `flush obj.method()` must compose the method call onto the operand. + let program = parse("flush obj.method()\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::FlushStreamStatement { target, .. } => { + assert!( + matches!(target, Expression::MethodCall { .. }), + "the flush operand must be a MethodCall, got {target:#?}" + ); + } + other => panic!("expected FlushStreamStatement, got {other:#?}"), + } +} + +fn leftmost_variable(expr: &Expression) -> Option<&str> { + match expr { + Expression::Variable(name, ..) => Some(name), + Expression::IndexAccess { collection, .. } => leftmost_variable(collection), + Expression::PropertyAccess { object, .. } | Expression::MethodCall { object, .. } => { + leftmost_variable(object) + } + Expression::BinaryOperation { left, .. } => leftmost_variable(left), + Expression::FunctionCall { function, .. } => leftmost_variable(function), + _ => None, + } +} + +#[test] +fn flush_preserves_binary_continuation_for_both_interpretations() { + let program = parse("flush cache plus 1\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::FlushStreamStatement { + target, + action_fallback, + .. + } => { + assert!( + matches!(target, Expression::BinaryOperation { .. }), + "stream target must keep `plus 1`, got {target:#?}" + ); + assert_eq!(leftmost_variable(target), Some("cache")); + let fallback = action_fallback.as_ref().expect("legacy fallback"); + assert!( + matches!(fallback, Expression::BinaryOperation { .. }), + "legacy expression must keep `plus 1`, got {fallback:#?}" + ); + assert_eq!(leftmost_variable(fallback), Some("flush cache")); + } + other => panic!("expected FlushStreamStatement, got {other:#?}"), + } +} + +#[test] +fn flush_preserves_arbitrarily_nested_postfix_for_both_interpretations() { + let program = parse("flush cache[0][0]\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + match &program.statements[0] { + Statement::FlushStreamStatement { + target, + action_fallback, + .. + } => { + assert!( + matches!( + target, + Expression::IndexAccess { collection, .. } + if matches!(collection.as_ref(), Expression::IndexAccess { .. }) + ), + "stream target must retain both indexes, got {target:#?}" + ); + assert_eq!(leftmost_variable(target), Some("cache")); + let fallback = action_fallback.as_ref().expect("legacy fallback"); + assert!( + matches!( + fallback, + Expression::IndexAccess { collection, .. } + if matches!(collection.as_ref(), Expression::IndexAccess { .. }) + ), + "legacy expression must retain both indexes, got {fallback:#?}" + ); + assert_eq!(leftmost_variable(fallback), Some("flush cache")); + } + other => panic!("expected FlushStreamStatement, got {other:#?}"), + } +} + +#[test] +fn classic_indexed_file_write_still_works_at_runtime() { + // The ambiguous merged form's classic file-write reading must keep working with + // an indexed operand: `write line values[0] to `. The target is a text + // path, so the runtime takes the classic reading whose content is the merged + // lead `line values` indexed at 0 — it must write the first element, not fail on + // a dangling `[0]`. + let dir = TempDir::new().expect("tempdir"); + let out = dir.path().join("out.txt"); + let out_str = out.to_string_lossy().replace('\\', "/"); + let src = format!( + "store line values as [\"first\" and \"second\"]\n\ + write line values[0] to \"{out_str}\"\n" + ); + let program_file = dir.path().join("main.wfl"); + fs::write(&program_file, &src).unwrap(); + let status = Command::new(env!("CARGO_BIN_EXE_wfl")) + .arg(&program_file) + .status() + .expect("run wfl"); + assert!( + status.success(), + "classic indexed file write should succeed" + ); + let written = fs::read_to_string(&out).expect("output file written"); + assert_eq!( + written.trim_end(), + "first", + "the indexed element must be written, not the whole list" + ); +} + +fn run_file_write_with_line_binding(statement: &str, declaration: &str) -> String { + let dir = TempDir::new().expect("tempdir"); + let out = dir.path().join("out.txt"); + let out_str = out.to_string_lossy().replace('\\', "/"); + let src = format!("{declaration}\n{statement} to \"{out_str}\"\n"); + let program_file = dir.path().join("main.wfl"); + fs::write(&program_file, &src).expect("write program"); + let output = Command::new(env!("CARGO_BIN_EXE_wfl")) + .arg(&program_file) + .output() + .expect("run wfl"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.status.success(), + "`{statement}` must remain a classic file write; output:\n{combined}" + ); + fs::read_to_string(out).expect("classic write must create output file") +} + +#[test] +fn bare_line_binding_keeps_with_continuation_in_classic_file_write() { + let written = + run_file_write_with_line_binding("write line with \"!\"", "store line as \"hello\""); + assert_eq!(written, "hello!"); +} + +#[test] +fn bare_line_binding_keeps_natural_index_continuation_in_classic_file_write() { + let written = run_file_write_with_line_binding("write line at 0", "store line as [\"first\"]"); + assert_eq!(written, "first"); +} + +#[test] +fn bare_line_binding_keeps_bracket_index_continuation_in_classic_file_write() { + let written = run_file_write_with_line_binding("write line[0]", "store line as [\"first\"]"); + assert_eq!(written, "first"); +} + +#[test] +fn bare_line_binding_keeps_binary_continuation_in_classic_file_write() { + let written = run_file_write_with_line_binding("write line plus 1", "store line as 4"); + assert_eq!(written, "5"); +} + +#[test] +fn bare_line_binding_keeps_direct_integer_index_as_classic_fallback() { + let written = run_file_write_with_line_binding("write line 0", "store line as [\"first\"]"); + assert_eq!(written, "first"); +} + +#[test] +fn display_property_followed_by_spaced_list_keeps_legacy_statement_split() { + let program = parse("display alice.name [1, 2]\n"); + assert_eq!(program.statements.len(), 2, "got {:#?}", program.statements); + assert!( + matches!( + &program.statements[0], + Statement::DisplayStatement { + value: Expression::PropertyAccess { .. }, + .. + } + ), + "the property itself must remain the displayed value; got {:#?}", + program.statements[0] + ); + assert!( + matches!( + &program.statements[1], + Statement::ExpressionStatement { + expression: Expression::Literal(wfl::parser::ast::Literal::List(_), ..), + .. + } + ), + "the spaced list must remain the separate legacy expression, not an index; got {:#?}", + program.statements[1] + ); +} + +#[test] +fn display_property_followed_by_integer_remains_a_display_fold() { + let program = parse("display alice.name 5\n"); + assert_eq!(program.statements.len(), 1, "got {:#?}", program.statements); + assert!( + matches!( + &program.statements[0], + Statement::DisplayStatement { + value: Expression::Concatenation { left, right, .. }, + .. + } if matches!(left.as_ref(), Expression::PropertyAccess { .. }) + && matches!( + right.as_ref(), + Expression::Literal(wfl::parser::ast::Literal::Integer(5), ..) + ) + ), + "the integer must be a second display value, not direct indexing; got {:#?}", + program.statements[0] + ); +} + +#[test] +fn display_property_compatibility_executes_through_the_real_binary() { + let dir = TempDir::new().expect("tempdir"); + let program_file = dir.path().join("main.wfl"); + fs::write( + &program_file, + "create map alice:\n\ + \x20\x20\x20\x20\"name\" is \"Alice\"\n\ + end map\n\ + display alice.name 5\n", + ) + .expect("write program"); + let output = Command::new(env!("CARGO_BIN_EXE_wfl")) + .arg(&program_file) + .output() + .expect("run wfl"); + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + assert!( + output.status.success(), + "display fold must not index `Alice` by 5; output:\n{combined}" + ); + assert!( + combined.contains("Alice5"), + "legacy display fold must print both values; output:\n{combined}" + ); +}