diff --git a/.github/workflows/mk05-relay-checks.yml b/.github/workflows/mk05-relay-checks.yml new file mode 100644 index 000000000..e33693804 --- /dev/null +++ b/.github/workflows/mk05-relay-checks.yml @@ -0,0 +1,103 @@ +name: MK05 relay checks +on: + push: + branches: [wip/mk-05-relay] + pull_request: + branches: [wip/rust-core-go-extensions] + paths: + - apps/core-rust/** + - apps/api-go/** + - proto/** + - scripts/test-core-boundaries.py + - .github/workflows/mk05-relay-checks.yml +permissions: + contents: read +concurrency: + group: mk05-${{ github.event_name }}-${{ github.ref }} + cancel-in-progress: false +jobs: + relay: + runs-on: ubuntu-latest + timeout-minutes: 25 + env: + DATABASE_URL: postgres://postgres:postgres@127.0.0.1:5432/postgres + CARGO_BUILD_JOBS: '2' + CARGO_INCREMENTAL: '0' + CARGO_PROFILE_DEV_DEBUG: '0' + CARGO_PROFILE_TEST_DEBUG: '0' + GOWORK: 'off' + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_PASSWORD: postgres + ports: ['5432:5432'] + options: >- + --health-cmd "pg_isready -U postgres" --health-interval 5s + --health-timeout 3s --health-retries 10 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + persist-credentials: false + - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 + with: + go-version: '1.27.2' + cache-dependency-path: apps/api-go/go.sum + - name: Install Rust tools + run: rustup toolchain install 1.99.0 --profile minimal --component rustfmt --component clippy + - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + apps/core-rust/target + key: mk05-rust-${{ runner.os }}-${{ hashFiles('apps/core-rust/Cargo.lock') }} + restore-keys: mk05-rust-${{ runner.os }}- + save-always: true + - name: Record source and tool versions + run: | + set -euo pipefail + git rev-parse HEAD | tee "$RUNNER_TEMP/mk05-tested-commit.txt" + { + rustc +1.99.0 --version + cargo +1.99.0 --version + go version + } | tee "$RUNNER_TEMP/mk05-tools.log" + - name: Build the actual Go extension for lifecycle tests + working-directory: apps/api-go + run: go build -mod=readonly -o "$RUNNER_TEMP/lmm-extensions" ./cmd/extensions + - name: Rust format and strict checks + shell: bash + run: | + set -euo pipefail + cargo +1.99.0 fmt --manifest-path apps/core-rust/Cargo.toml --all --check + cargo +1.99.0 clippy --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- -D warnings 2>&1 | tee "$RUNNER_TEMP/mk05-clippy.log" + - name: Rust unit, PostgreSQL and real HTTP lifecycle tests + if: ${{ !cancelled() }} + shell: bash + run: | + set -euo pipefail + export LMM_RELAY_EXTENSION_BIN="$RUNNER_TEMP/lmm-extensions" + cargo +1.99.0 test --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- --nocapture 2>&1 | tee "$RUNNER_TEMP/mk05-tests.log" + - name: Verify service boundaries and clean sources + if: ${{ !cancelled() }} + run: | + set -euo pipefail + python3 -B scripts/test-core-boundaries.py 2>&1 | tee "$RUNNER_TEMP/mk05-boundaries.log" + test ! -e apps/core-rust/src/relay/.review.py + git diff --exit-code + - name: Preserve tested task sources + if: always() + run: | + tar -czf "$RUNNER_TEMP/mk05-source.tgz" \ + apps/core-rust/src/relay apps/core-rust/tests/relay_http.rs \ + apps/core-rust/src/lib.rs apps/core-rust/Cargo.toml \ + apps/core-rust/Cargo.lock .github/workflows/mk05-relay-checks.yml + - name: Preserve verification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: mk05-relay-${{ github.run_id }} + path: ${{ runner.temp }}/mk05-* + if-no-files-found: warn + retention-days: 14 diff --git a/apps/core-rust/Cargo.lock b/apps/core-rust/Cargo.lock index b3a49b14a..c14359898 100644 --- a/apps/core-rust/Cargo.lock +++ b/apps/core-rust/Cargo.lock @@ -113,6 +113,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" @@ -137,6 +143,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + [[package]] name = "byteorder" version = "1.5.0" @@ -165,6 +177,23 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -180,6 +209,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -410,6 +448,17 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -430,6 +479,7 @@ checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -454,8 +504,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -465,8 +517,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", + "rand_core 0.10.1", + "wasm-bindgen", ] [[package]] @@ -620,6 +675,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots 1.0.9", +] + [[package]] name = "hyper-timeout" version = "0.5.2" @@ -639,6 +710,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ + "base64 0.23.1", "bytes", "futures-channel", "futures-util", @@ -646,7 +718,9 @@ dependencies = [ "http-body", "httparse", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -768,6 +842,12 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + [[package]] name = "itertools" version = "0.14.0" @@ -783,6 +863,17 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -846,7 +937,8 @@ dependencies = [ "fs2", "prost", "protoc-bin-vendored", - "rand", + "rand 0.8.5", + "reqwest", "serde", "serde_json", "sha2", @@ -875,6 +967,12 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "matchit" version = "0.8.4" @@ -931,7 +1029,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -1259,6 +1357,62 @@ dependencies = [ "pulldown-cmark", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -1282,7 +1436,18 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", ] [[package]] @@ -1292,7 +1457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1304,6 +1469,21 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1351,6 +1531,48 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 1.0.9", +] + [[package]] name = "ring" version = "0.17.14" @@ -1378,13 +1600,19 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "spki", "subtle", "zeroize", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustix" version = "1.1.5" @@ -1418,6 +1646,7 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ + "web-time", "zeroize", ] @@ -1432,6 +1661,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "ryu" version = "1.0.23" @@ -1517,7 +1752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1528,7 +1763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -1555,7 +1790,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -1621,7 +1856,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", @@ -1696,7 +1931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -1718,7 +1953,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -1738,7 +1973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "crc", @@ -1756,7 +1991,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha2", @@ -1842,6 +2077,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -1930,6 +2168,16 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -1963,7 +2211,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2", "http", @@ -2042,6 +2290,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -2188,6 +2454,95 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.26.11" diff --git a/apps/core-rust/Cargo.toml b/apps/core-rust/Cargo.toml index 56f465977..6116d59d7 100644 --- a/apps/core-rust/Cargo.toml +++ b/apps/core-rust/Cargo.toml @@ -10,6 +10,7 @@ publish = false [dependencies] axum = "=0.8.9" rand = "=0.8.5" +reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "rustls-tls", "http2"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" sha2 = "=0.10.9" diff --git a/apps/core-rust/src/lib.rs b/apps/core-rust/src/lib.rs index 7b1f3d6ba..6dc7873c9 100644 --- a/apps/core-rust/src/lib.rs +++ b/apps/core-rust/src/lib.rs @@ -1,7 +1,7 @@ //! Stable model-request core. No extension runtime or Go dependency belongs here. //! -//! PostgreSQL identity is usable in isolation. The ledger and model forwarding -//! are not ready; identity authorization alone must never permit a paid request. +//! PostgreSQL identity is usable in isolation. Public paid model endpoints stay +//! disabled until durable billing and routing adapters are integrated. pub mod accounts; pub mod billing; pub mod config; @@ -11,3 +11,4 @@ pub mod identity; pub mod identity_http; #[cfg(unix)] pub mod internal_rpc; +pub mod relay; diff --git a/apps/core-rust/src/relay/README.md b/apps/core-rust/src/relay/README.md new file mode 100644 index 000000000..c05fc5db1 --- /dev/null +++ b/apps/core-rust/src/relay/README.md @@ -0,0 +1,145 @@ +# Rust relay (MK-05) + +Parent: #675. Work branch: `wip/mk-05-relay`. Base at start: +`b83723cdac3f148140806b8025cb5360a2f6a26d`. + +This is an internal relay library, not an enabled public model API. Existing +HTTP model endpoints and business readiness remain unavailable. No production +billing adapter or task-04 routing adapter is supplied in this task. Do not +open paid endpoints until those core-owned adapters pass integration tests. + +## Supported surface + +Clients: OpenAI Chat Completions and Responses. Upstreams: OpenAI Chat, +OpenAI Responses, Anthropic Messages, and Gemini GenerateContent. Each pair +supports JSON and SSE, one candidate, text, function tools and tool results. +The HTTP tests exercise all 16 provider/client/output-mode combinations. + +The route selects the complete upstream operation URL and real model name. +Gemini streaming needs `:streamGenerateContent?alt=sse`; JSON needs +`:generateContent`. The client still sees its requested model alias. + +The bounded internal block format keeps function call IDs separate from +Responses output-item IDs. It keeps tool argument fragments, thinking text, +provider signatures, usage and finish reasons separate. Responses event +sequence numbers are increasing. Length-limited tool JSON remains incomplete; +a normal tool completion with invalid JSON is an error. An HTTP EOF without a +required semantic end is an error, never an invented success. + +Anthropic cache-read and cache-creation tokens are added to its uncached input +count. Gemini thoughts are included in normalized output usage. Missing usage +stays `None`, not zero. Counters cannot decrease or overflow; invalid counters leave the last accepted snapshot unchanged. Thinking text +uses `reasoning_content` for Chat and reasoning-summary items for Responses. +Plain-text signatures use the Chat `content_details` extension or Responses item metadata. These details are preserved on output; replay of signed plain-text history is not yet supported. +Opaque signatures remain provider-tagged metadata, not fabricated OpenAI +reasoning tokens. They cannot be replayed to a different provider. + +This is deliberately not a universal lossless bridge. Images, audio, video, +multiple candidates, hosted/server tools, encrypted or stateful Responses +history, unknown request fields, annotated Responses output, unsupported provider content blocks and +unsupported finish reasons fail explicitly. Anthropic redacted-thinking blocks +are not supported. Chat streams that fragment the tool name/ID before the +initial complete identity are not supported. Multiple signed thinking blocks +are returned as provider-tagged details; replay of those details is not yet a +supported input. Provider-specific features must not be silently discarded. + +## Integration contract for tasks 03 and 04 + +`RouteProvider::select` receives the verified actor/key context and normalized +request. Return an immutable Rust-owned `Route`, including its price version, +complete URL, credential and model. Do not query or wait for Go in this method. +Route origins must come from trusted active configuration, not request input. +HTTPS is mandatory except for explicit numeric loopback upstreams. Redirects, +URL credentials, credential query parameters and ambient proxy settings are +not used. Upstream error bodies and credentials are not returned to clients. + +`Billing::reserve` must authorize the actor/key, payer, team restrictions, +budget and subscription policy, then commit a durable reservation. It must be +idempotent by `RequestContext.request_id`. Return only a reservation handle, +not a SQL connection or transaction. The relay sends no model request until +this hook succeeds. The relay does not import SQLx or a Go extension client. + +`Billing::finalize` runs exactly once in each live worker after a reservation +handle was returned, including cancellation, timeout and upstream failure. +The adapter must make the durable operation idempotent across retries/restarts. +It must persist a final charge or a durable reconciliation obligation. Missing +usage, an ambiguous send, failed finalization and a reservation timeout are +not evidence of zero cost. A cancelled reserve future may already have +committed: recover by request ID. The billing implementation must own a +persistent sweeper/recovery queue; an in-process callback is not sufficient. + +The client may receive streaming deltas before settlement. Successful terminal +markers (`[DONE]`, `response.completed`, `response.incomplete`) and a JSON 200 +response are withheld until finalization confirms persistence. Finalization +failure is a `settlement_pending` error. A report's outcome describes upstream +work; a later client delivery failure is reported separately in Completion. + +Integration sketch (not a public route): + +```rust,ignore +let request = Request::parse(ClientProtocol::Chat, body, &limits)?; +let session = relay.start(request, verified_context)?; +let (response, completion) = session.into_http().await?; +// Observe Completion and durable reconciliation state outside the HTTP body. +// Route registration and authentication belong to the final integration task. +``` + +## Resource and shutdown limits + +Defaults: 16 admitted requests; 8 queued chunks per request; 16 KiB per normal +output chunk; 2 MiB input body; 256 KiB SSE event; 1 MiB accumulated output; +256 KiB per tool's arguments; 128 output blocks; 64 MiB total upstream bytes. +There is no unbounded event queue. HTTP chunks are consumed byte by byte by the +SSE framer, so a large network chunk does not create an unbounded event list. +Both source and JSON-escaped output have explicit limits. JSON escaping and +Responses final snapshots require extra bounded copies; these figures are +not an RSS guarantee. Allocator, TLS, HTTP, JSON tree and kernel socket buffers +are additional. Increase concurrency/output caps only after a memory load test. + +Defaults: 5 s connect; 30 s headers; 30 s idle read; 10 s slow-client write; +300 s forwarding deadline; 10 s per billing/routing hook. Bounded money cleanup can continue after the forwarding deadline. Client-body drop cancels reading, +writing and the upstream request. The reservation and finalizer have their own +short deadline and are not aborted just because the client disconnected. + +Retry defaults to zero. At most two explicit retries may be configured, only +for connect failures. HTTP statuses, redirects, read failures, partial output, +header timeouts and ambiguous POST failures are not replayed. Reqwest's own +retry policy is disabled. A retry does not reset the total request deadline. + +`Relay::drain(grace)` closes admission atomically, waits for current requests, +then cancels remaining upstreams and waits for bounded hooks. It reports both +cancelled and remaining workers. The host must inspect `remaining`; do not +claim success while work remains. Drop upstream bodies after terminal events +instead of consuming arbitrary trailing bytes to reuse a connection. Idle +connection pooling is disabled to avoid retaining connections for an unbounded +set of configured origins; active connections and HTTP/2 windows are bounded. + +Stopping Go does not notify or cancel relay workers. The acceptance test builds +and starts the actual Go extension binary, reads a Rust HTTP stream, kills Go, +then checks later output and one finalization. This is not a Rust/container +restart test. A single Rust instance restart can break a stream. Zero lost +tokens across restarts requires deployment-level draining and multiple healthy +instances, and is not guaranteed here. + +## Reproduce verification + +Use the repository's pinned Rust 1.99.0 and Go 1.27.2 tools. PostgreSQL is needed +only for existing core database tests, not the relay HTTP tests. No provider +account, production database, API key or paid model call is used by these tests. + +```sh +export GOWORK=off +(cd apps/api-go && go build -mod=readonly -o /tmp/lmm-relay-extension ./cmd/extensions) +export LMM_RELAY_EXTENSION_BIN=/tmp/lmm-relay-extension +cargo +1.99.0 fmt --manifest-path apps/core-rust/Cargo.toml --all --check +cargo +1.99.0 clippy --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- -D warnings +cargo +1.99.0 test --manifest-path apps/core-rust/Cargo.toml --locked --test relay_http -- --nocapture +# Set DATABASE_URL to an isolated test PostgreSQL instance before all-targets. +cargo +1.99.0 test --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- --nocapture +python3 -B scripts/test-core-boundaries.py +``` + +The real Go process test fails, rather than silently skipping, when its binary +is not supplied. The CI evidence artifact includes the tested commit, strict +checks, test output, boundary checks and task-source snapshot. The PR records +which exact run has passed. Source submission alone is not a passing result. diff --git a/apps/core-rust/src/relay/VERIFICATION.md b/apps/core-rust/src/relay/VERIFICATION.md new file mode 100644 index 000000000..10bdaa499 --- /dev/null +++ b/apps/core-rust/src/relay/VERIFICATION.md @@ -0,0 +1,135 @@ +# MK-05 verification record + +Date: 2026-10-10 (UTC). Draft PR: [#696](https://github.com/TokenNotIncluded/api.lmm.best/pull/696). +Parent: [#675](https://github.com/TokenNotIncluded/api.lmm.best/pull/675). +Work branch: `wip/mk-05-relay`. Target: `wip/rust-core-go-extensions`. +Initial base: `b83723cdac3f148140806b8025cb5360a2f6a26d`. + +## Evidence inspected before this record + +Implementation and tests: `6c2a39afd0c17cc0ee8bcc32185c4a1ef5f0483f`. +Successful run: [38022308784](https://github.com/TokenNotIncluded/api.lmm.best/actions/runs/38022308784). +Evidence: [artifact 11658308770](https://github.com/TokenNotIncluded/api.lmm.best/actions/runs/38022308784/artifacts/11658308770), +`mk05-relay-38022308784.zip`, SHA-256: +`d1974ee2ffc4411be98f7a3f93d8920ecb9a6a224b2b7dbbb32174a8bbf71a60`. + +The run was triggered by `33901931e50599b612d76815ffe3cdf95c46a6ed`. +During development, its branch-only normalization step applied reviewed source +corrections and committed the tested tree. The artifact's +`mk05-tested-commit.txt` records the actual tested commit above; do not confuse +it with the trigger commit. The temporary source-editing script was removed. + +The workflow accompanying this record is read-only. It has no source-editing, +format-fix, commit, push or repository-write permission. It checks both branch +pushes and pull requests. Its evidence records the exact checkout commit and +tool versions. Use the PR checks and their artifacts for results after this +record; this document does not assert a future run has already passed. + +## Observed results + +`cargo fmt --all --check`: passed. +`cargo clippy --locked --all-targets -- -D warnings`: passed. +`cargo test --locked --all-targets -- --nocapture`: 69 passed, 0 failed, +0 ignored. Breakdown: 36 library unit tests, 4 schema tests, 2 identity HTTP +PostgreSQL tests, 7 identity PostgreSQL tests, 20 relay HTTP tests. The 69 +includes existing core tests; this task adds 17 relay unit tests and 20 relay +HTTP tests. Binary targets without tests report zero tests. + +`python3 -B scripts/test-core-boundaries.py`: 8 passed. +The Go extension executable was built from this checkout with `-mod=readonly`. +The Rust toolchain was 1.99.0; Go was 1.27.2. Tests ran in GitHub Actions on +Linux with an isolated PostgreSQL service. They were not run in the assistant's +local environment, which lacks those build tools. + +The protocol matrix contains 16 cases inside one of the 20 HTTP tests: +4 upstream protocols x 2 client formats x JSON/SSE. It is not 16 additional +Rust test functions. The upstream protocols are OpenAI Chat, OpenAI Responses, +Anthropic Messages and Gemini GenerateContent. Clients are OpenAI Chat and +Responses. All matrix cases printed a passing record. + +## What the real HTTP tests verify + +The fixture uses loopback TCP sockets and chunked HTTP, not just JSON decoder +unit tests. Three-byte upstream chunks split UTF-8 and SSE boundaries. The +matrix checks text, reasoning text, tool IDs, split function arguments, +normalized usage, finish reasons, model alias preservation and upstream auth. +Separate tests read an actual Axum HTTP response before upstream completion. + +Disconnect/cancel tests verify upstream closure, release of admission slots, +and one billing finalization after an accepted reservation. Other tests cover +reserve/finalize timeouts, header/read/total deadlines, refused connections, +HTTP 302/401/429/500/503, malformed or truncated streams, invalid UTF-8, +no retries after ambiguous work, slow-client backpressure, queue/chunk limits, +SSE event bounds, output/tool/body caps and incomplete function JSON. + +Billing denial sends no upstream request. A failed or timed-out finalizer +withholds success markers and returns `settlement_pending`. Missing usage +stays unknown. Invalid cumulative counters do not replace the last valid +snapshot. No upstream error body or credential is exposed to the client. + +Regression tests also check Gemini signatures on ordinary text, separate +content/thinking metadata, unsupported encrypted/annotated Responses output, +rejected ambiguous input, sparse oversized tool indexes, and error packets +obeying the same chunk cap as model output. + +## Go process independence result + +The lifecycle test builds and starts the actual `lmm-extensions` binary. It +receives a first Rust HTTP output chunk, kills the Go process, then permits the +mock upstream to send another chunk. The Rust client receives later output, +`[DONE]`, and a single finalization with input/output usage 2/3. + +The inspected run logged: + +```text +actual Go extension pid=3699 killed after first HTTP output; Rust emitted later output and settled once; no Rust restart was performed +``` + +This proves independence from that Go process in this test configuration. +It does not test restarting Rust, restarting a container, changing a proxy or +a load balancer, or terminating the host. It does not guarantee uninterrupted +tokens during a single-instance restart. + +## Limits of this evidence and remaining integration + +No live provider account or paid API call was used. Upstreams are simulated +HTTP servers. Routing and billing use test adapters. The tests do not certify +complete compatibility with all provider features or SDK versions. + +The byte, queue, block and concurrency limits are exercised. This is not an +RSS or allocator-peak benchmark. TLS, HTTP, JSON trees, escaped output copies, +Responses terminal snapshots and kernel buffers use additional bounded or +implementation-managed memory. Tune limits only after a deployment load test. + +No public model route is connected. Existing model endpoints and business +readiness remain 503. No production billing, payer selection, budget, +subscription, durable reconciliation or active route store is supplied here. +Tasks 03/04 must implement the documented `Billing` and `RouteProvider` +contracts and pass paid-request integration tests before exposing an endpoint. +A reservation timeout may have committed; recovery must use durable state, +not assume zero cost. The relay itself holds no SQL transaction over a stream. + +Unsupported features and provider-specific metadata rules are in [README.md](README.md). +The PR remains Draft. No merge, production deployment or old-data migration +is part of this task. + +## Reproduce + +From the repository root, using the exact tested checkout: + +```sh +export GOWORK=off +(cd apps/api-go && go build -mod=readonly -o /tmp/lmm-relay-extension ./cmd/extensions) +export LMM_RELAY_EXTENSION_BIN=/tmp/lmm-relay-extension +cargo +1.99.0 fmt --manifest-path apps/core-rust/Cargo.toml --all --check +cargo +1.99.0 clippy --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- -D warnings +cargo +1.99.0 test --manifest-path apps/core-rust/Cargo.toml --locked --test relay_http -- --nocapture +# Set DATABASE_URL to an isolated PostgreSQL instance, never production. +cargo +1.99.0 test --manifest-path apps/core-rust/Cargo.toml --locked --all-targets -- --nocapture +python3 -B scripts/test-core-boundaries.py +git diff --exit-code +``` + +The real Go lifecycle test fails instead of silently skipping when +`LMM_RELAY_EXTENSION_BIN` is missing. CI artifacts include the source snapshot, +exact tested commit, strict-check log, full test log and boundary-check log. diff --git a/apps/core-rust/src/relay/decode.rs b/apps/core-rust/src/relay/decode.rs new file mode 100644 index 000000000..6f9d4049b --- /dev/null +++ b/apps/core-rust/src/relay/decode.rs @@ -0,0 +1,906 @@ +//! Provider decoders share a bounded block model. Usage counters are cumulative. +use super::sse::Frame; +use super::{BlockKind, Event, FinishReason, Limits, RelayError, UpstreamProtocol, Usage}; +use serde_json::Value; +use std::collections::BTreeMap; + +type Key = (u64, u64); +struct Slot { + index: usize, + kind: BlockKind, + ended: bool, + has_text: bool, + default_object: bool, +} +pub(crate) struct Decoder { + protocol: UpstreamProtocol, + started: bool, + source_id: Option, + done: bool, + finish: Option, + blocks: BTreeMap, + max_blocks: usize, + gemini_calls: u64, +} +fn text(v: &Value) -> Result<&str, RelayError> { + v.as_str() + .ok_or(RelayError::Protocol("expected upstream string")) +} +fn index(v: &Value) -> Result { + v.as_u64() + .ok_or(RelayError::Protocol("invalid upstream index")) +} +fn counter(v: &Value, key: &str) -> Result, RelayError> { + v.get(key) + .filter(|v| !v.is_null()) + .map(|v| { + v.as_u64() + .ok_or(RelayError::Protocol("invalid usage counter")) + }) + .transpose() +} +fn add(a: Option, b: Option, c: Option) -> Result, RelayError> { + a.map(|a| { + a.checked_add(b.unwrap_or(0)) + .and_then(|n| n.checked_add(c.unwrap_or(0))) + .ok_or(RelayError::Protocol("usage overflow")) + }) + .transpose() +} +fn usage(protocol: UpstreamProtocol, v: &Value) -> Result { + if v.is_null() { + return Ok(Usage::default()); + } + if !v.is_object() { + return Err(RelayError::Protocol("invalid usage object")); + } + let mut u = Usage::default(); + match protocol { + UpstreamProtocol::Chat => { + u.input_tokens = counter(v, "prompt_tokens")?; + u.output_tokens = counter(v, "completion_tokens")?; + u.cached_input_tokens = counter(&v["prompt_tokens_details"], "cached_tokens")?; + u.reasoning_output_tokens = + counter(&v["completion_tokens_details"], "reasoning_tokens")?; + } + UpstreamProtocol::Responses => { + u.input_tokens = counter(v, "input_tokens")?; + u.output_tokens = counter(v, "output_tokens")?; + u.cached_input_tokens = counter(&v["input_tokens_details"], "cached_tokens")?; + u.reasoning_output_tokens = counter(&v["output_tokens_details"], "reasoning_tokens")?; + } + UpstreamProtocol::Anthropic => { + u.cached_input_tokens = counter(v, "cache_read_input_tokens")?; + u.cache_creation_input_tokens = counter(v, "cache_creation_input_tokens")?; + u.input_tokens = add( + counter(v, "input_tokens")?, + u.cached_input_tokens, + u.cache_creation_input_tokens, + )?; + u.output_tokens = counter(v, "output_tokens")?; + } + UpstreamProtocol::Gemini => { + u.input_tokens = counter(v, "promptTokenCount")?; + u.cached_input_tokens = counter(v, "cachedContentTokenCount")?; + u.reasoning_output_tokens = counter(v, "thoughtsTokenCount")?; + u.output_tokens = add( + counter(v, "candidatesTokenCount")?, + u.reasoning_output_tokens, + None, + )?; + } + } + Ok(u) +} +fn finish_reason( + protocol: UpstreamProtocol, + v: &str, + tools: bool, +) -> Result { + match (protocol, v) { + (UpstreamProtocol::Chat, "stop") + | (UpstreamProtocol::Anthropic, "end_turn" | "stop_sequence") => Ok(FinishReason::Stop), + (UpstreamProtocol::Chat, "tool_calls" | "function_call") + | (UpstreamProtocol::Anthropic, "tool_use") => Ok(FinishReason::ToolCalls), + (UpstreamProtocol::Chat, "length") + | (UpstreamProtocol::Anthropic, "max_tokens") + | (UpstreamProtocol::Gemini, "MAX_TOKENS") => Ok(FinishReason::Length), + (UpstreamProtocol::Chat, "content_filter") + | (UpstreamProtocol::Anthropic, "refusal") + | ( + UpstreamProtocol::Gemini, + "SAFETY" | "RECITATION" | "BLOCKLIST" | "PROHIBITED_CONTENT" | "SPII" | "IMAGE_SAFETY", + ) => Ok(FinishReason::ContentFilter), + (UpstreamProtocol::Gemini, "STOP") => Ok(if tools { + FinishReason::ToolCalls + } else { + FinishReason::Stop + }), + _ => Err(RelayError::Protocol("unsupported upstream finish reason")), + } +} +fn plain_response_part(part: &Value) -> Result<(), RelayError> { + if part + .get("annotations") + .is_some_and(|a| !a.is_null() && a.as_array().is_none_or(|a| !a.is_empty())) + { + return Err(RelayError::Protocol( + "annotated Responses output is not supported", + )); + } + Ok(()) +} +fn plain_response_item(item: &Value) -> Result<(), RelayError> { + if item.get("encrypted_content").is_some_and(|v| !v.is_null()) { + return Err(RelayError::Protocol( + "encrypted Responses output is not supported", + )); + } + if let Some(parts) = item.get("content").filter(|v| !v.is_null()) { + for part in parts + .as_array() + .ok_or(RelayError::Protocol("invalid Responses content"))? + { + plain_response_part(part)?; + } + } + Ok(()) +} +fn plain_response(response: &Value) -> Result<(), RelayError> { + if let Some(items) = response.get("output").filter(|v| !v.is_null()) { + for item in items + .as_array() + .ok_or(RelayError::Protocol("invalid Responses output"))? + { + plain_response_item(item)?; + } + } + Ok(()) +} + +impl Decoder { + pub fn new(protocol: UpstreamProtocol, limits: &Limits) -> Self { + Self { + protocol, + started: false, + source_id: None, + done: false, + finish: None, + blocks: BTreeMap::new(), + max_blocks: limits.max_blocks, + gemini_calls: 0, + } + } + pub fn is_done(&self) -> bool { + self.done + } + fn start(&mut self, id: Option<&str>, out: &mut Vec) -> Result<(), RelayError> { + if let Some(id) = id { + if id.is_empty() || id.len() > 256 || id.chars().any(char::is_control) { + return Err(RelayError::Protocol("invalid upstream ID")); + } + if self.source_id.as_deref().is_some_and(|old| old != id) { + return Err(RelayError::Protocol("upstream response ID changed")); + } + self.source_id = Some(id.to_owned()); + } + if !self.started { + self.started = true; + out.push(Event::Start(self.source_id.clone().unwrap_or_default())); + } + Ok(()) + } + fn tools(&self) -> bool { + self.blocks + .values() + .any(|s| matches!(s.kind, BlockKind::Tool { .. })) + } + fn block( + &mut self, + key: Key, + kind: BlockKind, + out: &mut Vec, + ) -> Result { + if self.finish.is_some() { + return Err(RelayError::Protocol("content after finish")); + } + if let Some(s) = self.blocks.get(&key) { + let compatible = match (&s.kind, &kind) { + (BlockKind::Text, BlockKind::Text) + | (BlockKind::Thinking, BlockKind::Thinking) + | (BlockKind::Refusal, BlockKind::Refusal) => true, + (BlockKind::Tool { id: a, name: b }, BlockKind::Tool { id: c, name: d }) => { + a == c && b == d + } + _ => false, + }; + if s.ended || !compatible { + return Err(RelayError::Protocol("invalid block transition")); + } + return Ok(s.index); + } + if self.blocks.len() >= self.max_blocks { + return Err(RelayError::Limit("output block count")); + } + if let BlockKind::Tool { id, name } = &kind { + if id.is_empty() || name.is_empty() || id.len() > 256 || name.len() > 128 { + return Err(RelayError::Protocol("invalid tool identity")); + } + if self + .blocks + .values() + .any(|s| matches!(&s.kind, BlockKind::Tool { id: old, .. } if old == id)) + { + return Err(RelayError::Protocol("duplicate upstream tool ID")); + } + } + let index = self.blocks.len(); + out.push(Event::Block { + index, + kind: kind.clone(), + }); + self.blocks.insert( + key, + Slot { + index, + kind, + ended: false, + has_text: false, + default_object: false, + }, + ); + Ok(index) + } + fn delta(&mut self, key: Key, value: &str, out: &mut Vec) -> Result<(), RelayError> { + let slot = self + .blocks + .get_mut(&key) + .ok_or(RelayError::Protocol("delta without a block"))?; + if slot.ended || self.finish.is_some() { + return Err(RelayError::Protocol("delta after block end")); + } + if !value.is_empty() { + slot.has_text = true; + out.push(Event::Delta { + index: slot.index, + text: value.to_owned(), + }); + } + Ok(()) + } + fn simple( + &mut self, + key: Key, + kind: BlockKind, + value: &str, + out: &mut Vec, + ) -> Result<(), RelayError> { + self.block(key, kind, out)?; + self.delta(key, value, out) + } + fn signature(&self, key: Key, value: &str, out: &mut Vec) -> Result<(), RelayError> { + let s = self + .blocks + .get(&key) + .ok_or(RelayError::Protocol("signature without a block"))?; + if s.ended { + return Err(RelayError::Protocol("signature after block end")); + } + out.push(Event::Signature { + index: s.index, + provider: self.protocol, + value: value.to_owned(), + }); + Ok(()) + } + fn close(&mut self, key: Key, out: &mut Vec) -> Result<(), RelayError> { + let s = self + .blocks + .get_mut(&key) + .ok_or(RelayError::Protocol("unknown block end"))?; + if !s.ended { + if s.default_object && !s.has_text { + out.push(Event::Delta { + index: s.index, + text: "{}".into(), + }); + } + s.ended = true; + out.push(Event::EndBlock(s.index)); + } + Ok(()) + } + fn close_all(&mut self, out: &mut Vec) -> Result<(), RelayError> { + let keys: Vec<_> = self.blocks.keys().copied().collect(); + for key in keys { + self.close(key, out)?; + } + Ok(()) + } + fn finish(&mut self, reason: FinishReason, out: &mut Vec) -> Result<(), RelayError> { + if self.finish.is_some() { + return Err(RelayError::Protocol("duplicate finish")); + } + self.close_all(out)?; + self.finish = Some(reason); + out.push(Event::Finish(reason)); + Ok(()) + } + fn done(&mut self, out: &mut Vec) -> Result<(), RelayError> { + if self.finish.is_none() || self.done { + return Err(RelayError::Truncated); + } + self.done = true; + out.push(Event::Done); + Ok(()) + } + pub fn frame(&mut self, frame: Frame) -> Result, RelayError> { + if self.done { + return Err(RelayError::Protocol("event after terminal event")); + } + let mut out = Vec::new(); + if frame.data.is_empty() { + return Ok(out); + } + if frame.data == "[DONE]" { + if self.protocol != UpstreamProtocol::Chat { + return Err(RelayError::Protocol("unexpected DONE marker")); + } + self.done(&mut out)?; + return Ok(out); + } + let v: Value = serde_json::from_str(&frame.data) + .map_err(|_| RelayError::Protocol("invalid SSE JSON"))?; + if !v.is_object() { + return Err(RelayError::Protocol("upstream event is not an object")); + } + if v.get("error").is_some_and(|e| !e.is_null()) { + return Err(RelayError::Protocol("upstream reported an error")); + } + if let (Some(wire), Some(kind)) = (frame.event.as_deref(), v["type"].as_str()) + && wire != "message" + && wire != kind + { + return Err(RelayError::Protocol("SSE event type mismatch")); + } + match self.protocol { + UpstreamProtocol::Chat => self.chat(&v, false, &mut out)?, + UpstreamProtocol::Responses => self.responses_event(&v, &mut out)?, + UpstreamProtocol::Anthropic => self.anthropic_event(&v, &mut out)?, + UpstreamProtocol::Gemini => self.gemini(&v, &mut out)?, + } + Ok(out) + } + pub fn eof(&mut self) -> Result, RelayError> { + let mut out = Vec::new(); + // Gemini ends with HTTP EOF, but still must have a semantic finish reason. + if !self.done && self.protocol == UpstreamProtocol::Gemini { + self.done(&mut out)?; + } + if !self.done { + return Err(RelayError::Truncated); + } + Ok(out) + } + pub fn json(&mut self, v: &Value) -> Result, RelayError> { + if !v.is_object() || v.get("error").is_some_and(|e| !e.is_null()) { + return Err(RelayError::Protocol("invalid upstream JSON response")); + } + let mut out = Vec::new(); + match self.protocol { + UpstreamProtocol::Chat => self.chat(v, true, &mut out)?, + UpstreamProtocol::Responses => { + plain_response(v)?; + self.start(v["id"].as_str(), &mut out)?; + for (i, item) in v["output"] + .as_array() + .ok_or(RelayError::Protocol("missing Responses output"))? + .iter() + .enumerate() + { + let i = i as u64; + match item["type"].as_str() { + Some("function_call") => { + self.block( + (i, u64::MAX), + BlockKind::Tool { + id: text(&item["call_id"])?.into(), + name: text(&item["name"])?.into(), + }, + &mut out, + )?; + self.delta((i, u64::MAX), text(&item["arguments"])?, &mut out)?; + } + Some("message") => { + for (j, p) in item["content"] + .as_array() + .ok_or(RelayError::Protocol("invalid Responses content"))? + .iter() + .enumerate() + { + let (kind, s) = match p["type"].as_str() { + Some("output_text") => (BlockKind::Text, text(&p["text"])?), + Some("refusal") => (BlockKind::Refusal, text(&p["refusal"])?), + _ => { + return Err(RelayError::Protocol( + "unsupported output part", + )); + } + }; + self.simple((i, j as u64), kind, s, &mut out)?; + } + } + Some("reasoning") => { + for (j, p) in item["summary"] + .as_array() + .ok_or(RelayError::Protocol("invalid reasoning summary"))? + .iter() + .enumerate() + { + self.simple( + (i, j as u64), + BlockKind::Thinking, + text(&p["text"])?, + &mut out, + )?; + } + if item.get("encrypted_content").is_some_and(|e| !e.is_null()) { + return Err(RelayError::Protocol( + "encrypted Responses output is not supported", + )); + } + } + _ => return Err(RelayError::Protocol("unsupported Responses output item")), + } + } + self.response_finish(v, &mut out)?; + } + UpstreamProtocol::Anthropic => { + self.start(v["id"].as_str(), &mut out)?; + for (i, b) in v["content"] + .as_array() + .ok_or(RelayError::Protocol("invalid Anthropic content"))? + .iter() + .enumerate() + { + self.anthropic_block((0, i as u64), b, false, &mut out)?; + } + out.push(Event::Usage(usage(self.protocol, &v["usage"])?)); + self.finish( + finish_reason(self.protocol, text(&v["stop_reason"])?, self.tools())?, + &mut out, + )?; + } + UpstreamProtocol::Gemini => self.gemini(v, &mut out)?, + } + self.done(&mut out)?; + Ok(out) + } + fn chat(&mut self, v: &Value, full: bool, out: &mut Vec) -> Result<(), RelayError> { + self.start(v["id"].as_str(), out)?; + let choices = v["choices"] + .as_array() + .ok_or(RelayError::Protocol("missing choices"))?; + if choices.len() > 1 || (full && choices.len() != 1) { + return Err(RelayError::Protocol( + "only one upstream candidate is supported", + )); + } + for c in choices { + if c.get("index").is_some_and(|i| i.as_u64() != Some(0)) { + return Err(RelayError::Protocol("invalid candidate index")); + } + let d = if full { &c["message"] } else { &c["delta"] }; + if !d.is_object() { + return Err(RelayError::Protocol("missing message delta")); + } + for (field, key, kind) in [ + ("reasoning_content", (1, 0), BlockKind::Thinking), + ("content", (0, 0), BlockKind::Text), + ("refusal", (2, 0), BlockKind::Refusal), + ] { + if let Some(s) = d.get(field).filter(|s| !s.is_null()) { + self.simple(key, kind, text(s)?, out)?; + } + } + if let Some(calls) = d.get("tool_calls").filter(|v| !v.is_null()) { + for (ordinal, c) in calls + .as_array() + .ok_or(RelayError::Protocol("invalid tool calls"))? + .iter() + .enumerate() + { + let key = ( + 3, + if full { + ordinal as u64 + } else { + index(&c["index"])? + }, + ); + if !self.blocks.contains_key(&key) { + self.block( + key, + BlockKind::Tool { + id: text(&c["id"])?.into(), + name: text(&c["function"]["name"])?.into(), + }, + out, + )?; + } else if let Some(Slot { + kind: BlockKind::Tool { id, name }, + .. + }) = self.blocks.get(&key) + && (c + .get("id") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty() && s != id) + || c["function"] + .get("name") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty() && s != name)) + { + return Err(RelayError::Protocol("tool identity changed")); + } + if let Some(a) = c["function"].get("arguments").filter(|a| !a.is_null()) { + self.delta(key, text(a)?, out)?; + } + } + } + if d.get("function_call").is_some() { + return Err(RelayError::Protocol( + "legacy function_call output is unsupported", + )); + } + if let Some(reason) = c.get("finish_reason").filter(|r| !r.is_null()) { + self.finish( + finish_reason(self.protocol, text(reason)?, self.tools())?, + out, + )?; + } + } + if let Some(u) = v.get("usage").filter(|u| !u.is_null()) { + out.push(Event::Usage(usage(self.protocol, u)?)); + } + Ok(()) + } + fn anthropic_block( + &mut self, + key: Key, + b: &Value, + stream: bool, + out: &mut Vec, + ) -> Result<(), RelayError> { + match b["type"].as_str() { + Some("text") => self.simple(key, BlockKind::Text, text(&b["text"])?, out)?, + Some("thinking") => { + self.simple(key, BlockKind::Thinking, text(&b["thinking"])?, out)?; + if let Some(s) = b.get("signature") { + self.signature(key, text(s)?, out)?; + } + } + Some("tool_use") => { + self.block( + key, + BlockKind::Tool { + id: text(&b["id"])?.into(), + name: text(&b["name"])?.into(), + }, + out, + )?; + let input = b["input"] + .as_object() + .ok_or(RelayError::Protocol("invalid tool input"))?; + if stream && input.is_empty() { + if let Some(s) = self.blocks.get_mut(&key) { + s.default_object = true; + } + } else { + self.delta(key, &b["input"].to_string(), out)?; + } + } + _ => return Err(RelayError::Protocol("unsupported Anthropic content block")), + } + Ok(()) + } + fn anthropic_event(&mut self, v: &Value, out: &mut Vec) -> Result<(), RelayError> { + match v["type"].as_str() { + Some("ping") => {} + Some("message_start") => { + if self.started { + return Err(RelayError::Protocol("duplicate message start")); + } + self.start(v["message"]["id"].as_str(), out)?; + out.push(Event::Usage(usage(self.protocol, &v["message"]["usage"])?)); + } + Some("content_block_start") => { + if !self.started { + return Err(RelayError::Protocol("content before message start")); + } + let key = (0, index(&v["index"])?); + if self.blocks.contains_key(&key) { + return Err(RelayError::Protocol("duplicate content block")); + } + self.anthropic_block(key, &v["content_block"], true, out)?; + } + Some("content_block_delta") => { + let key = (0, index(&v["index"])?); + let d = &v["delta"]; + let kind = &self + .blocks + .get(&key) + .ok_or(RelayError::Protocol("unknown delta block"))? + .kind; + match (d["type"].as_str(), kind) { + (Some("text_delta"), BlockKind::Text) => { + self.delta(key, text(&d["text"])?, out)? + } + (Some("thinking_delta"), BlockKind::Thinking) => { + self.delta(key, text(&d["thinking"])?, out)? + } + (Some("input_json_delta"), BlockKind::Tool { .. }) => { + self.delta(key, text(&d["partial_json"])?, out)? + } + (Some("signature_delta"), BlockKind::Thinking) => { + self.signature(key, text(&d["signature"])?, out)? + } + _ => return Err(RelayError::Protocol("unsupported Anthropic delta")), + } + } + Some("content_block_stop") => self.close((0, index(&v["index"])?), out)?, + Some("message_delta") => { + if let Some(u) = v.get("usage") { + out.push(Event::Usage(usage(self.protocol, u)?)); + } + if let Some(r) = v["delta"].get("stop_reason").filter(|r| !r.is_null()) { + self.finish(finish_reason(self.protocol, text(r)?, self.tools())?, out)?; + } + } + Some("message_stop") => self.done(out)?, + Some("error") => return Err(RelayError::Protocol("upstream stream failed")), + _ => return Err(RelayError::Protocol("unsupported Anthropic event")), + } + Ok(()) + } + fn responses_event(&mut self, v: &Value, out: &mut Vec) -> Result<(), RelayError> { + if let Some(item) = v.get("item") { + plain_response_item(item)?; + } + if let Some(part) = v.get("part") { + plain_response_part(part)?; + } + if let Some(response) = v.get("response") { + plain_response(response)?; + } + match v["type"].as_str() { + Some("response.created" | "response.in_progress") => { + self.start(v["response"]["id"].as_str(), out)? + } + Some("response.output_item.added") => { + if !self.started { + return Err(RelayError::Protocol("output before response start")); + } + let i = index(&v["output_index"])?; + let item = &v["item"]; + match item["type"].as_str() { + Some("function_call") => { + let key = (i, u64::MAX); + self.block( + key, + BlockKind::Tool { + id: text(&item["call_id"])?.into(), + name: text(&item["name"])?.into(), + }, + out, + )?; + if let Some(a) = item.get("arguments") { + self.delta(key, text(a)?, out)?; + } + } + Some("message" | "reasoning") => {} + _ => return Err(RelayError::Protocol("unsupported Responses item")), + } + } + Some("response.content_part.added") => { + let key = (index(&v["output_index"])?, index(&v["content_index"])?); + let p = &v["part"]; + let (kind, s) = match p["type"].as_str() { + Some("output_text") => (BlockKind::Text, text(&p["text"])?), + Some("refusal") => (BlockKind::Refusal, text(&p["refusal"])?), + _ => return Err(RelayError::Protocol("unsupported response content")), + }; + self.simple(key, kind, s, out)?; + } + Some("response.reasoning_summary_part.added") => { + let key = (index(&v["output_index"])?, index(&v["summary_index"])?); + self.simple(key, BlockKind::Thinking, text(&v["part"]["text"])?, out)?; + } + Some("response.output_text.delta" | "response.refusal.delta") => { + self.delta( + (index(&v["output_index"])?, index(&v["content_index"])?), + text(&v["delta"])?, + out, + )?; + } + Some("response.reasoning_summary_text.delta") => self.delta( + (index(&v["output_index"])?, index(&v["summary_index"])?), + text(&v["delta"])?, + out, + )?, + Some("response.function_call_arguments.delta") => self.delta( + (index(&v["output_index"])?, u64::MAX), + text(&v["delta"])?, + out, + )?, + Some("response.function_call_arguments.done") => { + let key = (index(&v["output_index"])?, u64::MAX); + if self.blocks.get(&key).is_some_and(|s| !s.has_text) { + self.delta(key, text(&v["arguments"])?, out)?; + } + } + Some("response.content_part.done") => self.close( + (index(&v["output_index"])?, index(&v["content_index"])?), + out, + )?, + Some("response.reasoning_summary_part.done") => self.close( + (index(&v["output_index"])?, index(&v["summary_index"])?), + out, + )?, + Some("response.output_item.done") => { + let i = index(&v["output_index"])?; + let keys: Vec<_> = self + .blocks + .keys() + .filter(|(a, _)| *a == i) + .copied() + .collect(); + for k in keys { + self.close(k, out)?; + } + } + Some( + "response.output_text.done" + | "response.refusal.done" + | "response.reasoning_summary_text.done", + ) => {} + Some("response.completed" | "response.incomplete") => { + self.response_finish(&v["response"], out)?; + self.done(out)?; + } + Some("response.failed" | "error") => { + return Err(RelayError::Protocol("Responses stream failed")); + } + _ => return Err(RelayError::Protocol("unsupported Responses event")), + } + Ok(()) + } + fn response_finish(&mut self, v: &Value, out: &mut Vec) -> Result<(), RelayError> { + out.push(Event::Usage(usage(self.protocol, &v["usage"])?)); + let reason = match v["status"].as_str() { + Some("completed") => { + if self.tools() { + FinishReason::ToolCalls + } else { + FinishReason::Stop + } + } + Some("incomplete") => match v["incomplete_details"]["reason"].as_str() { + Some("max_output_tokens") => FinishReason::Length, + Some("content_filter") => FinishReason::ContentFilter, + _ => return Err(RelayError::Protocol("unknown incomplete response")), + }, + _ => return Err(RelayError::Protocol("upstream response is not complete")), + }; + self.finish(reason, out) + } + fn gemini(&mut self, v: &Value, out: &mut Vec) -> Result<(), RelayError> { + self.start(v["responseId"].as_str(), out)?; + if let Some(u) = v.get("usageMetadata") { + out.push(Event::Usage(usage(self.protocol, u)?)); + } + if v["promptFeedback"].get("blockReason").is_some() { + self.finish(FinishReason::ContentFilter, out)?; + return Ok(()); + } + let Some(candidates) = v.get("candidates") else { + return Ok(()); + }; + let candidates = candidates + .as_array() + .ok_or(RelayError::Protocol("invalid Gemini candidates"))?; + if candidates.len() > 1 { + return Err(RelayError::Protocol("multiple Gemini candidates")); + } + for c in candidates { + if c.get("index").is_some_and(|i| i.as_u64() != Some(0)) { + return Err(RelayError::Protocol("invalid Gemini candidate index")); + } + if let Some(parts) = c["content"].get("parts") { + for p in parts + .as_array() + .ok_or(RelayError::Protocol("invalid Gemini parts"))? + { + let key = if let Some(s) = p.get("text") { + let thought = p.get("thought").and_then(Value::as_bool).unwrap_or(false); + let key = (u64::from(thought), 0); + self.simple( + key, + if thought { + BlockKind::Thinking + } else { + BlockKind::Text + }, + text(s)?, + out, + )?; + key + } else if let Some(f) = p.get("functionCall") { + let key = (2, self.gemini_calls); + self.gemini_calls += 1; + let id = f["id"] + .as_str() + .map(str::to_owned) + .unwrap_or_else(|| format!("call_gemini_{}", key.1)); + self.block( + key, + BlockKind::Tool { + id, + name: text(&f["name"])?.into(), + }, + out, + )?; + if !f["args"].is_object() { + return Err(RelayError::Protocol("invalid Gemini function arguments")); + } + self.delta(key, &f["args"].to_string(), out)?; + key + } else { + return Err(RelayError::Protocol("unsupported Gemini part")); + }; + if let Some(s) = p.get("thoughtSignature") { + self.signature(key, text(s)?, out)?; + } + } + } + if let Some(r) = c.get("finishReason") { + self.finish(finish_reason(self.protocol, text(r)?, self.tools())?, out)?; + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn usage_is_not_zero_when_absent_and_cache_is_normalized() { + assert_eq!( + usage(UpstreamProtocol::Chat, &Value::Null).unwrap(), + Usage::default() + ); + let u = usage(UpstreamProtocol::Anthropic,&serde_json::json!({"input_tokens":10,"cache_read_input_tokens":20,"cache_creation_input_tokens":30,"output_tokens":4})).unwrap(); + assert_eq!(u.input_tokens, Some(60)); + assert_eq!(u.output_tokens, Some(4)); + let g = usage(UpstreamProtocol::Gemini,&serde_json::json!({"promptTokenCount":5,"candidatesTokenCount":7,"thoughtsTokenCount":3})).unwrap(); + assert_eq!(g.output_tokens, Some(10)); + assert_eq!(g.reasoning_output_tokens, Some(3)); + } + #[test] + fn usage_overflow_and_negative_counters_are_errors() { + assert!( + usage( + UpstreamProtocol::Chat, + &serde_json::json!({"prompt_tokens":-1}) + ) + .is_err() + ); + assert!( + usage( + UpstreamProtocol::Anthropic, + &serde_json::json!({"input_tokens":u64::MAX,"cache_read_input_tokens":1}) + ) + .is_err() + ); + } +} diff --git a/apps/core-rust/src/relay/encode.rs b/apps/core-rust/src/relay/encode.rs new file mode 100644 index 000000000..9e54b1646 --- /dev/null +++ b/apps/core-rust/src/relay/encode.rs @@ -0,0 +1,532 @@ +//! OpenAI client encoders. Successful terminal events are withheld until settlement. +use super::{ + BlockKind, ClientProtocol, Event, FinishReason, Limits, RelayError, Request, UpstreamProtocol, + Usage, +}; +use serde_json::{Value, json}; + +struct Block { + kind: BlockKind, + text: String, + signature: Option<(UpstreamProtocol, String)>, + ended: bool, + tool_index: usize, +} +pub(crate) struct Encoder { + client: ClientProtocol, + stream: bool, + include_usage: bool, + id: String, + model: String, + created: u64, + sequence: u64, + started: bool, + done: bool, + blocks: Vec, + usage: Usage, + finish: Option, + bytes: usize, + limits: Limits, +} +impl Encoder { + pub fn new(request: &Request, request_id: &str, limits: &Limits) -> Self { + let prefix = match request.client { + ClientProtocol::Chat => "chatcmpl", + ClientProtocol::Responses => "resp", + }; + Self { + client: request.client, + stream: request.stream, + include_usage: request.include_usage, + id: format!("{prefix}_{request_id}"), + model: request.model.clone(), + created: std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + sequence: 0, + started: false, + done: false, + blocks: Vec::new(), + usage: Usage::default(), + finish: None, + bytes: 0, + limits: limits.clone(), + } + } + pub fn usage(&self) -> Usage { + self.usage + } + pub fn finish_reason(&self) -> Option { + self.finish + } + fn account(&mut self, bytes: usize) -> Result<(), RelayError> { + self.bytes = self + .bytes + .checked_add(bytes) + .ok_or(RelayError::Limit("aggregate output"))?; + if self.bytes > self.limits.output_bytes { + return Err(RelayError::Limit("aggregate output")); + } + Ok(()) + } + fn serialize(&self, value: &Value) -> Result, RelayError> { + let data = serde_json::to_vec(value) + .map_err(|_| RelayError::Protocol("response encoding failed"))?; + // JSON may escape each input byte into six bytes. Structure overhead is + // bounded separately by max_blocks. This is not a second unbounded cache. + let max = self + .limits + .output_bytes + .saturating_mul(8) + .saturating_add(self.limits.max_blocks.saturating_mul(4096)); + if data.len() > max { + return Err(RelayError::Limit("encoded response")); + } + Ok(data) + } + fn packet(&self, event: Option<&str>, value: &Value) -> Result, RelayError> { + let mut out = Vec::new(); + if let Some(name) = event { + out.extend_from_slice(b"event: "); + out.extend_from_slice(name.as_bytes()); + out.push(b'\n'); + } + out.extend_from_slice(b"data: "); + out.extend(self.serialize(value)?); + out.extend_from_slice(b"\n\n"); + Ok(out) + } + fn response_event(&mut self, kind: &str, mut v: Value) -> Result, RelayError> { + v["type"] = json!(kind); + v["sequence_number"] = json!(self.sequence); + self.sequence = self + .sequence + .checked_add(1) + .ok_or(RelayError::Limit("event sequence"))?; + self.packet(Some(kind), &v) + } + fn chat_chunk( + &self, + delta: Value, + finish: Option, + ) -> Result, RelayError> { + self.packet(None,&json!({"id":self.id,"object":"chat.completion.chunk","created":self.created,"model":self.model, + "choices":[{"index":0,"delta":delta,"finish_reason":finish.map(FinishReason::chat)}]})) + } + fn block_id(&self, index: usize) -> String { + let prefix = match self.blocks[index].kind { + BlockKind::Tool { .. } => "fc", + BlockKind::Thinking => "rs", + _ => "msg", + }; + format!("{prefix}_{}_{}", self.id, index) + } + fn part(&self, index: usize, empty: bool) -> Value { + let b = &self.blocks[index]; + let text = if empty { "" } else { &b.text }; + match b.kind { + BlockKind::Text => json!({"type":"output_text","text":text,"annotations":[]}), + BlockKind::Refusal => json!({"type":"refusal","refusal":text}), + BlockKind::Thinking => json!({"type":"summary_text","text":text}), + BlockKind::Tool { .. } => Value::Null, + } + } + fn item(&self, index: usize, empty: bool) -> Value { + let b = &self.blocks[index]; + let text = if empty { "" } else { &b.text }; + let status = if empty { + "in_progress" + } else if matches!( + self.finish, + Some(FinishReason::Length | FinishReason::ContentFilter) + ) { + "incomplete" + } else { + "completed" + }; + let mut item = match &b.kind { + BlockKind::Tool { id, name } => { + json!({"id":self.block_id(index),"type":"function_call","call_id":id,"name":name,"arguments":text,"status":status}) + } + BlockKind::Thinking => { + json!({"id":self.block_id(index),"type":"reasoning","summary":if empty {vec![]} else {vec![self.part(index,false)]}}) + } + _ => { + json!({"id":self.block_id(index),"type":"message","role":"assistant","status":status, + "content":if empty {vec![]} else {vec![self.part(index,false)]}}) + } + }; + if !empty && let Some((p, s)) = &b.signature { + item["provider_metadata"] = json!({"protocol":p,"signature":s}); + } + item + } + fn response(&self, initial: bool) -> Value { + let incomplete = !initial + && matches!( + self.finish, + Some(FinishReason::Length | FinishReason::ContentFilter) + ); + let status = if initial { + "in_progress" + } else if incomplete { + "incomplete" + } else { + "completed" + }; + json!({"id":self.id,"object":"response","created_at":self.created,"status":status,"model":self.model, + "error":Value::Null,"incomplete_details":if incomplete {json!({"reason":if self.finish == Some(FinishReason::Length) {"max_output_tokens"} else {"content_filter"}})} else {Value::Null}, + "output":if initial {vec![]} else {(0..self.blocks.len()).map(|i|self.item(i,false)).collect::>()}, + "usage":if initial {Value::Null} else {self.usage.wire(ClientProtocol::Responses)},"store":false}) + } + pub fn consume(&mut self, event: Event) -> Result>, RelayError> { + let mut out = Vec::new(); + match event { + Event::Start(source_id) => { + if self.started { + return Err(RelayError::Protocol("duplicate response start")); + } + self.account(source_id.len())?; + self.started = true; + if self.stream { + match self.client { + ClientProtocol::Chat => out + .push(self.chat_chunk(json!({"role":"assistant","content":""}), None)?), + ClientProtocol::Responses => { + out.push(self.response_event( + "response.created", + json!({"response":self.response(true)}), + )?); + out.push(self.response_event( + "response.in_progress", + json!({"response":self.response(true)}), + )?); + } + } + } + } + Event::Block { index, kind } => { + if !self.started || index != self.blocks.len() || index >= self.limits.max_blocks { + return Err(RelayError::Protocol("invalid output block index")); + } + let tool_index = self + .blocks + .iter() + .filter(|b| matches!(b.kind, BlockKind::Tool { .. })) + .count(); + if let BlockKind::Tool { id, name } = &kind { + self.account(id.len() + name.len())?; + } + self.blocks.push(Block { + kind, + text: String::new(), + signature: None, + ended: false, + tool_index, + }); + if self.stream { + match self.client { + ClientProtocol::Chat => { + if let BlockKind::Tool { id, name } = &self.blocks[index].kind { + out.push(self.chat_chunk(json!({"tool_calls":[{"index":tool_index,"id":id,"type":"function","function":{"name":name,"arguments":""}}]}),None)?); + } + } + ClientProtocol::Responses => { + out.push(self.response_event( + "response.output_item.added", + json!({"output_index":index,"item":self.item(index,true)}), + )?); + match self.blocks[index].kind { + BlockKind::Text | BlockKind::Refusal => out.push(self.response_event("response.content_part.added", + json!({"output_index":index,"item_id":self.block_id(index),"content_index":0,"part":self.part(index,true)}))?), + BlockKind::Thinking => out.push(self.response_event("response.reasoning_summary_part.added", + json!({"output_index":index,"item_id":self.block_id(index),"summary_index":0,"part":self.part(index,true)}))?), + BlockKind::Tool { .. } => {}, + } + } + } + } + } + Event::Delta { index, text } => { + self.account(text.len())?; + let b = self + .blocks + .get_mut(index) + .ok_or(RelayError::Protocol("unknown output block"))?; + if b.ended { + return Err(RelayError::Protocol("output after block end")); + } + if matches!(b.kind, BlockKind::Tool { .. }) + && b.text.len().saturating_add(text.len()) > self.limits.tool_argument_bytes + { + return Err(RelayError::Limit("tool arguments")); + } + b.text.push_str(&text); + if self.stream { + match self.client { + ClientProtocol::Chat => { + let delta = match b.kind { + BlockKind::Text => json!({"content":text}), + BlockKind::Refusal => json!({"refusal":text}), + BlockKind::Thinking => json!({"reasoning_content":text}), + BlockKind::Tool { .. } => { + json!({"tool_calls":[{"index":b.tool_index,"function":{"arguments":text}}]}) + } + }; + out.push(self.chat_chunk(delta, None)?); + } + ClientProtocol::Responses => { + let mut v = json!({"output_index":index,"item_id":self.block_id(index),"delta":text}); + let kind = match self.blocks[index].kind { + BlockKind::Text => { + v["content_index"] = json!(0); + "response.output_text.delta" + } + BlockKind::Refusal => { + v["content_index"] = json!(0); + "response.refusal.delta" + } + BlockKind::Thinking => { + v["summary_index"] = json!(0); + "response.reasoning_summary_text.delta" + } + BlockKind::Tool { .. } => "response.function_call_arguments.delta", + }; + out.push(self.response_event(kind, v)?); + } + } + } + } + Event::Signature { + index, + provider, + value, + } => { + self.account(value.len())?; + let b = self + .blocks + .get_mut(index) + .ok_or(RelayError::Protocol("unknown signature block"))?; + if b.ended { + return Err(RelayError::Protocol("signature after output end")); + } + match &mut b.signature { + Some((p, s)) if *p == provider && provider == UpstreamProtocol::Anthropic => { + s.push_str(&value) + } + Some((p, s)) if *p == provider && *s == value => {} + None => b.signature = Some((provider, value.clone())), + _ => return Err(RelayError::Protocol("conflicting provider signature")), + } + if self.stream && self.client == ClientProtocol::Chat { + let delta = if matches!(b.kind, BlockKind::Tool { .. }) { + json!({"tool_calls":[{"index":b.tool_index,"provider_metadata":{"protocol":provider,"signature":value}}]}) + } else if matches!(b.kind, BlockKind::Thinking) { + json!({"reasoning_details":[{"index":index,"type":"signature_delta","protocol":provider,"signature":value}]}) + } else { + json!({"content_details":[{"index":index,"type":"signature_delta","protocol":provider,"signature":value}]}) + }; + out.push(self.chat_chunk(delta, None)?); + } + } + Event::EndBlock(i) => { + let b = self + .blocks + .get_mut(i) + .ok_or(RelayError::Protocol("unknown output end"))?; + b.ended = true; + } + Event::Usage(u) => self.usage.merge(u)?, + Event::Finish(f) => { + if self.finish.replace(f).is_some() { + return Err(RelayError::Protocol("duplicate finish reason")); + } + } + Event::Done => { + if self.finish.is_none() || self.done { + return Err(RelayError::Truncated); + } + self.done = true; + } + } + Ok(out) + } + fn validate_complete(&self) -> Result<(), RelayError> { + if !self.done + || !self.started + || self.finish.is_none() + || self.blocks.iter().any(|b| !b.ended) + { + return Err(RelayError::Truncated); + } + if self.finish == Some(FinishReason::ToolCalls) + && !self + .blocks + .iter() + .any(|b| matches!(b.kind, BlockKind::Tool { .. })) + { + return Err(RelayError::Protocol("tool finish without tools")); + } + if matches!( + self.finish, + Some(FinishReason::Stop | FinishReason::ToolCalls) + ) { + for b in &self.blocks { + if matches!(b.kind, BlockKind::Tool { .. }) { + let v: Value = serde_json::from_str(&b.text) + .map_err(|_| RelayError::Protocol("incomplete tool arguments"))?; + if !v.is_object() { + return Err(RelayError::Protocol("tool arguments are not an object")); + } + } + } + } + Ok(()) + } + fn chat_response(&self) -> Value { + let mut content = String::new(); + let mut reasoning = String::new(); + let mut refusal = String::new(); + let mut calls = Vec::new(); + let mut details = Vec::new(); + let mut content_details = Vec::new(); + for (i, b) in self.blocks.iter().enumerate() { + if matches!(b.kind, BlockKind::Text | BlockKind::Refusal) + && let Some((provider, signature)) = &b.signature + { + content_details.push(json!({"index":i,"protocol":provider,"signature":signature})); + } + match &b.kind { + BlockKind::Text => content.push_str(&b.text), + BlockKind::Refusal => refusal.push_str(&b.text), + BlockKind::Thinking => { + reasoning.push_str(&b.text); + if let Some((p, s)) = &b.signature { + details.push(json!({"index":i,"text":b.text,"protocol":p,"signature":s})); + } + } + BlockKind::Tool { id, name } => { + let mut c = json!({"id":id,"type":"function","function":{"name":name,"arguments":b.text}}); + if let Some((p, s)) = &b.signature { + c["provider_metadata"] = json!({"protocol":p,"signature":s}); + } + calls.push(c); + } + } + } + let mut message = json!({"role":"assistant","content":if content.is_empty() {Value::Null} else {json!(content)}}); + if !reasoning.is_empty() { + message["reasoning_content"] = json!(reasoning); + } + if !refusal.is_empty() { + message["refusal"] = json!(refusal); + } + if !calls.is_empty() { + message["tool_calls"] = json!(calls); + } + if !content_details.is_empty() { + message["content_details"] = json!(content_details); + } + if details.len() == 1 { + message["provider_metadata"] = + json!({"protocol":details[0]["protocol"],"signature":details[0]["signature"]}); + } else if !details.is_empty() { + message["reasoning_details"] = json!(details); + } + json!({"id":self.id,"object":"chat.completion","created":self.created,"model":self.model, + "choices":[{"index":0,"message":message,"finish_reason":self.finish.map(FinishReason::chat)}],"usage":self.usage.wire(ClientProtocol::Chat)}) + } + /// Build (but do not send) a terminal response. Validation must happen before + /// the billing hook. A length-limited tool may contain partial JSON. + pub fn terminal(&mut self) -> Result>, RelayError> { + self.validate_complete()?; + if !self.stream { + return Ok(vec![self.serialize(&match self.client { + ClientProtocol::Chat => self.chat_response(), + ClientProtocol::Responses => self.response(false), + })?]); + } + let mut out = Vec::new(); + match self.client { + ClientProtocol::Chat => { + out.push(self.chat_chunk(json!({}), self.finish)?); + if self.include_usage { + out.push(self.packet(None,&json!({"id":self.id,"object":"chat.completion.chunk","created":self.created, + "model":self.model,"choices":[],"usage":self.usage.wire(ClientProtocol::Chat)}))?); + } + out.push(b"data: [DONE]\n\n".to_vec()); + } + ClientProtocol::Responses => { + for i in 0..self.blocks.len() { + let mut v = json!({"output_index":i,"item_id":self.block_id(i)}); + match &self.blocks[i].kind { + BlockKind::Text | BlockKind::Refusal => { + let refusal = matches!(self.blocks[i].kind, BlockKind::Refusal); + v["content_index"] = json!(0); + v[if refusal { "refusal" } else { "text" }] = + json!(self.blocks[i].text); + out.push(self.response_event( + if refusal { + "response.refusal.done" + } else { + "response.output_text.done" + }, + v.clone(), + )?); + v.as_object_mut() + .ok_or(RelayError::Protocol("response encoding failed"))? + .remove(if refusal { "refusal" } else { "text" }); + v["part"] = self.part(i, false); + out.push(self.response_event("response.content_part.done", v)?); + } + BlockKind::Thinking => { + v["summary_index"] = json!(0); + v["text"] = json!(self.blocks[i].text); + out.push(self.response_event( + "response.reasoning_summary_text.done", + v.clone(), + )?); + v.as_object_mut() + .ok_or(RelayError::Protocol("response encoding failed"))? + .remove("text"); + v["part"] = self.part(i, false); + out.push( + self.response_event("response.reasoning_summary_part.done", v)?, + ); + } + BlockKind::Tool { .. } => { + v["arguments"] = json!(self.blocks[i].text); + out.push( + self.response_event("response.function_call_arguments.done", v)?, + ); + } + } + out.push(self.response_event( + "response.output_item.done", + json!({"output_index":i,"item":self.item(i,false)}), + )?); + } + let kind = if matches!( + self.finish, + Some(FinishReason::Length | FinishReason::ContentFilter) + ) { + "response.incomplete" + } else { + "response.completed" + }; + out.push(self.response_event(kind, json!({"response":self.response(false)}))?); + } + } + Ok(out) + } + pub fn error_packet(&mut self, error: &RelayError) -> Result, RelayError> { + match self.client { + ClientProtocol::Chat => self.packet(None, &error.json()), + ClientProtocol::Responses => self.response_event( + "error", + json!({"code":error.code(),"message":error.to_string(),"param":Value::Null}), + ), + } + } +} diff --git a/apps/core-rust/src/relay/mod.rs b/apps/core-rust/src/relay/mod.rs new file mode 100644 index 000000000..4c1a5d25e --- /dev/null +++ b/apps/core-rust/src/relay/mod.rs @@ -0,0 +1,712 @@ +//! Rust-owned model forwarding. There is no Go client or database connection here. +//! +//! This module does not register public routes or change readiness. Integration +//! must supply real core routing and durable billing adapters before enabling it. +mod decode; +mod encode; +mod request; +mod sse; +#[cfg(test)] +mod tests; +mod types; +pub use request::{FunctionTool, InputBlock, Message, Request, Role, ToolChoice}; +pub use types::*; + +use axum::{ + body::{Body, Bytes}, + http::{Response, header}, +}; +use decode::Decoder; +use encode::Encoder; +use std::{ + collections::BTreeMap, + future::Future, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll}, + time::Duration, +}; +use tokio::{ + sync::{Notify, OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch}, + time::{Instant, timeout, timeout_at}, +}; +use tokio_stream::Stream; + +#[derive(Clone)] +pub struct Cancellation(watch::Sender); +impl Default for Cancellation { + fn default() -> Self { + Self(watch::channel(false).0) + } +} +impl Cancellation { + pub fn cancel(&self) { + self.0.send_replace(true); + } + pub fn is_cancelled(&self) -> bool { + *self.0.borrow() + } + async fn cancelled(&self) { + let mut rx = self.0.subscribe(); + loop { + if *rx.borrow_and_update() { + return; + } + if rx.changed().await.is_err() { + return; + } + } + } +} + +/// Dropping the HTTP body cancels the upstream, but not the billing finalizer. +pub struct RelayBody { + receiver: mpsc::Receiver>, + cancel: Cancellation, +} +impl RelayBody { + pub async fn next_chunk(&mut self) -> Option> { + self.receiver.recv().await + } +} +impl Stream for RelayBody { + type Item = Result; + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.get_mut().receiver.poll_recv(cx) + } +} +impl Drop for RelayBody { + fn drop(&mut self) { + self.cancel.cancel(); + } +} + +pub struct Session { + pub head: oneshot::Receiver>, + pub body: RelayBody, + pub completion: oneshot::Receiver, + pub cancel: Cancellation, +} +impl Session { + /// A response helper, not a registered public HTTP endpoint. The integration + /// layer must retain completion records or otherwise observe reconciliation. + pub async fn into_http( + self, + ) -> Result<(Response, oneshot::Receiver), RelayError> { + let head = self + .head + .await + .map_err(|_| RelayError::SettlementPending)??; + let response = Response::builder() + .status(head.status) + .header(header::CONTENT_TYPE, head.content_type) + .header(header::CACHE_CONTROL, "no-store") + .header("x-accel-buffering", "no") + .body(Body::from_stream(self.body)) + .map_err(|_| RelayError::Invalid("invalid response head"))?; + Ok((response, self.completion)) + } +} + +struct Registry { + next: u64, + draining: bool, + active: BTreeMap, +} +struct Inner { + client: reqwest::Client, + router: Arc, + billing: Arc, + limits: Limits, + permits: Arc, + registry: Mutex, + changed: Notify, +} +#[derive(Clone)] +pub struct Relay { + inner: Arc, +} +struct Active { + inner: Arc, + id: u64, + _permit: OwnedSemaphorePermit, +} +impl Drop for Active { + fn drop(&mut self) { + self.inner + .registry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active + .remove(&self.id); + self.inner.changed.notify_waiters(); + } +} +impl Relay { + pub fn new( + router: Arc, + billing: Arc, + limits: Limits, + ) -> Result { + limits.validate()?; + let client = reqwest::Client::builder() + .connect_timeout(limits.connect_timeout) + .redirect(reqwest::redirect::Policy::none()) + .retry(reqwest::retry::never()) + .referer(false) + .no_proxy() + .no_gzip() + .no_brotli() + .no_deflate() + .no_zstd() + // No idle connections for an unbounded set of route origins. Active + // connections are bounded by the admission semaphore, including H2. + .pool_max_idle_per_host(0) + .http2_adaptive_window(false) + .http2_initial_stream_window_size(65_536) + .http2_initial_connection_window_size(262_144) + .http2_max_frame_size(16_384) + .http2_max_header_list_size(16_384) + .user_agent("lmm-core-relay/0.1") + .build() + .map_err(|_| RelayError::Invalid("HTTP client initialization failed"))?; + Ok(Self { + inner: Arc::new(Inner { + client, + router, + billing, + permits: Arc::new(Semaphore::new(limits.max_in_flight)), + limits, + registry: Mutex::new(Registry { + next: 0, + draining: false, + active: BTreeMap::new(), + }), + changed: Notify::new(), + }), + }) + } + /// Call Request::parse before this method. No permissive billing adapter is + /// provided; admission cannot send any upstream bytes before reserve succeeds. + pub fn start(&self, request: Request, context: RequestContext) -> Result { + context.validate()?; + let permit = self + .inner + .permits + .clone() + .try_acquire_owned() + .map_err(|e| match e { + tokio::sync::TryAcquireError::Closed => RelayError::Draining, + tokio::sync::TryAcquireError::NoPermits => RelayError::Busy, + })?; + let cancel = Cancellation::default(); + let id = { + let mut registry = self + .inner + .registry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if registry.draining { + return Err(RelayError::Draining); + } + let id = registry.next; + registry.next = registry.next.checked_add(1).ok_or(RelayError::Busy)?; + registry.active.insert(id, cancel.clone()); + id + }; + let active = Active { + inner: self.inner.clone(), + id, + _permit: permit, + }; + let (sender, receiver) = mpsc::channel(self.inner.limits.queue_chunks); + let (head_tx, head) = oneshot::channel(); + let (done_tx, completion) = oneshot::channel(); + let body = RelayBody { + receiver, + cancel: cancel.clone(), + }; + let worker = Worker::new( + self.inner.clone(), + request, + context, + cancel.clone(), + sender, + head_tx, + done_tx, + ); + tokio::spawn(async move { + let _active = active; + worker.run().await; + }); + Ok(Session { + head, + body, + completion, + cancel, + }) + } + pub fn active_requests(&self) -> usize { + self.inner + .registry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .active + .len() + } + async fn wait_empty(&self, until: Instant) -> bool { + loop { + let changed = self.inner.changed.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if self.active_requests() == 0 { + return true; + } + if timeout_at(until, changed).await.is_err() { + return self.active_requests() == 0; + } + } + } + /// Reject new work, let accepted requests finish, then cancel remaining + /// upstreams. Never abort a finalizer. `remaining` must be checked by shutdown. + pub async fn drain(&self, grace: Duration) -> DrainReport { + { + let mut registry = self + .inner + .registry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + registry.draining = true; + self.inner.permits.close(); + } + if self + .wait_empty(Instant::now() + grace.min(Duration::from_secs(86_400))) + .await + { + return DrainReport { + cancelled: 0, + remaining: 0, + }; + } + let cancelled = { + let registry = self + .inner + .registry + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for c in registry.active.values() { + c.cancel(); + } + registry.active.len() + }; + self.wait_empty( + Instant::now() + self.inner.limits.hook_timeout * 2 + Duration::from_secs(1), + ) + .await; + DrainReport { + cancelled, + remaining: self.active_requests(), + } + } +} + +struct Gate { + cancel: Cancellation, + deadline: Instant, +} +impl Gate { + async fn run(&self, duration: Duration, phase: Phase, future: F) -> Result + where + F: Future>, + { + if self.cancel.is_cancelled() { + return Err(RelayError::Cancelled); + } + if Instant::now() >= self.deadline { + return Err(RelayError::Timeout(Phase::Total)); + } + let end = self.deadline.min(Instant::now() + duration); + let timeout_phase = if end == self.deadline { + Phase::Total + } else { + phase + }; + tokio::select! { + biased; + _ = self.cancel.cancelled() => Err(RelayError::Cancelled), + result = timeout_at(end,future) => result.unwrap_or(Err(RelayError::Timeout(timeout_phase))), + } + } +} +struct Worker { + inner: Arc, + request: Request, + context: RequestContext, + gate: Gate, + sender: mpsc::Sender>, + head: Option>>, + done: oneshot::Sender, + encoder: Encoder, + reservation: Option, + reserve_ambiguous: bool, + report: Report, + terminal: Vec>, +} +impl Worker { + fn new( + inner: Arc, + request: Request, + context: RequestContext, + cancel: Cancellation, + sender: mpsc::Sender>, + head: oneshot::Sender>, + done: oneshot::Sender, + ) -> Self { + let report = Report { + request_id: context.request_id.clone(), + route_id: None, + price_version: None, + outcome: Outcome::Failed, + usage: Usage::default(), + finish_reason: None, + upstream_attempted: false, + attempts: 0, + upstream_status: None, + upstream_bytes: 0, + emitted_bytes: 0, + }; + let encoder = Encoder::new(&request, &context.request_id, &inner.limits); + let deadline = Instant::now() + inner.limits.total_timeout; + Self { + inner, + request, + context, + gate: Gate { cancel, deadline }, + sender, + head: Some(head), + done, + encoder, + reservation: None, + reserve_ambiguous: false, + report, + terminal: Vec::new(), + } + } + fn send_head(&mut self) -> Result<(), RelayError> { + if let Some(head) = self.head.take() { + head.send(Ok(ResponseHead { + status: 200, + content_type: if self.request.stream { + "text/event-stream; charset=utf-8" + } else { + "application/json" + }, + })) + .map_err(|_| RelayError::Cancelled)?; + } + Ok(()) + } + async fn write(&mut self, bytes: &[u8]) -> Result<(), RelayError> { + for chunk in bytes.chunks(self.inner.limits.chunk_bytes) { + self.gate + .run(self.inner.limits.write_timeout, Phase::Write, async { + self.sender + .send(Ok(Bytes::copy_from_slice(chunk))) + .await + .map_err(|_| RelayError::Cancelled) + }) + .await?; + self.report.emitted_bytes = self.report.emitted_bytes.saturating_add(chunk.len()); + } + Ok(()) + } + async fn events(&mut self, events: Vec) -> Result<(), RelayError> { + for event in events { + for packet in self.encoder.consume(event)? { + self.write(&packet).await?; + } + } + Ok(()) + } + fn count_input(&mut self, chunk: &Bytes) -> Result<(), RelayError> { + if chunk.len() > self.inner.limits.transport_chunk_bytes { + return Err(RelayError::Limit("transport chunk")); + } + self.report.upstream_bytes = self + .report + .upstream_bytes + .checked_add(chunk.len()) + .ok_or(RelayError::Limit("upstream bytes"))?; + if self.report.upstream_bytes > self.inner.limits.upstream_bytes { + return Err(RelayError::Limit("upstream bytes")); + } + Ok(()) + } + async fn open( + &mut self, + route: &Route, + body: Vec, + ) -> Result { + loop { + if self.gate.cancel.is_cancelled() { + return Err(RelayError::Cancelled); + } + if Instant::now() >= self.gate.deadline { + return Err(RelayError::Timeout(Phase::Total)); + } + self.report.attempts += 1; + self.report.upstream_attempted = true; + let request = self + .inner + .client + .post(route.endpoint.clone()) + .header(header::CONTENT_TYPE, "application/json") + .header(header::ACCEPT_ENCODING, "identity") + .header( + header::ACCEPT, + if self.request.stream { + "text/event-stream" + } else { + "application/json" + }, + ) + .body(body.clone()); + let request = match route.protocol { + UpstreamProtocol::Chat | UpstreamProtocol::Responses => { + request.header(header::AUTHORIZATION, route.credential.clone()) + } + UpstreamProtocol::Anthropic => request + .header("x-api-key", route.credential.clone()) + .header("anthropic-version", "2023-06-01"), + UpstreamProtocol::Gemini => { + request.header("x-goog-api-key", route.credential.clone()) + } + }; + let result = self + .gate + .run(self.inner.limits.header_timeout, Phase::Headers, async { + Ok(request.send().await) + }) + .await?; + match result { + Ok(response) => return Ok(response), + Err(e) + if e.is_connect() + && self.report.attempts <= self.inner.limits.connect_retries => + { + self.gate + .run(self.inner.limits.header_timeout, Phase::Connect, async { + tokio::time::sleep(self.inner.limits.retry_backoff).await; + Ok(()) + }) + .await?; + } + Err(e) if e.is_timeout() => return Err(RelayError::Timeout(Phase::Connect)), + Err(_) => return Err(RelayError::UpstreamConnection), + } + } + } + async fn execute(&mut self) -> Result<(), RelayError> { + let route = self + .gate + .run( + self.inner.limits.hook_timeout, + Phase::Route, + self.inner.router.select(&self.context, &self.request), + ) + .await?; + self.report.route_id = Some(route.id.clone()); + self.report.price_version = Some(route.price_version); + let payload = self.request.upstream(&route)?; + let body = serde_json::to_vec(&payload) + .map_err(|_| RelayError::Invalid("request encoding failed"))?; + if body.len() > self.inner.limits.request_bytes { + return Err(RelayError::Limit("encoded request")); + } + // Do not cancel a reserve future just because the client disconnected: + // first obtain its committed handle, then finalize without sending HTTP. + self.reserve_ambiguous = true; + let reservation = timeout( + self.inner.limits.hook_timeout, + self.inner + .billing + .reserve(&self.context, &route, &self.request), + ) + .await + .map_err(|_| RelayError::Timeout(Phase::Reserve))??; + self.reservation = Some(reservation); + self.reserve_ambiguous = false; + let mut response = self.open(&route, body).await?; + self.report.upstream_status = Some(response.status().as_u16()); + if response.status() != reqwest::StatusCode::OK { + return Err(RelayError::UpstreamStatus(response.status().as_u16())); + } + let expected = if self.request.stream { + "text/event-stream" + } else { + "application/json" + }; + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.split(';').next()) + .map(str::trim); + if content_type != Some(expected) { + return Err(RelayError::Protocol("unexpected upstream content type")); + } + if response + .headers() + .get(header::CONTENT_ENCODING) + .is_some_and(|v| v.as_bytes() != b"identity") + { + return Err(RelayError::Protocol( + "compressed upstream bodies are disabled", + )); + } + let max_body = if self.request.stream { + self.inner.limits.upstream_bytes + } else { + self.inner.limits.output_bytes + }; + if response + .content_length() + .is_some_and(|n| n > max_body as u64) + { + return Err(RelayError::Limit("upstream body")); + } + let mut decoder = Decoder::new(route.protocol, &self.inner.limits); + if self.request.stream { + self.send_head()?; + let mut parser = sse::Parser::new(self.inner.limits.event_bytes); + 'read: loop { + let next = self + .gate + .run(self.inner.limits.read_timeout, Phase::Read, async { + response + .chunk() + .await + .map_err(|_| RelayError::UpstreamConnection) + }) + .await?; + let Some(chunk) = next else { + parser.finish()?; + self.events(decoder.eof()?).await?; + break; + }; + self.count_input(&chunk)?; + for byte in chunk { + if let Some(frame) = parser.feed(byte)? { + self.events(decoder.frame(frame)?).await?; + if decoder.is_done() { + break 'read; + } + } + } + } + } else { + let mut data = Vec::new(); + loop { + let next = self + .gate + .run(self.inner.limits.read_timeout, Phase::Read, async { + response + .chunk() + .await + .map_err(|_| RelayError::UpstreamConnection) + }) + .await?; + let Some(chunk) = next else { + break; + }; + self.count_input(&chunk)?; + if data.len().saturating_add(chunk.len()) > max_body { + return Err(RelayError::Limit("non-stream response")); + } + data.extend_from_slice(&chunk); + } + let value = serde_json::from_slice(&data) + .map_err(|_| RelayError::Protocol("invalid upstream JSON"))?; + self.events(decoder.json(&value)?).await?; + } + // Drop the upstream before any money write. Do not drain an unbounded + // body after a terminal marker merely to reuse its connection. + drop(response); + self.terminal = self.encoder.terminal()?; + Ok(()) + } + async fn run(mut self) { + let mut error = self.execute().await.err(); + self.report.usage = self.encoder.usage(); + self.report.finish_reason = self.encoder.finish_reason(); + self.report.outcome = match &error { + None => Outcome::Completed, + Some(RelayError::Cancelled) => Outcome::Cancelled, + Some(RelayError::Timeout(_)) => Outcome::TimedOut, + _ => Outcome::Failed, + }; + let settlement = if let Some(reservation) = &self.reservation { + match timeout( + self.inner.limits.hook_timeout, + self.inner.billing.finalize(reservation, &self.report), + ) + .await + { + Ok(Ok(())) => Settlement::Confirmed, + _ => Settlement::Pending, + } + } else if self.reserve_ambiguous { + Settlement::Pending + } else { + Settlement::NotReserved + }; + if error.is_none() && settlement != Settlement::Confirmed { + error = Some(RelayError::SettlementPending); + } + if error.is_none() { + if let Err(e) = self.send_head() { + error = Some(e); + } + if error.is_none() { + for packet in std::mem::take(&mut self.terminal) { + if let Err(e) = self.write(&packet).await { + error = Some(e); + break; + } + } + } + } + if let Some(e) = &error { + if let Some(head) = self.head.take() { + let _ = head.send(Err(e.clone())); + } else if !self.sender.is_closed() { + let packet = if self.request.stream { + self.encoder.error_packet(e).map(Bytes::from) + } else { + Err(e.clone()) + }; + // Error delivery cannot keep a cancelled or stalled request alive. + let _ = timeout(Duration::from_millis(100), async { + match packet { + Ok(bytes) => { + for chunk in bytes.chunks(self.inner.limits.chunk_bytes) { + self.sender + .send(Ok(Bytes::copy_from_slice(chunk))) + .await + .map_err(|_| ())?; + } + } + Err(error) => self.sender.send(Err(error)).await.map_err(|_| ())?, + } + Ok::<(), ()>(()) + }) + .await; + } + } + let _ = self.done.send(Completion { + report: self.report, + settlement, + error, + }); + } +} diff --git a/apps/core-rust/src/relay/request.rs b/apps/core-rust/src/relay/request.rs new file mode 100644 index 000000000..63539d4fa --- /dev/null +++ b/apps/core-rust/src/relay/request.rs @@ -0,0 +1,903 @@ +//! Bounded, text/function-call request conversion. Unsupported inputs fail closed. +use super::{ClientProtocol, Limits, RelayError, Route, UpstreamProtocol}; +use serde_json::{Map, Value, json}; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone, Debug)] +pub enum InputBlock { + Text(String), + Refusal(String), + Thinking { + text: String, + signature: Option<(UpstreamProtocol, String)>, + }, + ToolCall { + id: String, + name: String, + arguments: String, + signature: Option<(UpstreamProtocol, String)>, + }, + ToolResult { + id: String, + content: String, + }, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Role { + System, + Developer, + User, + Assistant, + Tool, +} +impl Role { + fn parse(v: &Value) -> Result { + match v.as_str() { + Some("system") => Ok(Self::System), + Some("developer") => Ok(Self::Developer), + Some("user") => Ok(Self::User), + Some("assistant") => Ok(Self::Assistant), + Some("tool") => Ok(Self::Tool), + _ => Err(RelayError::Invalid("unsupported message role")), + } + } + fn name(self) -> &'static str { + match self { + Self::System => "system", + Self::Developer => "developer", + Self::User => "user", + Self::Assistant => "assistant", + Self::Tool => "tool", + } + } +} +#[derive(Clone, Debug)] +pub struct Message { + pub role: Role, + pub content: Vec, +} +#[derive(Clone, Debug)] +pub struct FunctionTool { + pub name: String, + pub description: Option, + pub parameters: Value, + pub strict: Option, +} +#[derive(Clone, Debug)] +pub enum ToolChoice { + Auto, + None, + Required, + Named(String), +} +#[derive(Clone, Debug)] +pub struct Request { + pub client: ClientProtocol, + pub model: String, + pub stream: bool, + pub max_output_tokens: u32, + pub messages: Vec, + pub tools: Vec, + pub temperature: Option, + pub top_p: Option, + pub stop: Vec, + pub tool_choice: ToolChoice, + pub parallel_tool_calls: Option, + pub reasoning_effort: Option, + pub(crate) include_usage: bool, +} + +fn object(v: &Value) -> Result<&Map, RelayError> { + v.as_object() + .ok_or(RelayError::Invalid("expected an object")) +} +fn keys(v: &Value, allowed: &[&str]) -> Result<(), RelayError> { + if object(v)?.keys().any(|k| !allowed.contains(&k.as_str())) { + return Err(RelayError::Unsupported( + "request contains an unsupported field", + )); + } + Ok(()) +} +fn string(v: &Value) -> Result { + v.as_str() + .map(str::to_owned) + .ok_or(RelayError::Invalid("expected a string")) +} +fn name(v: &Value) -> Result { + let s = string(v)?; + if s.is_empty() + || s.len() > 128 + || !s + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) + { + return Err(RelayError::Invalid("invalid function name or call ID")); + } + Ok(s) +} +fn boolean(v: Option<&Value>) -> Result, RelayError> { + v.filter(|v| !v.is_null()) + .map(|v| v.as_bool().ok_or(RelayError::Invalid("expected a boolean"))) + .transpose() +} +fn number(v: Option<&Value>, max: f64) -> Result, RelayError> { + v.filter(|v| !v.is_null()) + .map(|v| { + v.as_f64() + .filter(|n| n.is_finite() && *n >= 0.0 && *n <= max) + .ok_or(RelayError::Invalid("invalid sampling value")) + }) + .transpose() +} +fn signature(v: Option<&Value>) -> Result, RelayError> { + let Some(v) = v.filter(|v| !v.is_null()) else { + return Ok(None); + }; + keys(v, &["protocol", "signature"])?; + let protocol = match v["protocol"].as_str() { + Some("anthropic") => UpstreamProtocol::Anthropic, + Some("gemini") => UpstreamProtocol::Gemini, + Some("chat") => UpstreamProtocol::Chat, + Some("responses") => UpstreamProtocol::Responses, + _ => return Err(RelayError::Invalid("invalid signature provider")), + }; + Ok(Some((protocol, string(&v["signature"])?))) +} +fn arguments(v: &Value, limit: usize) -> Result { + let text = string(v)?; + if text.len() > limit { + return Err(RelayError::Limit("tool arguments")); + } + let parsed: Value = serde_json::from_str(&text) + .map_err(|_| RelayError::Invalid("tool arguments must be JSON"))?; + if !parsed.is_object() { + return Err(RelayError::Invalid("tool arguments must be an object")); + } + Ok(text) +} +fn text_blocks(v: &Value, assistant: bool) -> Result, RelayError> { + if v.is_null() { + return Ok(Vec::new()); + } + if let Some(s) = v.as_str() { + return Ok(vec![InputBlock::Text(s.into())]); + } + let parts = v + .as_array() + .ok_or(RelayError::Invalid("invalid message content"))?; + let mut out = Vec::new(); + for p in parts { + match p["type"].as_str() { + Some("text" | "input_text" | "output_text") => { + keys(p, &["type", "text", "annotations"])?; + if p.get("annotations") + .is_some_and(|a| a.as_array().is_none_or(|a| !a.is_empty())) + { + return Err(RelayError::Unsupported("annotated input content")); + } + out.push(InputBlock::Text(string(&p["text"])?)); + } + Some("refusal") if assistant => { + keys(p, &["type", "refusal"])?; + out.push(InputBlock::Refusal(string(&p["refusal"])?)); + } + _ => { + return Err(RelayError::Unsupported( + "only text and function tools are supported", + )); + } + } + } + Ok(out) +} + +impl Request { + pub fn parse(client: ClientProtocol, body: &[u8], limits: &Limits) -> Result { + limits.validate()?; + if body.len() > limits.request_bytes { + return Err(RelayError::Limit("request body")); + } + let v: Value = + serde_json::from_slice(body).map_err(|_| RelayError::Invalid("invalid JSON body"))?; + match client { + ClientProtocol::Chat => keys( + &v, + &[ + "model", + "messages", + "stream", + "stream_options", + "max_tokens", + "max_completion_tokens", + "temperature", + "top_p", + "stop", + "tools", + "tool_choice", + "parallel_tool_calls", + "n", + "reasoning_effort", + "store", + ], + )?, + ClientProtocol::Responses => keys( + &v, + &[ + "model", + "input", + "instructions", + "stream", + "max_output_tokens", + "temperature", + "top_p", + "tools", + "tool_choice", + "parallel_tool_calls", + "reasoning", + "store", + ], + )?, + } + let model = string(&v["model"])?; + if model.is_empty() || model.len() > 256 { + return Err(RelayError::Invalid("invalid model")); + } + if boolean(v.get("store"))? == Some(true) { + return Err(RelayError::Unsupported( + "stored responses are not implemented", + )); + } + if v.get("n").is_some_and(|n| n.as_u64() != Some(1)) { + return Err(RelayError::Unsupported("only one candidate is supported")); + } + let stream = boolean(v.get("stream"))?.unwrap_or(false); + let include_usage = if let Some(opts) = v.get("stream_options").filter(|v| !v.is_null()) { + keys(opts, &["include_usage"])?; + boolean(opts.get("include_usage"))?.unwrap_or(false) + } else { + false + }; + let token_values: Vec<&Value> = + ["max_tokens", "max_completion_tokens", "max_output_tokens"] + .iter() + .filter_map(|k| v.get(k)) + .filter(|v| !v.is_null()) + .collect(); + if token_values.len() > 1 { + return Err(RelayError::Invalid("conflicting output limits")); + } + let max_output_tokens = if let Some(n) = token_values.first() { + n.as_u64() + .and_then(|n| u32::try_from(n).ok()) + .filter(|n| *n > 0 && *n <= limits.max_output_tokens) + .ok_or(RelayError::Invalid("invalid output token limit"))? + } else { + 4096.min(limits.max_output_tokens) + }; + let mut stop = Vec::new(); + if let Some(s) = v.get("stop").filter(|s| !s.is_null()) { + if let Some(s) = s.as_str() { + stop.push(s.to_owned()); + } else { + for s in s + .as_array() + .ok_or(RelayError::Invalid("invalid stop sequences"))? + { + stop.push(string(s)?); + } + } + } + if stop.len() > 4 || stop.iter().any(|s| s.is_empty() || s.len() > 1024) { + return Err(RelayError::Invalid("invalid stop sequences")); + } + let mut tools = Vec::new(); + let mut tool_names = BTreeSet::new(); + if let Some(raw) = v.get("tools").filter(|v| !v.is_null()) { + for t in raw.as_array().ok_or(RelayError::Invalid("invalid tools"))? { + if t["type"] != "function" { + return Err(RelayError::Unsupported("only function tools are supported")); + } + let f = match client { + ClientProtocol::Chat => { + keys(t, &["type", "function"])?; + &t["function"] + } + ClientProtocol::Responses => t, + }; + keys(f, &["type", "name", "description", "parameters", "strict"])?; + let name = name(&f["name"])?; + if !tool_names.insert(name.clone()) { + return Err(RelayError::Invalid("duplicate function name")); + } + let parameters = f + .get("parameters") + .cloned() + .unwrap_or_else(|| json!({"type":"object","properties":{}})); + if !parameters.is_object() { + return Err(RelayError::Invalid("function parameters must be an object")); + } + tools.push(FunctionTool { + name, + parameters, + description: f.get("description").map(string).transpose()?, + strict: boolean(f.get("strict"))?, + }); + } + } + if tools.len() > limits.max_blocks { + return Err(RelayError::Limit("too many tools")); + } + let tool_choice = match v.get("tool_choice").filter(|v| !v.is_null()) { + None => ToolChoice::Auto, + Some(Value::String(s)) => match s.as_str() { + "auto" => ToolChoice::Auto, + "none" => ToolChoice::None, + "required" => ToolChoice::Required, + _ => return Err(RelayError::Unsupported("unsupported tool choice")), + }, + Some(t) => { + if t["type"] != "function" { + return Err(RelayError::Unsupported("unsupported tool choice")); + } + let n = match client { + ClientProtocol::Chat => { + keys(t, &["type", "function"])?; + keys(&t["function"], &["name"])?; + &t["function"]["name"] + } + ClientProtocol::Responses => { + keys(t, &["type", "name"])?; + &t["name"] + } + }; + let n = name(n)?; + if !tool_names.contains(&n) { + return Err(RelayError::Invalid("tool choice names an unknown function")); + } + ToolChoice::Named(n) + } + }; + if tools.is_empty() && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_)) { + return Err(RelayError::Invalid("tool choice requires tools")); + } + let reasoning_effort = match client { + ClientProtocol::Chat => v + .get("reasoning_effort") + .filter(|v| !v.is_null()) + .map(string) + .transpose()?, + ClientProtocol::Responses => { + if let Some(r) = v.get("reasoning").filter(|v| !v.is_null()) { + keys(r, &["effort"])?; + r.get("effort").map(string).transpose()? + } else { + None + } + } + }; + if reasoning_effort + .as_ref() + .is_some_and(|s| !["low", "medium", "high"].contains(&s.as_str())) + { + return Err(RelayError::Unsupported("unsupported reasoning effort")); + } + let mut messages = Vec::new(); + if let Some(s) = v.get("instructions").filter(|v| !v.is_null()) { + messages.push(Message { + role: Role::System, + content: vec![InputBlock::Text(string(s)?)], + }); + } + match client { + ClientProtocol::Chat => { + for m in v["messages"] + .as_array() + .ok_or(RelayError::Invalid("messages must be an array"))? + { + keys( + m, + &[ + "role", + "content", + "tool_calls", + "tool_call_id", + "reasoning_content", + "provider_metadata", + "refusal", + ], + )?; + let role = Role::parse(&m["role"])?; + if role == Role::Tool { + keys(m, &["role", "content", "tool_call_id"])?; + } else if m.get("tool_call_id").is_some() { + return Err(RelayError::Invalid("tool_call_id requires tool role")); + } + let mut content = Vec::new(); + if role == Role::Tool { + content.push(InputBlock::ToolResult { + id: name(&m["tool_call_id"])?, + content: string(&m["content"])?, + }); + } else { + if let Some(t) = m.get("reasoning_content").filter(|v| !v.is_null()) { + if role != Role::Assistant { + return Err(RelayError::Invalid( + "reasoning requires assistant role", + )); + } + content.push(InputBlock::Thinking { + text: string(t)?, + signature: signature(m.get("provider_metadata"))?, + }); + } else if m.get("provider_metadata").is_some() { + return Err(RelayError::Invalid("signature without reasoning content")); + } + content.extend(text_blocks(&m["content"], role == Role::Assistant)?); + if let Some(t) = m.get("refusal").filter(|v| !v.is_null()) { + if role != Role::Assistant { + return Err(RelayError::Invalid("refusal requires assistant role")); + } + content.push(InputBlock::Refusal(string(t)?)); + } + } + if let Some(calls) = m.get("tool_calls").filter(|v| !v.is_null()) { + if role != Role::Assistant { + return Err(RelayError::Invalid("tool calls require assistant role")); + } + for c in calls + .as_array() + .ok_or(RelayError::Invalid("invalid tool calls"))? + { + keys(c, &["id", "type", "function", "provider_metadata"])?; + keys(&c["function"], &["name", "arguments"])?; + if c["type"] != "function" { + return Err(RelayError::Unsupported("unsupported tool call")); + } + content.push(InputBlock::ToolCall { + id: name(&c["id"])?, + name: name(&c["function"]["name"])?, + arguments: arguments( + &c["function"]["arguments"], + limits.tool_argument_bytes, + )?, + signature: signature(c.get("provider_metadata"))?, + }); + } + } + messages.push(Message { role, content }); + } + } + ClientProtocol::Responses => { + let input = &v["input"]; + if let Some(s) = input.as_str() { + messages.push(Message { + role: Role::User, + content: vec![InputBlock::Text(s.into())], + }); + } else { + for i in input + .as_array() + .ok_or(RelayError::Invalid("input must be a string or array"))? + { + let kind = match i.get("type") { + None => "message", + Some(kind) => kind + .as_str() + .ok_or(RelayError::Invalid("invalid Responses input type"))?, + }; + match kind { + "message" => { + keys(i, &["type", "role", "content"])?; + let role = Role::parse(&i["role"])?; + if role == Role::Tool { + return Err(RelayError::Invalid("use function_call_output")); + } + messages.push(Message { + role, + content: text_blocks(&i["content"], role == Role::Assistant)?, + }); + } + "function_call" => { + keys( + i, + &["type", "call_id", "name", "arguments", "provider_metadata"], + )?; + messages.push(Message { + role: Role::Assistant, + content: vec![InputBlock::ToolCall { + id: name(&i["call_id"])?, + name: name(&i["name"])?, + arguments: arguments( + &i["arguments"], + limits.tool_argument_bytes, + )?, + signature: signature(i.get("provider_metadata"))?, + }], + }); + } + "function_call_output" => { + keys(i, &["type", "call_id", "output"])?; + messages.push(Message { + role: Role::Tool, + content: vec![InputBlock::ToolResult { + id: name(&i["call_id"])?, + content: string(&i["output"])?, + }], + }); + } + _ => { + return Err(RelayError::Unsupported( + "stateful, encrypted or non-text Responses input", + )); + } + } + } + } + } + } + if messages.is_empty() || messages.len() > 1024 { + return Err(RelayError::Limit("message count")); + } + let mut calls = BTreeSet::new(); + let mut results = BTreeSet::new(); + let mut count = 0usize; + for m in &messages { + for b in &m.content { + count += 1; + match b { + InputBlock::ToolCall { id, .. } if !calls.insert(id) => { + return Err(RelayError::Invalid("duplicate call ID")); + } + InputBlock::ToolResult { id, .. } + if !calls.contains(id) || !results.insert(id) => + { + return Err(RelayError::Invalid("orphan or duplicate tool result")); + } + _ => {} + } + } + } + if count > 4096 { + return Err(RelayError::Limit("input block count")); + } + Ok(Self { + client, + model, + stream, + max_output_tokens, + messages, + tools, + stop, + tool_choice, + include_usage, + temperature: number(v.get("temperature"), 2.0)?, + top_p: number(v.get("top_p"), 1.0)?, + parallel_tool_calls: boolean(v.get("parallel_tool_calls"))?, + reasoning_effort, + }) + } + + pub(crate) fn upstream(&self, route: &Route) -> Result { + for m in &self.messages { + for b in &m.content { + if route.protocol == UpstreamProtocol::Anthropic + && matches!( + b, + InputBlock::ToolCall { + signature: Some(_), + .. + } + ) + { + return Err(RelayError::Unsupported( + "Anthropic tool history cannot carry standalone signatures", + )); + } + let sig = match b { + InputBlock::Thinking { signature, .. } + | InputBlock::ToolCall { signature, .. } => signature, + _ => continue, + }; + if sig.as_ref().is_some_and(|(p, _)| *p != route.protocol) { + return Err(RelayError::Unsupported( + "provider signatures cannot be replayed to a different protocol", + )); + } + } + } + if matches!( + route.protocol, + UpstreamProtocol::Anthropic | UpstreamProtocol::Gemini + ) && (self.reasoning_effort.is_some() + || self.tools.iter().any(|t| t.strict == Some(true))) + { + return Err(RelayError::Unsupported( + "reasoning effort or strict tools require an OpenAI protocol route", + )); + } + match route.protocol { + UpstreamProtocol::Chat => self.chat(route), + UpstreamProtocol::Responses => self.responses(route), + UpstreamProtocol::Anthropic => self.anthropic(route), + UpstreamProtocol::Gemini => self.gemini(), + } + } + fn openai_tools(&self, chat: bool) -> Vec { + self.tools + .iter() + .map(|t| { + let mut v = json!({"name":t.name,"parameters":t.parameters}); + if let Some(d) = &t.description { + v["description"] = json!(d); + } + if let Some(s) = t.strict { + v["strict"] = json!(s); + } + if chat { + json!({"type":"function","function":v}) + } else { + v["type"] = json!("function"); + v + } + }) + .collect() + } + fn openai_choice(&self, chat: bool) -> Value { + match &self.tool_choice { + ToolChoice::Auto => json!("auto"), + ToolChoice::None => json!("none"), + ToolChoice::Required => json!("required"), + ToolChoice::Named(n) => { + if chat { + json!({"type":"function","function":{"name":n}}) + } else { + json!({"type":"function","name":n}) + } + } + } + } + fn sampling(&self, v: &mut Value) { + if let Some(n) = self.temperature { + v["temperature"] = json!(n); + } + if let Some(n) = self.top_p { + v["top_p"] = json!(n); + } + } + fn chat(&self, route: &Route) -> Result { + let mut messages = Vec::new(); + for m in &self.messages { + let mut v = json!({"role":m.role.name(),"content":Value::Null}); + let mut text = String::new(); + let mut thoughts = String::new(); + let mut refusals = String::new(); + let mut calls = Vec::new(); + for b in &m.content { + match b { + InputBlock::Text(s) => text.push_str(s), + InputBlock::Refusal(s) => refusals.push_str(s), + InputBlock::Thinking { text, signature } => { + thoughts.push_str(text); + if signature.is_some() { + return Err(RelayError::Unsupported( + "opaque Chat reasoning signatures", + )); + } + } + InputBlock::ToolCall { + id, + name, + arguments, + signature, + } => { + if signature.is_some() { + return Err(RelayError::Unsupported("opaque Chat tool signatures")); + } + calls.push(json!({"id":id,"type":"function","function":{"name":name,"arguments":arguments}})); + } + InputBlock::ToolResult { id, content } => { + v["tool_call_id"] = json!(id); + text.push_str(content); + } + } + } + if !text.is_empty() || m.role != Role::Assistant { + v["content"] = json!(text); + } + if !thoughts.is_empty() { + v["reasoning_content"] = json!(thoughts); + } + if !refusals.is_empty() { + v["refusal"] = json!(refusals); + } + if !calls.is_empty() { + v["tool_calls"] = json!(calls); + } + messages.push(v); + } + let mut v = json!({"model":route.model,"messages":messages,"stream":self.stream,"max_completion_tokens":self.max_output_tokens,"store":false}); + if self.stream { + v["stream_options"] = json!({"include_usage":true}); + } + if !self.tools.is_empty() { + v["tools"] = json!(self.openai_tools(true)); + v["tool_choice"] = self.openai_choice(true); + } + if let Some(p) = self.parallel_tool_calls { + v["parallel_tool_calls"] = json!(p); + } + if let Some(e) = &self.reasoning_effort { + v["reasoning_effort"] = json!(e); + } + if !self.stop.is_empty() { + v["stop"] = json!(self.stop); + } + self.sampling(&mut v); + Ok(v) + } + fn responses(&self, route: &Route) -> Result { + if !self.stop.is_empty() { + return Err(RelayError::Unsupported("Responses stop sequences")); + } + let mut input = Vec::new(); + for m in &self.messages { + for b in &m.content { + match b { + InputBlock::Text(s) => input.push(json!({"role":m.role.name(),"content":[{"type":if m.role == Role::Assistant {"output_text"} else {"input_text"},"text":s}]})), + InputBlock::Refusal(s) => input.push(json!({"role":"assistant","content":[{"type":"refusal","refusal":s}]})), + InputBlock::Thinking { .. } => return Err(RelayError::Unsupported("Responses reasoning history requires native item identity")), + InputBlock::ToolCall { id, name, arguments, signature } => { + if signature.is_some() { return Err(RelayError::Unsupported("opaque Responses tool signatures")); } + input.push(json!({"type":"function_call","call_id":id,"name":name,"arguments":arguments})); + } + InputBlock::ToolResult { id, content } => input.push(json!({"type":"function_call_output","call_id":id,"output":content})), + } + } + } + let mut v = json!({"model":route.model,"input":input,"stream":self.stream,"max_output_tokens":self.max_output_tokens,"store":false}); + if !self.tools.is_empty() { + v["tools"] = json!(self.openai_tools(false)); + v["tool_choice"] = self.openai_choice(false); + } + if let Some(p) = self.parallel_tool_calls { + v["parallel_tool_calls"] = json!(p); + } + if let Some(e) = &self.reasoning_effort { + v["reasoning"] = json!({"effort":e}); + } + self.sampling(&mut v); + Ok(v) + } + fn split_system(&self) -> Result<(Vec, Vec<&Message>), RelayError> { + let mut system = Vec::new(); + let mut messages = Vec::new(); + for m in &self.messages { + if matches!(m.role, Role::System | Role::Developer) { + if !messages.is_empty() { + return Err(RelayError::Unsupported( + "late system instructions cannot be reordered", + )); + } + for b in &m.content { + match b { + InputBlock::Text(s) => system.push(json!({"type":"text","text":s})), + _ => return Err(RelayError::Unsupported("non-text system instruction")), + } + } + } else { + messages.push(m); + } + } + Ok((system, messages)) + } + fn anthropic(&self, route: &Route) -> Result { + if self.temperature.is_some_and(|t| t > 1.0) { + return Err(RelayError::Unsupported("Anthropic temperature exceeds one")); + } + let (system, source) = self.split_system()?; + let mut messages = Vec::new(); + for m in source { + let mut content = Vec::new(); + for b in &m.content { + content.push(match b { + InputBlock::Text(s) | InputBlock::Refusal(s) => json!({"type":"text","text":s}), + InputBlock::Thinking { text, signature: Some((UpstreamProtocol::Anthropic, s)) } => json!({"type":"thinking","thinking":text,"signature":s}), + InputBlock::Thinking { .. } => return Err(RelayError::Unsupported("Anthropic thinking history needs its original signature")), + InputBlock::ToolCall { id, name, arguments, .. } => json!({"type":"tool_use","id":id,"name":name,"input":serde_json::from_str::(arguments).map_err(|_| RelayError::Invalid("invalid tool JSON"))?}), + InputBlock::ToolResult { id, content } => json!({"type":"tool_result","tool_use_id":id,"content":content}), + }); + } + messages.push(json!({"role":if m.role == Role::Assistant {"assistant"} else {"user"},"content":content})); + } + let mut v = json!({"model":route.model,"system":system,"messages":messages,"stream":self.stream,"max_tokens":self.max_output_tokens}); + if !self.tools.is_empty() { + v["tools"] = json!( + self.tools + .iter() + .map(|t| { + let mut v = json!({"name":t.name,"input_schema":t.parameters}); + if let Some(d) = &t.description { + v["description"] = json!(d); + } + v + }) + .collect::>() + ); + v["tool_choice"] = match &self.tool_choice { + ToolChoice::Auto => json!({"type":"auto"}), + ToolChoice::None => json!({"type":"none"}), + ToolChoice::Required => json!({"type":"any"}), + ToolChoice::Named(n) => json!({"type":"tool","name":n}), + }; + if let Some(p) = self.parallel_tool_calls { + v["tool_choice"]["disable_parallel_tool_use"] = json!(!p); + } + } + if !self.stop.is_empty() { + v["stop_sequences"] = json!(self.stop); + } + self.sampling(&mut v); + Ok(v) + } + fn gemini(&self) -> Result { + if self.parallel_tool_calls == Some(false) { + return Err(RelayError::Unsupported( + "Gemini cannot enforce serial tool calls", + )); + } + let (system, source) = self.split_system()?; + let mut calls = BTreeMap::new(); + let mut contents = Vec::new(); + for m in source { + let mut parts = Vec::new(); + for b in &m.content { + parts.push(match b { + InputBlock::Text(s) | InputBlock::Refusal(s) => json!({"text":s}), + InputBlock::Thinking { text, signature: Some((UpstreamProtocol::Gemini, s)) } => json!({"text":text,"thought":true,"thoughtSignature":s}), + InputBlock::Thinking { .. } => return Err(RelayError::Unsupported("Gemini thinking history needs its original signature")), + InputBlock::ToolCall { id, name, arguments, signature } => { + calls.insert(id, name); + let mut p = json!({"functionCall":{"id":id,"name":name,"args":serde_json::from_str::(arguments).map_err(|_| RelayError::Invalid("invalid tool JSON"))?}}); + if let Some((_, s)) = signature { p["thoughtSignature"] = json!(s); } p + } + InputBlock::ToolResult { id, content } => { + let name = calls.get(id).ok_or(RelayError::Invalid("Gemini tool result requires its original call"))?; + let result = serde_json::from_str::(content).ok().filter(Value::is_object).unwrap_or_else(|| json!({"output":content})); + json!({"functionResponse":{"id":id,"name":name,"response":result}}) + } + }); + } + contents.push( + json!({"role":if m.role == Role::Assistant {"model"} else {"user"},"parts":parts}), + ); + } + let mut generation = json!({"maxOutputTokens":self.max_output_tokens,"candidateCount":1}); + if let Some(t) = self.temperature { + generation["temperature"] = json!(t); + } + if let Some(t) = self.top_p { + generation["topP"] = json!(t); + } + if !self.stop.is_empty() { + generation["stopSequences"] = json!(self.stop); + } + let mut v = json!({"contents":contents,"generationConfig":generation}); + if !system.is_empty() { + v["systemInstruction"] = json!({"parts":system.iter().map(|v| json!({"text":v["text"]})).collect::>()}); + } + if !self.tools.is_empty() { + v["tools"] = json!([{"functionDeclarations":self.tools.iter().map(|t| { let mut v = json!({"name":t.name,"parametersJsonSchema":t.parameters}); + if let Some(d) = &t.description { v["description"] = json!(d); } v }).collect::>()}]); + v["toolConfig"] = match &self.tool_choice { + ToolChoice::Auto => json!({"functionCallingConfig":{"mode":"AUTO"}}), + ToolChoice::None => json!({"functionCallingConfig":{"mode":"NONE"}}), + ToolChoice::Required => json!({"functionCallingConfig":{"mode":"ANY"}}), + ToolChoice::Named(n) => { + json!({"functionCallingConfig":{"mode":"ANY","allowedFunctionNames":[n]}}) + } + }; + } + Ok(v) + } +} diff --git a/apps/core-rust/src/relay/sse.rs b/apps/core-rust/src/relay/sse.rs new file mode 100644 index 000000000..7d8abcbdd --- /dev/null +++ b/apps/core-rust/src/relay/sse.rs @@ -0,0 +1,176 @@ +//! Incremental SSE framing. HTTP chunks are not event or UTF-8 boundaries. +use super::RelayError; + +#[derive(Debug)] +pub(crate) struct Frame { + pub event: Option, + pub data: String, +} + +pub(crate) struct Parser { + line: Vec, + data: String, + event: Option, + has_data: bool, + skip_lf: bool, + first_line: bool, + bytes: usize, + limit: usize, +} +impl Parser { + pub fn new(limit: usize) -> Self { + Self { + line: Vec::new(), + data: String::new(), + event: None, + has_data: false, + skip_lf: false, + first_line: true, + bytes: 0, + limit, + } + } + /// Process one byte so callers can apply backpressure between events even + /// when a single network chunk contains thousands of small events. + pub fn feed(&mut self, byte: u8) -> Result, RelayError> { + if self.skip_lf { + self.skip_lf = false; + if byte == b'\n' { + return Ok(None); + } + } + self.bytes = self + .bytes + .checked_add(1) + .ok_or(RelayError::Limit("SSE event"))?; + if self.bytes > self.limit { + return Err(RelayError::Limit("SSE event")); + } + match byte { + b'\r' => { + self.skip_lf = true; + self.end_line() + } + b'\n' => self.end_line(), + _ => { + self.line.push(byte); + Ok(None) + } + } + } + fn end_line(&mut self) -> Result, RelayError> { + let raw = std::mem::take(&mut self.line); + let line = + std::str::from_utf8(&raw).map_err(|_| RelayError::Protocol("SSE is not UTF-8"))?; + let line = if self.first_line { + self.first_line = false; + line.strip_prefix('\u{feff}').unwrap_or(line) + } else { + line + }; + if line.is_empty() { + self.bytes = 0; + let event = self.event.take(); + if self.has_data { + self.has_data = false; + return Ok(Some(Frame { + event, + data: std::mem::take(&mut self.data), + })); + } + self.data.clear(); + return Ok(None); + } + if line.starts_with(':') { + return Ok(None); + } + let (field, value) = line.split_once(':').unwrap_or((line, "")); + let value = value.strip_prefix(' ').unwrap_or(value); + match field { + "event" => self.event = Some(value.to_owned()), + "data" => { + if self.has_data { + self.data.push('\n'); + } + self.data.push_str(value); + self.has_data = true; + } + // id/retry do not authorize reconnecting or replaying a paid POST. + _ => {} + } + Ok(None) + } + pub fn finish(&self) -> Result<(), RelayError> { + if !self.line.is_empty() || self.has_data { + Err(RelayError::Truncated) + } else { + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn split_utf8_crlf_multiline_comments_and_bom() { + let raw = "\u{feff}: hello\r\nevent: delta\r\ndata: {\"text\":\"鲸鱼\",\r\ndata: \"n\":1}\r\n\r\n"; + let mut parser = Parser::new(1024); + let mut out = Vec::new(); + for b in raw.as_bytes() { + if let Some(f) = parser.feed(*b).unwrap() { + out.push(f); + } + } + parser.finish().unwrap(); + assert_eq!(out.len(), 1); + assert_eq!(out[0].event.as_deref(), Some("delta")); + assert_eq!(out[0].data, "{\"text\":\"鲸鱼\",\n\"n\":1}"); + } + #[test] + fn lone_cr_and_empty_data_are_valid() { + let mut p = Parser::new(128); + let mut out = Vec::new(); + for b in b"data:\r\rdata: [DONE]\r\r" { + if let Some(f) = p.feed(*b).unwrap() { + out.push(f.data); + } + } + assert_eq!(out, ["", "[DONE]"]); + p.finish().unwrap(); + } + #[test] + fn unterminated_and_invalid_utf8_are_rejected() { + let mut p = Parser::new(128); + for b in b"data: partial" { + p.feed(*b).unwrap(); + } + assert_eq!(p.finish(), Err(RelayError::Truncated)); + let mut p = Parser::new(128); + p.feed(255).unwrap(); + assert!(matches!(p.feed(b'\n'), Err(RelayError::Protocol(_)))); + } + #[test] + fn comments_and_unknown_fields_cannot_evade_the_event_bound() { + let mut p = Parser::new(16); + for b in b":1234567\n:123456" { + p.feed(*b).unwrap(); + } + assert!(matches!(p.feed(b'7'), Err(RelayError::Limit(_)))); + } + #[test] + fn many_events_do_not_accumulate() { + let mut p = Parser::new(32); + let mut count = 0; + for _ in 0..10_000 { + for b in b"data: x\n\n" { + if p.feed(*b).unwrap().is_some() { + count += 1; + } + } + } + assert_eq!(count, 10_000); + assert_eq!(p.bytes, 0); + assert!(p.data.is_empty()); + } +} diff --git a/apps/core-rust/src/relay/tests.rs b/apps/core-rust/src/relay/tests.rs new file mode 100644 index 000000000..eb4b900e4 --- /dev/null +++ b/apps/core-rust/src/relay/tests.rs @@ -0,0 +1,281 @@ +use super::{ + ClientProtocol, Event, Limits, RelayError, Request, Route, UpstreamProtocol, Usage, + decode::Decoder, encode::Encoder, sse::Frame, +}; +use serde_json::{Value, json}; + +fn input(client: ClientProtocol, stream: bool) -> Request { + let body = match client { + ClientProtocol::Chat => { + json!({"model":"alias","messages":[{"role":"user","content":"hello"}],"stream":stream}) + } + ClientProtocol::Responses => json!({"model":"alias","input":"hello","stream":stream}), + }; + Request::parse( + client, + &serde_json::to_vec(&body).unwrap(), + &Limits::default(), + ) + .unwrap() +} +fn frame(decoder: &mut Decoder, v: Value) -> Result, RelayError> { + decoder.frame(Frame { + event: v["type"].as_str().map(str::to_owned), + data: v.to_string(), + }) +} + +#[test] +fn extra_tool_result_fields_are_not_silently_discarded() { + for field in ["reasoning_content", "refusal", "provider_metadata"] { + let mut body = json!({"model":"alias","messages":[ + {"role":"assistant","tool_calls":[{"type":"function","id":"old_call","function":{"name":"weather","arguments":"{}"}}]}, + {"role":"tool","tool_call_id":"old_call","content":"result"}]}); + body["messages"][1][field] = json!("must not disappear"); + assert!(matches!( + Request::parse( + ClientProtocol::Chat, + &serde_json::to_vec(&body).unwrap(), + &Limits::default() + ), + Err(RelayError::Unsupported(_)) + )); + } +} +#[test] +fn a_tool_call_id_is_only_valid_on_a_tool_message() { + let body = json!({"model":"alias","messages":[{"role":"user","content":"hello","tool_call_id":"unexpected"}]}); + assert!(matches!( + Request::parse( + ClientProtocol::Chat, + &serde_json::to_vec(&body).unwrap(), + &Limits::default() + ), + Err(RelayError::Invalid(_)) + )); +} +#[test] +fn responses_input_type_must_be_a_string_when_present() { + for kind in [Value::Null, json!(17), json!(true)] { + let body = json!({"model":"alias","input":[{"type":kind,"role":"user","content":"hello"}]}); + assert!(matches!( + Request::parse( + ClientProtocol::Responses, + &serde_json::to_vec(&body).unwrap(), + &Limits::default() + ), + Err(RelayError::Invalid(_)) + )); + } +} +#[test] +fn signatures_cannot_be_dropped_from_anthropic_tool_history_or_cross_providers() { + let body = json!({"model":"alias","messages":[{"role":"assistant","tool_calls":[ + {"type":"function","id":"old_call","function":{"name":"weather","arguments":"{}"},"provider_metadata":{"protocol":"anthropic","signature":"opaque"}}]}]}); + let request = Request::parse( + ClientProtocol::Chat, + &serde_json::to_vec(&body).unwrap(), + &Limits::default(), + ) + .unwrap(); + for provider in [UpstreamProtocol::Anthropic, UpstreamProtocol::Gemini] { + let route = Route::new( + "r".into(), + provider, + "https://example.test/operation", + "real".into(), + "test-only", + 1, + ) + .unwrap(); + assert!(matches!( + request.upstream(&route), + Err(RelayError::Unsupported(_)) + )); + } +} +#[test] +fn gemini_plain_text_signatures_survive_both_client_formats_and_output_modes() { + for client in [ClientProtocol::Chat, ClientProtocol::Responses] { + for stream in [false, true] { + let source = json!({"responseId":"g_1","candidates":[{"index":0,"content":{"parts":[{"text":"answer","thoughtSignature":"opaque_signature"}]},"finishReason":"STOP"}], + "usageMetadata":{"promptTokenCount":2,"candidatesTokenCount":3}}); + let mut decoder = Decoder::new(UpstreamProtocol::Gemini, &Limits::default()); + let events = if stream { + let mut events = frame(&mut decoder, source).unwrap(); + events.extend(decoder.eof().unwrap()); + events + } else { + decoder.json(&source).unwrap() + }; + let mut encoder = + Encoder::new(&input(client, stream), "signed_text", &Limits::default()); + let mut bytes = Vec::new(); + for event in events { + for packet in encoder.consume(event).unwrap() { + bytes.extend(packet); + } + } + for packet in encoder.terminal().unwrap() { + bytes.extend(packet); + } + let body = String::from_utf8(bytes).unwrap(); + assert!(body.contains("opaque_signature")); + assert!(!body.contains("reasoning_content")); + let packets: Vec = if stream { + body.lines() + .filter_map(|l| l.strip_prefix("data: ")) + .filter(|s| *s != "[DONE]") + .map(|s| serde_json::from_str(s).unwrap()) + .collect() + } else { + vec![serde_json::from_str(&body).unwrap()] + }; + match client { + ClientProtocol::Chat if stream => assert!(packets.iter().any(|p| p["choices"][0] + ["delta"]["content_details"][0]["signature"] + == "opaque_signature")), + ClientProtocol::Chat => assert_eq!( + packets[0]["choices"][0]["message"]["content_details"][0]["signature"], + "opaque_signature" + ), + ClientProtocol::Responses => { + let response = if stream { + &packets.last().unwrap()["response"] + } else { + &packets[0] + }; + assert_eq!( + response["output"][0]["provider_metadata"]["signature"], + "opaque_signature" + ); + assert_eq!( + response["output"][0]["provider_metadata"]["protocol"], + "gemini" + ); + } + } + } + } +} +#[test] +fn annotated_responses_fail_instead_of_losing_citations() { + let part = json!({"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test/"}]}); + let mut decoder = Decoder::new(UpstreamProtocol::Responses, &Limits::default()); + assert!(matches!(decoder.json(&json!({"id":"r_1","status":"completed","output":[{"id":"m_1","type":"message","content":[part.clone()]}]})),Err(RelayError::Protocol(_)))); + let mut decoder = Decoder::new(UpstreamProtocol::Responses, &Limits::default()); + frame( + &mut decoder, + json!({"type":"response.created","response":{"id":"r_1"}}), + ) + .unwrap(); + assert!(matches!( + frame( + &mut decoder, + json!({"type":"response.content_part.added","output_index":0,"content_index":0,"part":part}) + ), + Err(RelayError::Protocol(_)) + )); +} +#[test] +fn encrypted_reasoning_is_rejected_at_added_done_and_terminal_events() { + let item = json!({"type":"reasoning","id":"rs_1","summary":[],"encrypted_content":"opaque"}); + for kind in [ + "response.output_item.added", + "response.output_item.done", + "response.completed", + ] { + let mut decoder = Decoder::new(UpstreamProtocol::Responses, &Limits::default()); + frame( + &mut decoder, + json!({"type":"response.created","response":{"id":"r_1"}}), + ) + .unwrap(); + let event = if kind == "response.completed" { + json!({"type":kind,"response":{"id":"r_1","status":"completed","output":[item.clone()]}}) + } else { + json!({"type":kind,"output_index":0,"item":item.clone()}) + }; + assert!(matches!( + frame(&mut decoder, event), + Err(RelayError::Protocol(_)) + )); + } +} +#[test] +fn huge_sparse_tool_indexes_cannot_allocate_unbounded_output_blocks() { + let limits = Limits { + max_blocks: 1, + ..Limits::default() + }; + let mut decoder = Decoder::new(UpstreamProtocol::Chat, &limits); + let event = json!({"id":"r_1","choices":[{"index":0,"delta":{"tool_calls":[ + {"index":0,"id":"call_1","type":"function","function":{"name":"weather","arguments":"{}"}}, + {"index":u64::MAX,"id":"call_2","type":"function","function":{"name":"weather","arguments":"{}"}}]}}]}); + assert!(matches!( + frame(&mut decoder, event), + Err(RelayError::Limit(_)) + )); +} +#[test] +fn cumulative_usage_cannot_decrease_or_overflow() { + let mut usage = Usage { + input_tokens: Some(7), + output_tokens: Some(3), + ..Usage::default() + }; + assert!( + usage + .merge(Usage { + input_tokens: Some(6), + ..Usage::default() + }) + .is_err() + ); + let mut usage = Usage { + input_tokens: Some(u64::MAX), + ..Usage::default() + }; + assert!( + usage + .merge(Usage { + output_tokens: Some(1), + ..Usage::default() + }) + .is_err() + ); +} + +#[test] +fn invalid_usage_does_not_partially_replace_the_last_valid_snapshot() { + let before = Usage { + input_tokens: Some(7), + output_tokens: Some(3), + ..Usage::default() + }; + let mut current = before; + assert!( + current + .merge(Usage { + input_tokens: Some(8), + output_tokens: Some(2), + ..Usage::default() + }) + .is_err() + ); + assert_eq!(current, before); + let before = Usage { + input_tokens: Some(u64::MAX), + ..Usage::default() + }; + let mut current = before; + assert!( + current + .merge(Usage { + output_tokens: Some(1), + ..Usage::default() + }) + .is_err() + ); + assert_eq!(current, before); +} diff --git a/apps/core-rust/src/relay/types.rs b/apps/core-rust/src/relay/types.rs new file mode 100644 index 000000000..3a1b36439 --- /dev/null +++ b/apps/core-rust/src/relay/types.rs @@ -0,0 +1,499 @@ +//! Relay-local contracts. Routing and money implementations live outside this module. +use std::{fmt, future::Future, pin::Pin, time::Duration}; + +use reqwest::{Url, header::HeaderValue}; +use serde::Serialize; +use serde_json::{Value, json}; + +pub type RelayFuture<'a, T> = Pin> + Send + 'a>>; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ClientProtocol { + Chat, + Responses, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum UpstreamProtocol { + Chat, + Responses, + Anthropic, + Gemini, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Phase { + Route, + Reserve, + Connect, + Headers, + Read, + Write, + Total, + Settle, +} + +/// Public errors deliberately contain neither upstream bodies nor credentials/URLs. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum RelayError { + Invalid(&'static str), + Unsupported(&'static str), + Protocol(&'static str), + Limit(&'static str), + Timeout(Phase), + Cancelled, + Draining, + Busy, + RouteUnavailable, + BillingUnavailable, + UpstreamStatus(u16), + UpstreamConnection, + Truncated, + SettlementPending, +} + +impl RelayError { + pub fn code(&self) -> &'static str { + match self { + Self::Invalid(_) => "invalid_request", + Self::Unsupported(_) => "unsupported_feature", + Self::Protocol(_) | Self::Truncated => "invalid_upstream_response", + Self::Limit(_) => "relay_limit_exceeded", + Self::Timeout(_) => "relay_timeout", + Self::Cancelled => "request_cancelled", + Self::Draining => "relay_draining", + Self::Busy => "relay_busy", + Self::RouteUnavailable => "route_unavailable", + Self::BillingUnavailable => "billing_unavailable", + Self::UpstreamStatus(_) => "upstream_error", + Self::UpstreamConnection => "upstream_connection_error", + Self::SettlementPending => "settlement_pending", + } + } + pub fn status(&self) -> u16 { + match self { + Self::Invalid(_) | Self::Unsupported(_) => 400, + Self::Busy => 429, + Self::Limit(_) => 413, + Self::Timeout(_) => 504, + Self::Cancelled => 499, + Self::Draining + | Self::RouteUnavailable + | Self::BillingUnavailable + | Self::SettlementPending => 503, + _ => 502, + } + } + pub fn json(&self) -> Value { + json!({"error":{"type":self.code(),"code":self.code(),"message":self.to_string()}}) + } +} +impl fmt::Display for RelayError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Invalid(s) | Self::Unsupported(s) | Self::Protocol(s) | Self::Limit(s) => { + write!(f, "{}: {s}", self.code()) + } + Self::Timeout(p) => write!(f, "relay_timeout: {p:?}"), + Self::UpstreamStatus(s) => write!(f, "upstream_error: HTTP {s}"), + _ => f.write_str(self.code()), + } + } +} +impl std::error::Error for RelayError {} + +#[derive(Clone, Debug)] +pub struct Limits { + pub max_in_flight: usize, + pub queue_chunks: usize, + pub chunk_bytes: usize, + pub request_bytes: usize, + pub event_bytes: usize, + pub output_bytes: usize, + pub upstream_bytes: usize, + pub transport_chunk_bytes: usize, + pub max_blocks: usize, + pub tool_argument_bytes: usize, + pub max_output_tokens: u32, + pub connect_timeout: Duration, + pub header_timeout: Duration, + pub read_timeout: Duration, + pub write_timeout: Duration, + pub total_timeout: Duration, + pub hook_timeout: Duration, + /// Only connect failures, before an HTTP request is sent, may be retried. + pub connect_retries: u8, + pub retry_backoff: Duration, +} +impl Default for Limits { + fn default() -> Self { + Self { + max_in_flight: 16, + queue_chunks: 8, + chunk_bytes: 16 * 1024, + request_bytes: 2 * 1024 * 1024, + event_bytes: 256 * 1024, + output_bytes: 1024 * 1024, + upstream_bytes: 64 * 1024 * 1024, + transport_chunk_bytes: 1024 * 1024, + max_blocks: 128, + tool_argument_bytes: 256 * 1024, + max_output_tokens: 65_536, + connect_timeout: Duration::from_secs(5), + header_timeout: Duration::from_secs(30), + read_timeout: Duration::from_secs(30), + write_timeout: Duration::from_secs(10), + total_timeout: Duration::from_secs(300), + hook_timeout: Duration::from_secs(10), + connect_retries: 0, + retry_backoff: Duration::from_millis(200), + } + } +} +impl Limits { + pub(crate) fn validate(&self) -> Result<(), RelayError> { + let sizes = [ + self.max_in_flight, + self.queue_chunks, + self.chunk_bytes, + self.request_bytes, + self.event_bytes, + self.output_bytes, + self.upstream_bytes, + self.transport_chunk_bytes, + self.max_blocks, + self.tool_argument_bytes, + ]; + let times = [ + self.connect_timeout, + self.header_timeout, + self.read_timeout, + self.write_timeout, + self.total_timeout, + self.hook_timeout, + self.retry_backoff, + ]; + if sizes.contains(&0) + || times.iter().any(Duration::is_zero) + || self.max_output_tokens == 0 + || self.max_in_flight > 4096 + || self.queue_chunks > 1024 + || self.chunk_bytes > 65_536 + || self.max_blocks > 1024 + || self.connect_retries > 2 + || self.output_bytes > 64 * 1024 * 1024 + || self.request_bytes > 16 * 1024 * 1024 + || self.event_bytes > self.output_bytes + || self.tool_argument_bytes > self.output_bytes + || self.transport_chunk_bytes > self.upstream_bytes + || times.iter().any(|d| *d > Duration::from_secs(86_400)) + { + return Err(RelayError::Invalid("invalid relay limits")); + } + Ok(()) + } +} + +/// Construct this only after core identity verification. The billing adapter must +/// recheck the actor/key, account funding policy, budget and immutable price version. +#[derive(Clone, Debug)] +pub struct RequestContext { + pub request_id: String, + pub actor_user_id: i64, + pub key_id: i64, +} +impl RequestContext { + pub(crate) fn validate(&self) -> Result<(), RelayError> { + if self.actor_user_id <= 0 + || self.key_id <= 0 + || self.request_id.is_empty() + || self.request_id.len() > 128 + || !self + .request_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_".contains(&b)) + { + return Err(RelayError::Invalid("invalid request identity")); + } + Ok(()) + } +} + +/// An immutable route snapshot selected by task 04, not a URL from an HTTP caller. +/// `endpoint` is the complete operation URL. Gemini streaming uses `alt=sse`. +/// HTTPS is required except for explicit numeric loopback test/local upstreams. +#[derive(Clone)] +pub struct Route { + pub id: String, + pub protocol: UpstreamProtocol, + pub model: String, + pub price_version: u64, + pub(crate) endpoint: Url, + pub(crate) credential: HeaderValue, +} +impl fmt::Debug for Route { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Route") + .field("id", &self.id) + .field("protocol", &self.protocol) + .field("price_version", &self.price_version) + .finish_non_exhaustive() + } +} +impl Route { + pub fn new( + id: String, + protocol: UpstreamProtocol, + endpoint: &str, + model: String, + api_key: &str, + price_version: u64, + ) -> Result { + let endpoint = + Url::parse(endpoint).map_err(|_| RelayError::Invalid("invalid route URL"))?; + let local = endpoint + .host_str() + .and_then(|h| h.trim_matches(['[', ']']).parse::().ok()) + .is_some_and(|ip| ip.is_loopback()); + if id.is_empty() + || id.len() > 128 + || model.is_empty() + || model.len() > 256 + || price_version == 0 + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.fragment().is_some() + || endpoint.host_str().is_none() + || !(endpoint.scheme() == "https" || (endpoint.scheme() == "http" && local)) + || endpoint + .query_pairs() + .any(|(k, v)| protocol != UpstreamProtocol::Gemini || k != "alt" || v != "sse") + || api_key.is_empty() + || api_key.len() > 4096 + { + return Err(RelayError::Invalid("invalid route configuration")); + } + let auth = if matches!( + protocol, + UpstreamProtocol::Chat | UpstreamProtocol::Responses + ) { + format!("Bearer {api_key}") + } else { + api_key.to_owned() + }; + let mut credential = HeaderValue::from_str(&auth) + .map_err(|_| RelayError::Invalid("invalid upstream credential"))?; + credential.set_sensitive(true); + Ok(Self { + id, + protocol, + endpoint, + model, + credential, + price_version, + }) + } +} + +pub trait RouteProvider: Send + Sync + 'static { + /// Read an already active Rust-owned snapshot. Do not call the Go extension. + fn select<'a>( + &'a self, + context: &'a RequestContext, + request: &'a super::Request, + ) -> RelayFuture<'a, Route>; +} + +/// A durable reservation handle, never a live SQL transaction/connection. +#[derive(Clone, Debug)] +pub struct Reservation { + pub id: String, +} + +pub trait Billing: Send + Sync + 'static { + /// Idempotent by context.request_id. Commit before returning. A timeout may + /// mean the commit succeeded: durable recovery must resolve that ambiguity. + fn reserve<'a>( + &'a self, + context: &'a RequestContext, + route: &'a Route, + request: &'a super::Request, + ) -> RelayFuture<'a, Reservation>; + /// Persist settlement or a reconciliation obligation, idempotently. Missing + /// usage or a disconnected upstream must NEVER be interpreted as zero cost. + /// This hook runs even after the client drops its body. Do not wait for Go. + fn finalize<'a>( + &'a self, + reservation: &'a Reservation, + report: &'a Report, + ) -> RelayFuture<'a, ()>; +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)] +pub struct Usage { + pub input_tokens: Option, + pub output_tokens: Option, + pub cached_input_tokens: Option, + pub cache_creation_input_tokens: Option, + pub reasoning_output_tokens: Option, +} +impl Usage { + pub(crate) fn merge(&mut self, next: Self) -> Result<(), RelayError> { + fn field(old: &mut Option, next: Option) -> Result<(), RelayError> { + if let Some(n) = next { + if old.is_some_and(|o| n < o) { + return Err(RelayError::Protocol("usage counter decreased")); + } + *old = Some(n); + } + Ok(()) + } + let mut combined = *self; + field(&mut combined.input_tokens, next.input_tokens)?; + field(&mut combined.output_tokens, next.output_tokens)?; + field(&mut combined.cached_input_tokens, next.cached_input_tokens)?; + field( + &mut combined.cache_creation_input_tokens, + next.cache_creation_input_tokens, + )?; + field( + &mut combined.reasoning_output_tokens, + next.reasoning_output_tokens, + )?; + if let (Some(i), Some(o)) = (combined.input_tokens, combined.output_tokens) { + i.checked_add(o) + .ok_or(RelayError::Protocol("usage overflow"))?; + } + *self = combined; + Ok(()) + } + pub(crate) fn wire(&self, client: ClientProtocol) -> Value { + if *self == Self::default() { + return Value::Null; + } + let mut out = serde_json::Map::new(); + let (input, output) = match client { + ClientProtocol::Chat => ("prompt_tokens", "completion_tokens"), + ClientProtocol::Responses => ("input_tokens", "output_tokens"), + }; + if let Some(v) = self.input_tokens { + out.insert(input.into(), json!(v)); + } + if let Some(v) = self.output_tokens { + out.insert(output.into(), json!(v)); + } + if let (Some(i), Some(o)) = (self.input_tokens, self.output_tokens) + && let Some(total) = i.checked_add(o) + { + out.insert("total_tokens".into(), json!(total)); + } + if let Some(v) = self.cached_input_tokens { + out.insert(format!("{input}_details"), json!({"cached_tokens":v})); + } + if let Some(v) = self.reasoning_output_tokens { + out.insert(format!("{output}_details"), json!({"reasoning_tokens":v})); + } + if let Some(v) = self.cache_creation_input_tokens { + out.insert("cache_creation_input_tokens".into(), json!(v)); + } + Value::Object(out) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FinishReason { + Stop, + Length, + ToolCalls, + ContentFilter, +} +impl FinishReason { + pub(crate) fn chat(self) -> &'static str { + match self { + Self::Stop => "stop", + Self::Length => "length", + Self::ToolCalls => "tool_calls", + Self::ContentFilter => "content_filter", + } + } +} + +#[derive(Clone, Debug)] +pub(crate) enum BlockKind { + Text, + Refusal, + Thinking, + Tool { id: String, name: String }, +} +#[derive(Clone, Debug)] +pub(crate) enum Event { + Start(String), + Block { + index: usize, + kind: BlockKind, + }, + Delta { + index: usize, + text: String, + }, + Signature { + index: usize, + provider: UpstreamProtocol, + value: String, + }, + EndBlock(usize), + Usage(Usage), + Finish(FinishReason), + Done, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Outcome { + Completed, + Cancelled, + TimedOut, + Failed, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Settlement { + NotReserved, + Confirmed, + Pending, +} +#[derive(Clone, Debug, Serialize)] +pub struct Report { + pub request_id: String, + pub route_id: Option, + pub price_version: Option, + pub outcome: Outcome, + pub usage: Usage, + pub finish_reason: Option, + /// True as soon as send starts. A connection error is not proof of no cost. + pub upstream_attempted: bool, + pub attempts: u8, + pub upstream_status: Option, + pub upstream_bytes: usize, + pub emitted_bytes: usize, +} +#[derive(Clone, Debug)] +pub struct Completion { + pub report: Report, + pub settlement: Settlement, + pub error: Option, +} +#[derive(Clone, Copy, Debug)] +pub struct ResponseHead { + pub status: u16, + pub content_type: &'static str, +} +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct DrainReport { + pub cancelled: usize, + pub remaining: usize, +} diff --git a/apps/core-rust/tests/relay_http.rs b/apps/core-rust/tests/relay_http.rs new file mode 100644 index 000000000..8141fa777 --- /dev/null +++ b/apps/core-rust/tests/relay_http.rs @@ -0,0 +1,1542 @@ +//! Real TCP upstream and downstream tests; no external API keys or paid requests. +use axum::{ + Router, + body::{Body, Bytes}, + http::Response, + routing::post, +}; +use lmm_core::relay::*; +use serde_json::{Value, json}; +use std::{ + io, + process::{Child, Command, Stdio}, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{Semaphore, mpsc}, + task::JoinHandle, + time::{sleep, timeout}, +}; + +const WAIT: Duration = Duration::from_secs(8); +static IDS: AtomicUsize = AtomicUsize::new(1); +fn context() -> RequestContext { + RequestContext { + request_id: format!("test_{}", IDS.fetch_add(1, Ordering::Relaxed)), + actor_user_id: 7, + key_id: 9, + } +} + +#[derive(Clone)] +enum Step { + Data(Vec), + Sleep(Duration), + Wait(Arc), +} +#[derive(Clone)] +struct Script { + status: u16, + content_type: &'static str, + header_delay: Duration, + steps: Vec, + complete: bool, +} +impl Script { + fn sse(data: Vec) -> Self { + Self { + status: 200, + content_type: "text/event-stream", + header_delay: Duration::ZERO, + steps: vec![Step::Data(data)], + complete: true, + } + } + fn json(v: Value) -> Self { + Self { + content_type: "application/json", + ..Self::sse(serde_json::to_vec(&v).unwrap()) + } + } +} +#[derive(Default)] +struct Observed { + hits: AtomicUsize, + chunks: AtomicUsize, + disconnected: AtomicBool, + requests: Mutex>, +} +struct Upstream { + address: String, + observed: Arc, + task: JoinHandle<()>, +} +impl Drop for Upstream { + fn drop(&mut self) { + self.task.abort(); + } +} +async fn hold(socket: &mut TcpStream, step: &Step) -> bool { + let mut byte = [0u8; 1]; + match step { + Step::Sleep(d) => { + tokio::select! { _ = sleep(*d) => true, _ = socket.read(&mut byte) => false } + } + Step::Wait(s) => { + tokio::select! { permit = s.acquire() => { permit.unwrap().forget(); true }, _ = socket.read(&mut byte) => false } + } + Step::Data(_) => true, + } +} +async fn read_request(socket: &mut TcpStream) -> io::Result<(String, Value)> { + let mut bytes = Vec::new(); + let mut buf = [0u8; 4096]; + let end = loop { + if let Some(i) = bytes.windows(4).position(|b| b == b"\r\n\r\n") { + break i + 4; + } + let n = socket.read(&mut buf).await?; + if n == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + bytes.extend_from_slice(&buf[..n]); + if bytes.len() > 2 * 1024 * 1024 { + return Err(io::ErrorKind::InvalidData.into()); + } + }; + let headers = + String::from_utf8(bytes[..end].to_vec()).map_err(|_| io::ErrorKind::InvalidData)?; + let len = headers + .lines() + .find_map(|l| { + let (k, v) = l.split_once(':')?; + k.eq_ignore_ascii_case("content-length") + .then(|| v.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + if len > 2 * 1024 * 1024 { + return Err(io::ErrorKind::InvalidData.into()); + } + while bytes.len() < end + len { + let n = socket.read(&mut buf).await?; + if n == 0 { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + bytes.extend_from_slice(&buf[..n]); + } + let v = + serde_json::from_slice(&bytes[end..end + len]).map_err(|_| io::ErrorKind::InvalidData)?; + Ok((headers, v)) +} +impl Upstream { + async fn start(script: Script) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + let observed = Arc::new(Observed::default()); + let state = observed.clone(); + let task = tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let Ok(Ok(request)) = timeout(WAIT, read_request(&mut socket)).await else { + continue; + }; + state.hits.fetch_add(1, Ordering::SeqCst); + state.requests.lock().unwrap().push(request); + if !hold(&mut socket, &Step::Sleep(script.header_delay)).await { + state.disconnected.store(true, Ordering::SeqCst); + continue; + } + let headers = format!( + "HTTP/1.1 {} Test\r\nContent-Type: {}\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n", + script.status, script.content_type + ); + if socket.write_all(headers.as_bytes()).await.is_err() { + state.disconnected.store(true, Ordering::SeqCst); + continue; + } + for step in &script.steps { + if let Step::Data(data) = step { + if data.is_empty() { + continue; + } + let mut chunk = format!("{:x}\r\n", data.len()).into_bytes(); + chunk.extend(data); + chunk.extend(b"\r\n"); + if socket.write_all(&chunk).await.is_err() { + state.disconnected.store(true, Ordering::SeqCst); + break; + } + state.chunks.fetch_add(1, Ordering::SeqCst); + } else if !hold(&mut socket, step).await { + state.disconnected.store(true, Ordering::SeqCst); + break; + } + } + if script.complete { + let _ = socket.write_all(b"0\r\n\r\n").await; + } + let _ = socket.shutdown().await; + } + }); + Self { + address, + observed, + task, + } + } +} +struct Routes { + address: String, + protocol: UpstreamProtocol, +} +impl RouteProvider for Routes { + fn select<'a>(&'a self, _: &'a RequestContext, r: &'a Request) -> RelayFuture<'a, Route> { + Box::pin(async move { + let path = match self.protocol { + UpstreamProtocol::Chat => "/v1/chat/completions", + UpstreamProtocol::Responses => "/v1/responses", + UpstreamProtocol::Anthropic => "/v1/messages", + UpstreamProtocol::Gemini => { + if r.stream { + "/v1beta/models/upstream-real:streamGenerateContent?alt=sse" + } else { + "/v1beta/models/upstream-real:generateContent" + } + } + }; + Route::new( + "route_test".into(), + self.protocol, + &format!("{}{path}", self.address), + "upstream-real".into(), + "upstream-test-secret", + 42, + ) + }) + } +} +#[derive(Default)] +struct Money { + reserves: AtomicUsize, + finals: Mutex>, + pinned: Mutex>, + reject: bool, + fail_finalize: bool, + reserve_wait: Option>, + finalize_wait: Option>, +} +impl Billing for Money { + fn reserve<'a>( + &'a self, + c: &'a RequestContext, + r: &'a Route, + _: &'a Request, + ) -> RelayFuture<'a, Reservation> { + Box::pin(async move { + self.reserves.fetch_add(1, Ordering::SeqCst); + assert_eq!((c.actor_user_id, c.key_id), (7, 9)); + self.pinned + .lock() + .unwrap() + .push((c.request_id.clone(), r.price_version)); + if let Some(gate) = &self.reserve_wait { + gate.acquire().await.unwrap().forget(); + } + if self.reject { + return Err(RelayError::BillingUnavailable); + } + Ok(Reservation { + id: format!("reservation_{}", c.request_id), + }) + }) + } + fn finalize<'a>(&'a self, r: &'a Reservation, report: &'a Report) -> RelayFuture<'a, ()> { + Box::pin(async move { + assert_eq!(r.id, format!("reservation_{}", report.request_id)); + self.finals.lock().unwrap().push(report.clone()); + if let Some(gate) = &self.finalize_wait { + gate.acquire().await.unwrap().forget(); + } + if self.fail_finalize { + Err(RelayError::SettlementPending) + } else { + Ok(()) + } + }) + } +} +fn relay(u: &Upstream, p: UpstreamProtocol, l: Limits, m: Arc) -> Relay { + Relay::new( + Arc::new(Routes { + address: u.address.clone(), + protocol: p, + }), + m, + l, + ) + .unwrap() +} +fn input(client: ClientProtocol, stream: bool) -> Value { + let function = json!({"name":"weather","parameters":{"type":"object","properties":{"city":{"type":"string"}}}}); + match client { + ClientProtocol::Chat => { + json!({"model":"public-model","messages":[{"role":"user","content":"hello"}],"stream":stream, + "stream_options":{"include_usage":true},"tools":[{"type":"function","function":function}]}) + } + ClientProtocol::Responses => { + let mut tool = function; + tool["type"] = json!("function"); + json!({"model":"public-model","input":"hello","stream":stream,"tools":[tool]}) + } + } +} +fn request(client: ClientProtocol, stream: bool, l: &Limits) -> Request { + Request::parse( + client, + &serde_json::to_vec(&input(client, stream)).unwrap(), + l, + ) + .unwrap() +} +fn data(v: Value) -> Vec { + format!("data: {v}\r\n\r\n").into_bytes() +} +fn event(v: Value) -> Vec { + format!( + "event: {}\r\ndata: {v}\r\n\r\n", + v["type"].as_str().unwrap() + ) + .into_bytes() +} +fn cat(parts: Vec>) -> Vec { + parts.into_iter().flatten().collect() +} +fn openai_usage(chat: bool) -> Value { + if chat { + json!({"prompt_tokens":10,"completion_tokens":5,"total_tokens":15,"prompt_tokens_details":{"cached_tokens":3},"completion_tokens_details":{"reasoning_tokens":2}}) + } else { + json!({"input_tokens":10,"output_tokens":5,"total_tokens":15,"input_tokens_details":{"cached_tokens":3},"output_tokens_details":{"reasoning_tokens":2}}) + } +} +fn full(p: UpstreamProtocol) -> Value { + let args = json!({"city":"上海"}); + match p { + UpstreamProtocol::Chat => { + json!({"id":"up_1","choices":[{"index":0,"message":{"role":"assistant","content":"鲸鱼","reasoning_content":"think", + "tool_calls":[{"id":"call_1","type":"function","function":{"name":"weather","arguments":args.to_string()}}]},"finish_reason":"tool_calls"}],"usage":openai_usage(true)}) + } + UpstreamProtocol::Responses => json!({"id":"up_1","status":"completed","output":[ + {"id":"rs_1","type":"reasoning","summary":[{"type":"summary_text","text":"think"}]}, + {"id":"msg_1","type":"message","role":"assistant","content":[{"type":"output_text","text":"鲸鱼","annotations":[]}]}, + {"id":"fc_1","type":"function_call","call_id":"call_1","name":"weather","arguments":args.to_string()}],"usage":openai_usage(false)}), + UpstreamProtocol::Anthropic => { + json!({"id":"up_1","type":"message","content":[{"type":"thinking","thinking":"think","signature":"signed_thinking"}, + {"type":"text","text":"鲸鱼"},{"type":"tool_use","id":"call_1","name":"weather","input":args}],"stop_reason":"tool_use", + "usage":{"input_tokens":7,"cache_read_input_tokens":3,"output_tokens":5}}) + } + UpstreamProtocol::Gemini => { + json!({"responseId":"up_1","candidates":[{"index":0,"content":{"role":"model","parts":[ + {"text":"think","thought":true,"thoughtSignature":"gemini_think"},{"text":"鲸鱼"}, + {"functionCall":{"id":"call_1","name":"weather","args":args},"thoughtSignature":"gemini_call"}]},"finishReason":"STOP"}], + "usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":3,"thoughtsTokenCount":2,"cachedContentTokenCount":3}}) + } + } +} +fn streaming(p: UpstreamProtocol) -> Vec { + match p { + UpstreamProtocol::Chat => cat(vec![ + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}), + ), + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{"reasoning_content":"think"}}]}), + ), + data(json!({"id":"up_1","choices":[{"index":0,"delta":{"content":"鲸"}}]})), + data(json!({"id":"up_1","choices":[{"index":0,"delta":{"content":"鱼"}}]})), + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"weather","arguments":""}}]}}]}), + ), + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"city\":\""}}]}}]}), + ), + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"上海\"}"}}]}}]}), + ), + data( + json!({"id":"up_1","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}), + ), + data(json!({"id":"up_1","choices":[],"usage":openai_usage(true)})), + b"data: [DONE]\n\n".to_vec(), + ]), + UpstreamProtocol::Responses => cat(vec![ + event( + json!({"type":"response.created","response":{"id":"up_1","status":"in_progress"}}), + ), + event( + json!({"type":"response.output_item.added","output_index":0,"item":{"type":"reasoning","id":"rs_1","summary":[]}}), + ), + event( + json!({"type":"response.reasoning_summary_part.added","output_index":0,"summary_index":0,"part":{"type":"summary_text","text":""}}), + ), + event( + json!({"type":"response.reasoning_summary_text.delta","output_index":0,"summary_index":0,"delta":"think"}), + ), + event( + json!({"type":"response.reasoning_summary_part.done","output_index":0,"summary_index":0}), + ), + event( + json!({"type":"response.output_item.added","output_index":1,"item":{"type":"message","id":"msg_1","role":"assistant","content":[]}}), + ), + event( + json!({"type":"response.content_part.added","output_index":1,"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}), + ), + event( + json!({"type":"response.output_text.delta","output_index":1,"content_index":0,"delta":"鲸鱼"}), + ), + event(json!({"type":"response.content_part.done","output_index":1,"content_index":0})), + event( + json!({"type":"response.output_item.added","output_index":2,"item":{"type":"function_call","id":"fc_1","call_id":"call_1","name":"weather","arguments":""}}), + ), + event( + json!({"type":"response.function_call_arguments.delta","output_index":2,"delta":"{\"city\":\""}), + ), + event( + json!({"type":"response.function_call_arguments.delta","output_index":2,"delta":"上海\"}"}), + ), + event( + json!({"type":"response.function_call_arguments.done","output_index":2,"arguments":"{\"city\":\"上海\"}"}), + ), + event(json!({"type":"response.output_item.done","output_index":2})), + event( + json!({"type":"response.completed","response":{"id":"up_1","status":"completed","usage":openai_usage(false)}}), + ), + ]), + UpstreamProtocol::Anthropic => cat(vec![ + event( + json!({"type":"message_start","message":{"id":"up_1","usage":{"input_tokens":7,"cache_read_input_tokens":3,"output_tokens":0}}}), + ), + event( + json!({"type":"content_block_start","index":0,"content_block":{"type":"thinking","thinking":"","signature":""}}), + ), + event( + json!({"type":"content_block_delta","index":0,"delta":{"type":"thinking_delta","thinking":"think"}}), + ), + event( + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"signed_"}}), + ), + event( + json!({"type":"content_block_delta","index":0,"delta":{"type":"signature_delta","signature":"thinking"}}), + ), + event(json!({"type":"content_block_stop","index":0})), + event( + json!({"type":"content_block_start","index":1,"content_block":{"type":"text","text":""}}), + ), + event( + json!({"type":"content_block_delta","index":1,"delta":{"type":"text_delta","text":"鲸鱼"}}), + ), + event(json!({"type":"content_block_stop","index":1})), + event( + json!({"type":"content_block_start","index":2,"content_block":{"type":"tool_use","id":"call_1","name":"weather","input":{}}}), + ), + event( + json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"{\"city\":\""}}), + ), + event( + json!({"type":"content_block_delta","index":2,"delta":{"type":"input_json_delta","partial_json":"上海\"}"}}), + ), + event(json!({"type":"content_block_stop","index":2})), + event( + json!({"type":"message_delta","delta":{"stop_reason":"tool_use"},"usage":{"output_tokens":5}}), + ), + event(json!({"type":"message_stop"})), + ]), + UpstreamProtocol::Gemini => cat(vec![ + data( + json!({"responseId":"up_1","candidates":[{"index":0,"content":{"parts":[{"text":"think","thought":true,"thoughtSignature":"gemini_think"}]}}]}), + ), + data( + json!({"responseId":"up_1","candidates":[{"index":0,"content":{"parts":[{"text":"鲸鱼"}]}}]}), + ), + data( + json!({"responseId":"up_1","candidates":[{"index":0,"content":{"parts":[{"functionCall":{"id":"call_1","name":"weather","args":{"city":"上海"}},"thoughtSignature":"gemini_call"}]}}]}), + ), + data( + json!({"responseId":"up_1","candidates":[{"index":0,"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":10,"candidatesTokenCount":3,"thoughtsTokenCount":2,"cachedContentTokenCount":3}}), + ), + ]), + } +} +async fn collect(s: Session) -> (Result, String, Completion) { + let Session { + head, + mut body, + completion, + .. + } = s; + let head = timeout(WAIT, head).await.unwrap().unwrap(); + let mut data = Vec::new(); + while let Some(chunk) = timeout(WAIT, body.next_chunk()).await.unwrap() { + data.extend(chunk.unwrap()); + } + let done = timeout(WAIT, completion).await.unwrap().unwrap(); + (head, String::from_utf8(data).unwrap(), done) +} +fn packets(s: &str) -> Vec { + s.lines() + .filter_map(|l| l.strip_prefix("data: ")) + .filter(|s| *s != "[DONE]") + .map(|s| serde_json::from_str(s).unwrap()) + .collect() +} +fn assert_output(client: ClientProtocol, stream: bool, body: &str) { + if stream { + let packets = packets(body); + let mut text = String::new(); + let mut thinking = String::new(); + let mut args = String::new(); + match client { + ClientProtocol::Chat => { + assert!(body.ends_with("data: [DONE]\n\n")); + for p in &packets { + if let Some(c) = p["choices"].as_array().and_then(|c| c.first()) { + if let Some(t) = c["delta"]["content"].as_str() { + text.push_str(t); + } + if let Some(t) = c["delta"]["reasoning_content"].as_str() { + thinking.push_str(t); + } + if let Some(calls) = c["delta"]["tool_calls"].as_array() { + for c in calls { + if let Some(t) = c["function"]["arguments"].as_str() { + args.push_str(t); + } + } + } + } + } + let usage = packets.last().unwrap(); + assert_eq!(usage["usage"]["prompt_tokens"], 10); + assert_eq!(usage["usage"]["completion_tokens"], 5); + assert_eq!( + packets[packets.len() - 2]["choices"][0]["finish_reason"], + "tool_calls" + ); + } + ClientProtocol::Responses => { + for (i, p) in packets.iter().enumerate() { + assert_eq!(p["sequence_number"].as_u64(), Some(i as u64)); + match p["type"].as_str() { + Some("response.output_text.delta") => { + text.push_str(p["delta"].as_str().unwrap()) + } + Some("response.reasoning_summary_text.delta") => { + thinking.push_str(p["delta"].as_str().unwrap()) + } + Some("response.function_call_arguments.delta") => { + args.push_str(p["delta"].as_str().unwrap()) + } + _ => {} + } + } + let last = packets.last().unwrap(); + assert_eq!(last["type"], "response.completed"); + assert_eq!(last["response"]["usage"]["input_tokens"], 10); + assert_eq!(last["response"]["usage"]["output_tokens"], 5); + assert_eq!(last["response"]["output"][2]["call_id"], "call_1"); + assert_ne!(last["response"]["output"][2]["id"], "call_1"); + } + } + assert_eq!(text, "鲸鱼"); + assert_eq!(thinking, "think"); + assert_eq!( + serde_json::from_str::(&args).unwrap(), + json!({"city":"上海"}) + ); + } else { + let v: Value = serde_json::from_str(body).unwrap(); + let args = match client { + ClientProtocol::Chat => { + let message = &v["choices"][0]["message"]; + assert_eq!(message["content"], "鲸鱼"); + assert_eq!(message["reasoning_content"], "think"); + assert_eq!(message["tool_calls"][0]["id"], "call_1"); + assert_eq!(v["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!(v["usage"]["prompt_tokens"], 10); + assert_eq!(v["usage"]["completion_tokens"], 5); + message["tool_calls"][0]["function"]["arguments"] + .as_str() + .unwrap() + } + ClientProtocol::Responses => { + assert_eq!(v["status"], "completed"); + assert_eq!(v["output"][0]["summary"][0]["text"], "think"); + assert_eq!(v["output"][1]["content"][0]["text"], "鲸鱼"); + assert_eq!(v["output"][2]["call_id"], "call_1"); + assert_eq!(v["usage"]["input_tokens"], 10); + assert_eq!(v["usage"]["output_tokens"], 5); + v["output"][2]["arguments"].as_str().unwrap() + } + }; + assert_eq!( + serde_json::from_str::(args).unwrap(), + json!({"city":"上海"}) + ); + } + assert!(!body.contains("upstream-test-secret")); + assert!(!body.contains("upstream-real")); +} + +#[tokio::test] +async fn sixteen_protocol_streaming_and_json_round_trips_use_real_tcp() { + for p in [ + UpstreamProtocol::Chat, + UpstreamProtocol::Responses, + UpstreamProtocol::Anthropic, + UpstreamProtocol::Gemini, + ] { + for client in [ClientProtocol::Chat, ClientProtocol::Responses] { + for stream in [false, true] { + let script = if stream { + let mut s = Script::sse(Vec::new()); + s.steps = streaming(p) + .chunks(3) + .map(|b| Step::Data(b.to_vec())) + .collect(); + s + } else { + Script::json(full(p)) + }; + let upstream = Upstream::start(script).await; + let money = Arc::new(Money::default()); + let limits = Limits::default(); + let relay = relay(&upstream, p, limits.clone(), money.clone()); + let id = context(); + let expected_id = id.request_id.clone(); + let (head, body, done) = + collect(relay.start(request(client, stream, &limits), id).unwrap()).await; + assert!(head.is_ok(), "{p:?} {client:?} {stream}: {head:?}"); + assert!( + done.error.is_none(), + "{p:?} {client:?} {stream}: {:?} {body}", + done.error + ); + assert_eq!(done.settlement, Settlement::Confirmed); + assert_eq!(done.report.outcome, Outcome::Completed); + assert_eq!(done.report.usage.input_tokens, Some(10)); + assert_eq!(done.report.usage.output_tokens, Some(5)); + assert_eq!(done.report.finish_reason, Some(FinishReason::ToolCalls)); + assert_eq!(done.report.attempts, 1); + assert_output(client, stream, &body); + assert_eq!(money.finals.lock().unwrap().len(), 1); + assert_eq!(money.pinned.lock().unwrap()[0], (expected_id, 42)); + assert_eq!(upstream.observed.hits.load(Ordering::SeqCst), 1); + let requests = upstream.observed.requests.lock().unwrap(); + let (headers, payload) = &requests[0]; + let auth_name = match p { + UpstreamProtocol::Anthropic => "x-api-key", + UpstreamProtocol::Gemini => "x-goog-api-key", + _ => "authorization", + }; + assert!(headers.to_ascii_lowercase().contains(auth_name)); + assert!(headers.contains("upstream-test-secret")); + match p { + UpstreamProtocol::Chat => { + assert_eq!(payload["model"], "upstream-real"); + if stream { + assert_eq!(payload["stream_options"]["include_usage"], true); + } + } + UpstreamProtocol::Responses | UpstreamProtocol::Anthropic => { + assert_eq!(payload["model"], "upstream-real") + } + UpstreamProtocol::Gemini => { + assert!(headers.contains("models/upstream-real:")); + assert_eq!(payload["contents"][0]["role"], "user"); + } + } + println!("round_trip provider={p:?} client={client:?} stream={stream} passed"); + } + } + } +} + +fn first_text(text: &str) -> Vec { + data(json!({"id":"live_1","choices":[{"index":0,"delta":{"role":"assistant","content":text}}]})) +} +fn final_text(text: &str) -> Vec { + cat(vec![ + data( + json!({"id":"live_1","choices":[{"index":0,"delta":{"content":text},"finish_reason":"stop"}]}), + ), + data(json!({"id":"live_1","choices":[],"usage":{"prompt_tokens":2,"completion_tokens":3}})), + b"data: [DONE]\n\n".to_vec(), + ]) +} +async fn until(body: &mut reqwest::Response, seen: &mut Vec, needle: &str) { + timeout(WAIT, async { + while !String::from_utf8_lossy(seen).contains(needle) { + seen.extend( + body.chunk() + .await + .unwrap() + .expect("stream ended before the expected chunk"), + ); + } + }) + .await + .unwrap(); +} +struct Frontend { + url: String, + reports: mpsc::Receiver, + task: JoinHandle<()>, +} +impl Drop for Frontend { + fn drop(&mut self) { + self.task.abort(); + } +} +impl Frontend { + async fn start(relay: Relay, client: ClientProtocol, limits: Limits) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/model", listener.local_addr().unwrap()); + let (tx, reports) = mpsc::channel(4); + let app = Router::new().route( + "/model", + post(move |bytes: Bytes| { + let relay = relay.clone(); + let limits = limits.clone(); + let tx = tx.clone(); + async move { + let result = Request::parse(client, &bytes, &limits) + .and_then(|r| relay.start(r, context())); + match result { + Err(e) => Response::builder() + .status(e.status()) + .body(Body::from(e.json().to_string())) + .unwrap(), + Ok(session) => { + let Session { + head, + body, + completion, + .. + } = session; + tokio::spawn(async move { + if let Ok(c) = completion.await { + let _ = tx.send(c).await; + } + }); + match head.await.unwrap() { + Ok(h) => Response::builder() + .status(h.status) + .header("content-type", h.content_type) + .body(Body::from_stream(body)) + .unwrap(), + Err(e) => Response::builder() + .status(e.status()) + .body(Body::from(e.json().to_string())) + .unwrap(), + } + } + } + } + }), + ); + let task = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Self { url, reports, task } + } +} + +#[tokio::test] +async fn downstream_receives_text_before_upstream_completes() { + let gate = Arc::new(Semaphore::new(0)); + let upstream = Upstream::start(Script { + steps: vec![ + Step::Data(first_text("before")), + Step::Wait(gate.clone()), + Step::Data(final_text("after")), + ], + ..Script::sse(Vec::new()) + }) + .await; + let money = Arc::new(Money::default()); + let l = Limits::default(); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut front = Frontend::start(relay.clone(), ClientProtocol::Chat, l).await; + let mut response = reqwest::Client::new() + .post(&front.url) + .json(&input(ClientProtocol::Chat, true)) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let mut seen = Vec::new(); + until(&mut response, &mut seen, "before").await; + assert_eq!(relay.active_requests(), 1); + assert!(money.finals.lock().unwrap().is_empty()); + assert!(!String::from_utf8_lossy(&seen).contains("[DONE]")); + gate.add_permits(1); + while let Some(c) = response.chunk().await.unwrap() { + seen.extend(c); + } + let done = timeout(WAIT, front.reports.recv()).await.unwrap().unwrap(); + assert!(done.error.is_none()); + let output = String::from_utf8(seen).unwrap(); + assert!(output.contains("after")); + assert!(output.contains("[DONE]")); +} + +#[tokio::test] +async fn actual_downstream_disconnect_cancels_upstream_and_finalizes_once() { + let upstream = Upstream::start(Script { + steps: vec![Step::Data(first_text("before")), Step::Sleep(WAIT)], + ..Script::sse(Vec::new()) + }) + .await; + let l = Limits::default(); + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut front = Frontend::start(relay.clone(), ClientProtocol::Chat, l).await; + let mut response = reqwest::Client::builder() + .http1_only() + .build() + .unwrap() + .post(&front.url) + .json(&input(ClientProtocol::Chat, true)) + .send() + .await + .unwrap(); + let mut seen = Vec::new(); + until(&mut response, &mut seen, "before").await; + drop(response); + let done = timeout(WAIT, front.reports.recv()).await.unwrap().unwrap(); + assert_eq!(done.error, Some(RelayError::Cancelled)); + assert_eq!(done.report.outcome, Outcome::Cancelled); + assert_eq!(done.report.usage, Usage::default()); + assert_eq!(done.settlement, Settlement::Confirmed); + assert_eq!(money.finals.lock().unwrap().len(), 1); + timeout(WAIT, async { + while !upstream.observed.disconnected.load(Ordering::SeqCst) { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert_eq!(relay.drain(Duration::ZERO).await.remaining, 0); +} + +#[tokio::test] +async fn cancel_during_reservation_waits_for_handle_then_finalizes_without_sending() { + let gate = Arc::new(Semaphore::new(0)); + let money = Arc::new(Money { + reserve_wait: Some(gate.clone()), + ..Money::default() + }); + let upstream = Upstream::start(Script::json(full(UpstreamProtocol::Chat))).await; + let l = Limits::default(); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let session = relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(); + timeout(WAIT, async { + while money.reserves.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + session.cancel.cancel(); + gate.add_permits(1); + let (head, _, done) = collect(session).await; + assert_eq!(head.unwrap_err(), RelayError::Cancelled); + assert_eq!(done.report.outcome, Outcome::Cancelled); + assert_eq!(upstream.observed.hits.load(Ordering::SeqCst), 0); + assert_eq!(money.finals.lock().unwrap().len(), 1); + assert_eq!(done.settlement, Settlement::Confirmed); +} + +#[tokio::test] +async fn admission_and_drain_are_bounded_and_do_not_abort_settlement() { + let upstream = Upstream::start(Script { + steps: vec![Step::Data(first_text("live")), Step::Sleep(WAIT)], + ..Script::sse(Vec::new()) + }) + .await; + let l = Limits { + max_in_flight: 1, + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut s = relay + .start(request(ClientProtocol::Chat, true, &l), context()) + .unwrap(); + timeout(WAIT, &mut s.head).await.unwrap().unwrap().unwrap(); + assert!(matches!( + relay.start(request(ClientProtocol::Chat, true, &l), context()), + Err(RelayError::Busy) + )); + let drained = relay.drain(Duration::ZERO).await; + assert_eq!( + drained, + DrainReport { + cancelled: 1, + remaining: 0 + } + ); + assert!(matches!( + relay.start(request(ClientProtocol::Chat, true, &l), context()), + Err(RelayError::Draining) + )); + let done = timeout(WAIT, s.completion).await.unwrap().unwrap(); + assert_eq!(done.error, Some(RelayError::Cancelled)); + assert_eq!(money.finals.lock().unwrap().len(), 1); + assert_eq!(relay.active_requests(), 0); +} + +#[tokio::test] +async fn rejected_billing_never_opens_upstream_and_never_changes_readiness() { + let upstream = Upstream::start(Script::json(full(UpstreamProtocol::Chat))).await; + let l = Limits::default(); + let money = Arc::new(Money { + reject: true, + ..Money::default() + }); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (head, body, done) = collect( + relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(), + ) + .await; + assert_eq!(head.unwrap_err(), RelayError::BillingUnavailable); + assert!(body.is_empty()); + assert_eq!(upstream.observed.hits.load(Ordering::SeqCst), 0); + assert!(money.finals.lock().unwrap().is_empty()); + assert!(!done.report.upstream_attempted); + // The module itself neither registers endpoints nor touches core readiness. + let public_source = include_str!("../src/http.rs"); + assert!(public_source.contains("SERVICE_UNAVAILABLE")); +} + +#[tokio::test] +async fn settlement_failure_suppresses_success_for_both_clients_and_modes() { + for client in [ClientProtocol::Chat, ClientProtocol::Responses] { + for stream in [false, true] { + let upstream = Upstream::start(if stream { + Script::sse(streaming(UpstreamProtocol::Chat)) + } else { + Script::json(full(UpstreamProtocol::Chat)) + }) + .await; + let l = Limits::default(); + let money = Arc::new(Money { + fail_finalize: true, + ..Money::default() + }); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (head, body, done) = + collect(relay.start(request(client, stream, &l), context()).unwrap()).await; + assert_eq!(done.error, Some(RelayError::SettlementPending)); + assert_eq!(done.settlement, Settlement::Pending); + assert!(!body.contains("[DONE]")); + assert!(!body.contains("response.completed")); + assert_eq!(money.finals.lock().unwrap().len(), 1); + if stream { + assert!(head.is_ok()); + assert!(body.contains("settlement_pending")); + } else { + assert_eq!(head.unwrap_err(), RelayError::SettlementPending); + assert!(body.is_empty()); + } + } + } +} + +#[tokio::test] +async fn non_stream_head_waits_for_settlement_and_hooks_have_a_deadline() { + let gate = Arc::new(Semaphore::new(0)); + let upstream = Upstream::start(Script::json(full(UpstreamProtocol::Chat))).await; + let l = Limits { + hook_timeout: Duration::from_millis(250), + ..Limits::default() + }; + let money = Arc::new(Money { + finalize_wait: Some(gate), + ..Money::default() + }); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut session = relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(); + timeout(WAIT, async { + while money.finals.lock().unwrap().is_empty() { + sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + assert!(matches!( + session.head.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + let (head, body, done) = collect(session).await; + assert_eq!(head.unwrap_err(), RelayError::SettlementPending); + assert!(body.is_empty()); + assert_eq!(done.settlement, Settlement::Pending); +} + +#[tokio::test] +async fn reservation_timeout_is_ambiguous_not_a_free_request() { + let upstream = Upstream::start(Script::json(full(UpstreamProtocol::Chat))).await; + let l = Limits { + hook_timeout: Duration::from_millis(150), + ..Limits::default() + }; + let money = Arc::new(Money { + reserve_wait: Some(Arc::new(Semaphore::new(0))), + ..Money::default() + }); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (head, _, done) = collect( + relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(), + ) + .await; + assert_eq!(head.unwrap_err(), RelayError::Timeout(Phase::Reserve)); + assert_eq!(done.settlement, Settlement::Pending); + assert_eq!(upstream.observed.hits.load(Ordering::SeqCst), 0); + assert!(money.finals.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn http_statuses_and_redirects_are_never_retried_or_leaked() { + for status in [302, 401, 429, 500, 503] { + let upstream = Upstream::start(Script { + status, + ..Script::json(json!({"error":"secret-upstream-body"})) + }) + .await; + let l = Limits { + connect_retries: 2, + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (head, body, done) = collect( + relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(), + ) + .await; + let e = head.unwrap_err(); + assert_eq!(e, RelayError::UpstreamStatus(status)); + assert!(!e.to_string().contains("secret")); + assert!(body.is_empty()); + assert_eq!(done.report.attempts, 1); + assert_eq!(upstream.observed.hits.load(Ordering::SeqCst), 1); + assert_eq!(money.finals.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn connect_retries_are_limited_to_the_configured_number() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let l = Limits { + connect_retries: 2, + retry_backoff: Duration::from_millis(20), + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = Relay::new( + Arc::new(Routes { + address, + protocol: UpstreamProtocol::Chat, + }), + money.clone(), + l.clone(), + ) + .unwrap(); + let (_, _, done) = collect( + relay + .start(request(ClientProtocol::Chat, false, &l), context()) + .unwrap(), + ) + .await; + assert_eq!(done.error, Some(RelayError::UpstreamConnection)); + assert_eq!(done.report.attempts, 3); + assert_eq!(money.finals.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn header_read_and_total_deadlines_are_distinct() { + let cases = [ + ( + Phase::Headers, + Script { + header_delay: Duration::from_secs(2), + ..Script::sse(Vec::new()) + }, + Limits { + header_timeout: Duration::from_millis(150), + ..Limits::default() + }, + ), + ( + Phase::Read, + Script { + steps: vec![ + Step::Data(first_text("live")), + Step::Sleep(Duration::from_secs(2)), + ], + ..Script::sse(Vec::new()) + }, + Limits { + read_timeout: Duration::from_millis(150), + ..Limits::default() + }, + ), + ( + Phase::Total, + Script { + steps: (0..20) + .flat_map(|_| { + [ + Step::Data(b": heartbeat\n\n".to_vec()), + Step::Sleep(Duration::from_millis(50)), + ] + }) + .collect(), + ..Script::sse(Vec::new()) + }, + Limits { + total_timeout: Duration::from_millis(250), + read_timeout: Duration::from_millis(200), + ..Limits::default() + }, + ), + ]; + for (phase, script, l) in cases { + let upstream = Upstream::start(script).await; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (_, body, done) = collect( + relay + .start(request(ClientProtocol::Chat, true, &l), context()) + .unwrap(), + ) + .await; + assert_eq!(done.error, Some(RelayError::Timeout(phase))); + assert_eq!(done.report.outcome, Outcome::TimedOut); + assert_eq!(done.report.attempts, 1); + assert!(!body.contains("[DONE]")); + assert_eq!(money.finals.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn a_slow_client_cannot_create_an_unbounded_queue() { + let upstream = Upstream::start(Script::sse(streaming(UpstreamProtocol::Chat))).await; + let l = Limits { + queue_chunks: 1, + chunk_bytes: 32, + write_timeout: Duration::from_millis(150), + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut session = relay + .start(request(ClientProtocol::Chat, true, &l), context()) + .unwrap(); + timeout(WAIT, &mut session.head) + .await + .unwrap() + .unwrap() + .unwrap(); + // Deliberately retain, but do not poll, the body until the worker terminates. + let done = timeout(WAIT, &mut session.completion) + .await + .unwrap() + .unwrap(); + assert_eq!(done.error, Some(RelayError::Timeout(Phase::Write))); + assert!(done.report.emitted_bytes <= 32); + let mut count = 0; + while let Some(chunk) = session.body.next_chunk().await { + assert!(chunk.unwrap().len() <= 32); + count += 1; + } + assert_eq!(count, 1); + assert_eq!(relay.drain(Duration::ZERO).await.remaining, 0); + assert_eq!(money.finals.lock().unwrap().len(), 1); +} + +#[tokio::test] +async fn event_output_tool_and_raw_body_limits_are_enforced() { + let base = Limits { + output_bytes: 2048, + event_bytes: 1024, + tool_argument_bytes: 64, + ..Limits::default() + }; + let mut argument = full(UpstreamProtocol::Chat); + argument["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = + json!({"city":"x".repeat(100)}).to_string().into(); + let cases = [ + ( + Script::sse([b"data: ".as_slice(), vec![b'x'; 2048].as_slice()].concat()), + true, + base.clone(), + ), + ( + Script::sse(cat((0..30).map(|_| first_text(&"x".repeat(100))).collect())), + true, + base.clone(), + ), + (Script::json(argument), false, base.clone()), + ( + Script::json(json!({"padding":"x".repeat(4096)})), + false, + base.clone(), + ), + ]; + for (script, stream, l) in cases { + let upstream = Upstream::start(script).await; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (_, body, done) = collect( + relay + .start(request(ClientProtocol::Chat, stream, &l), context()) + .unwrap(), + ) + .await; + assert!( + matches!(done.error, Some(RelayError::Limit(_))), + "{:?}", + done.error + ); + assert!(!body.contains("[DONE]")); + assert_eq!(money.finals.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn missing_terminal_invalid_utf8_and_broken_http_fail_without_replay() { + let cases = [ + Script::sse(first_text("partial")), + Script { + complete: false, + steps: vec![Step::Data(first_text("partial"))], + ..Script::sse(Vec::new()) + }, + Script::sse(b"data: \xff\n\n".to_vec()), + Script { + content_type: "text/html", + ..Script::sse(b"secret error page".to_vec()) + }, + ]; + for script in cases { + let upstream = Upstream::start(script).await; + let l = Limits { + connect_retries: 2, + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (_, body, done) = collect( + relay + .start(request(ClientProtocol::Chat, true, &l), context()) + .unwrap(), + ) + .await; + assert!(done.error.is_some()); + assert_ne!(done.report.outcome, Outcome::Completed); + assert_eq!(done.report.attempts, 1); + assert!(!body.contains("[DONE]")); + assert!(!body.contains("secret error page")); + assert_eq!(money.finals.lock().unwrap().len(), 1); + } +} + +#[tokio::test] +async fn missing_usage_stays_unknown_and_partial_tools_only_allow_incomplete_finish() { + for reason in ["length", "tool_calls"] { + let mut v = full(UpstreamProtocol::Chat); + v.as_object_mut().unwrap().remove("usage"); + v["choices"][0]["finish_reason"] = json!(reason); + v["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"] = json!("{\"city\":"); + let upstream = Upstream::start(Script::json(v)).await; + let l = Limits::default(); + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let (_, body, done) = collect( + relay + .start(request(ClientProtocol::Responses, false, &l), context()) + .unwrap(), + ) + .await; + assert_eq!(done.report.usage, Usage::default()); + assert_eq!(money.finals.lock().unwrap()[0].usage, Usage::default()); + if reason == "length" { + assert!(done.error.is_none()); + let v: Value = serde_json::from_str(&body).unwrap(); + assert_eq!(v["status"], "incomplete"); + assert!(v["usage"].is_null()); + } else { + assert!(matches!(done.error, Some(RelayError::Protocol(_)))); + } + } +} + +#[tokio::test] +async fn immutable_request_history_is_converted_without_losing_tool_results() { + for p in [ + UpstreamProtocol::Chat, + UpstreamProtocol::Responses, + UpstreamProtocol::Anthropic, + UpstreamProtocol::Gemini, + ] { + let upstream = Upstream::start(Script::json(full(p))).await; + let l = Limits::default(); + let money = Arc::new(Money::default()); + let relay = relay(&upstream, p, l.clone(), money); + let body = json!({"model":"public-model","messages":[{"role":"system","content":"rules"},{"role":"user","content":"weather?"}, + {"role":"assistant","tool_calls":[{"type":"function","id":"call_old","function":{"name":"weather","arguments":"{\"city\":\"上海\"}"}}]}, + {"role":"tool","tool_call_id":"call_old","content":"sunny"}],"max_tokens":100}); + let r = Request::parse( + ClientProtocol::Chat, + &serde_json::to_vec(&body).unwrap(), + &l, + ) + .unwrap(); + let (_, _, done) = collect(relay.start(r, context()).unwrap()).await; + assert!(done.error.is_none(), "{p:?}: {:?}", done.error); + let observed = upstream.observed.requests.lock().unwrap(); + let v = &observed[0].1; + match p { + UpstreamProtocol::Chat => { + assert_eq!(v["messages"][3]["tool_call_id"], "call_old"); + assert_eq!(v["messages"][3]["content"], "sunny"); + } + UpstreamProtocol::Responses => { + assert_eq!(v["input"][3]["call_id"], "call_old"); + assert_eq!(v["input"][3]["output"], "sunny"); + } + UpstreamProtocol::Anthropic => { + assert_eq!(v["system"][0]["text"], "rules"); + assert_eq!(v["messages"][2]["content"][0]["tool_use_id"], "call_old"); + } + UpstreamProtocol::Gemini => { + assert_eq!(v["systemInstruction"]["parts"][0]["text"], "rules"); + assert_eq!( + v["contents"][2]["parts"][0]["functionResponse"]["name"], + "weather" + ); + assert_eq!( + v["contents"][2]["parts"][0]["functionResponse"]["response"]["output"], + "sunny" + ); + } + } + } +} + +#[test] +fn unsupported_or_ambiguous_input_and_invalid_routes_fail_closed() { + let l = Limits::default(); + for v in [ + json!({"model":"x","messages":[],"n":2}), + json!({"model":"x","messages":[],"unknown":true}), + json!({"model":"x","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.com/x"}}]}]}), + json!({"model":"x","messages":[{"role":"tool","tool_call_id":"orphan","content":"result"}]}), + json!({"model":"x","messages":[],"max_tokens":1,"max_completion_tokens":2}), + json!({"model":"x","messages":[],"store":true}), + ] { + assert!( + Request::parse(ClientProtocol::Chat, &serde_json::to_vec(&v).unwrap(), &l).is_err() + ); + } + for url in [ + "file:///tmp/key", + "http://example.com/v1", + "https://user:password@example.com/v1", + "https://example.com/v1?key=secret", + "https://example.com/v1#fragment", + ] { + assert!( + Route::new( + "r".into(), + UpstreamProtocol::Chat, + url, + "m".into(), + "test", + 1 + ) + .is_err() + ); + } + let route = Route::new( + "r".into(), + UpstreamProtocol::Chat, + "https://example.com/v1", + "m".into(), + "hidden-key", + 1, + ) + .unwrap(); + assert!(!format!("{route:?}").contains("hidden-key")); + assert!(!format!("{route:?}").contains("example.com")); +} + +struct Process(Child); +impl Drop for Process { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} +#[tokio::test] +async fn stopping_the_actual_go_extension_does_not_cut_a_rust_http_stream() { + let binary = std::env::var("LMM_RELAY_EXTENSION_BIN").expect("build apps/api-go/cmd/extensions and set LMM_RELAY_EXTENSION_BIN; this acceptance test must not be skipped"); + let dir = std::env::temp_dir().join(format!( + "lmm-relay-go-{}-{}", + std::process::id(), + IDS.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir_all(&dir).unwrap(); + let token = dir.join("extension-token"); + std::fs::write(&token, b"relay-test-only-credential-0123456789abcdef").unwrap(); + let port = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let address = port.local_addr().unwrap(); + drop(port); + let mut command = Command::new(binary); + command + .env("LMM_EXTENSION_TOKEN_FILE", &token) + .env("LMM_EXTENSION_LISTEN", address.to_string()) + .env_remove("LMM_CORE_RPC_SOCKET") + .env_remove("LMM_CORE_RPC_TOKEN_FILE") + .env_remove("LMM_EXTENSION_MODULES") + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + for name in [ + "SQL_DSN", + "LOG_SQL_DSN", + "DATABASE_URL", + "LMM_CORE_DATABASE_URL", + "LMM_CORE_DATABASE_URL_FILE", + "LMM_DB_MIGRATION_MODE", + ] { + command.env_remove(name); + } + let mut process = Process(command.spawn().unwrap()); + let client = reqwest::Client::builder() + .timeout(Duration::from_secs(2)) + .build() + .unwrap(); + timeout(WAIT, async { + loop { + assert!( + process.0.try_wait().unwrap().is_none(), + "the real Go extension exited before the test" + ); + if client + .get(format!("http://{address}/healthz")) + .send() + .await + .is_ok() + { + break; + } + sleep(Duration::from_millis(25)).await; + } + }) + .await + .unwrap(); + let gate = Arc::new(Semaphore::new(0)); + let upstream = Upstream::start(Script { + steps: vec![ + Step::Data(first_text("before_go_stop")), + Step::Wait(gate.clone()), + Step::Data(final_text("after_go_stop")), + ], + ..Script::sse(Vec::new()) + }) + .await; + let l = Limits::default(); + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, l.clone(), money.clone()); + let mut front = Frontend::start(relay.clone(), ClientProtocol::Chat, l).await; + let mut response = reqwest::Client::new() + .post(&front.url) + .json(&input(ClientProtocol::Chat, true)) + .send() + .await + .unwrap(); + let mut seen = Vec::new(); + until(&mut response, &mut seen, "before_go_stop").await; + assert_eq!(relay.active_requests(), 1); + let pid = process.0.id(); + process.0.kill().unwrap(); + assert!(!process.0.wait().unwrap().success()); + assert_eq!(relay.active_requests(), 1); + gate.add_permits(1); + while let Some(bytes) = timeout(WAIT, response.chunk()).await.unwrap().unwrap() { + seen.extend(bytes); + } + let done = timeout(WAIT, front.reports.recv()).await.unwrap().unwrap(); + let text = String::from_utf8(seen).unwrap(); + assert!(text.contains("after_go_stop")); + assert!(text.ends_with("data: [DONE]\n\n")); + assert!(done.error.is_none()); + assert_eq!(done.report.usage.input_tokens, Some(2)); + assert_eq!(done.report.usage.output_tokens, Some(3)); + assert_eq!(money.finals.lock().unwrap().len(), 1); + assert_eq!(relay.drain(Duration::ZERO).await.remaining, 0); + std::fs::remove_dir_all(dir).unwrap(); + println!( + "actual Go extension pid={pid} killed after first HTTP output; Rust emitted later output and settled once; no Rust restart was performed" + ); +} + +#[tokio::test] +async fn error_packets_obey_the_same_chunk_limit_as_model_output() { + let upstream = Upstream::start(Script::sse(first_text("partial"))).await; + let limits = Limits { + queue_chunks: 2, + chunk_bytes: 17, + ..Limits::default() + }; + let money = Arc::new(Money::default()); + let relay = relay(&upstream, UpstreamProtocol::Chat, limits.clone(), money); + let Session { + head, + mut body, + completion, + .. + } = relay + .start(request(ClientProtocol::Chat, true, &limits), context()) + .unwrap(); + timeout(WAIT, head).await.unwrap().unwrap().unwrap(); + let mut bytes = Vec::new(); + while let Some(chunk) = timeout(WAIT, body.next_chunk()).await.unwrap() { + let chunk = chunk.unwrap(); + assert!(chunk.len() <= 17); + bytes.extend(chunk); + } + let done = timeout(WAIT, completion).await.unwrap().unwrap(); + assert_eq!(done.error, Some(RelayError::Truncated)); + let body = String::from_utf8(bytes).unwrap(); + assert!(body.contains("invalid_upstream_response")); + assert!(!body.contains("[DONE]")); +}