diff --git a/Cargo.lock b/Cargo.lock index b99df352..7fa8a031 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -29,6 +41,12 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "base64" version = "0.22.1" @@ -56,6 +74,22 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -68,6 +102,23 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + [[package]] name = "clipboard-win" version = "5.4.1" @@ -77,6 +128,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "cranelift-assembler-x64" version = "0.129.1" @@ -243,6 +303,17 @@ version = "0.129.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d953932541249c91e3fa70a75ff1e52adc62979a2a8132145d4b9b3e6d1a9b6a" +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "endian-type" version = "0.1.2" @@ -271,6 +342,18 @@ version = "3.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fd-lock" version = "4.0.4" @@ -282,6 +365,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "fnv" version = "1.0.7" @@ -294,6 +383,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + [[package]] name = "futures-channel" version = "0.3.32" @@ -305,9 +403,81 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", +] [[package]] name = "gimli" @@ -321,6 +491,15 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -336,6 +515,15 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -351,6 +539,207 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + [[package]] name = "indexmap" version = "2.13.0" @@ -361,12 +750,29 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + [[package]] name = "itoa" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -385,18 +791,41 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "mach2" version = "0.4.3" @@ -412,6 +841,17 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -429,10 +869,16 @@ checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ "bitflags 2.11.0", "cfg-if", - "cfg_aliases", + "cfg_aliases 0.1.1", "libc", ] +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + [[package]] name = "paste" version = "1.0.15" @@ -446,7 +892,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9676d58588b220f7af69d7aa86108042d2acaf21dd24c641a6d9ef3c4e193ba" dependencies = [ "pd-host-function 0.22.2", - "syn", + "syn 2.0.117", ] [[package]] @@ -455,7 +901,7 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -466,7 +912,7 @@ checksum = "d9c941589fbbb839a40f7b80595d7b8f3742a8811268d787218f0c45c274d1f9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -480,18 +926,22 @@ dependencies = [ "cranelift-module", "cranelift-native", "futures-channel", + "futures-util", "libc", "paste", "pd-edge-abi", "pd-host-function 0.1.0", "regex", + "reqwest", "rt-format", + "rusqlite", "rustyline", "self_cell", "serde", "serde_json", - "syn", + "syn 2.0.117", "tokio", + "url", "windows-sys 0.59.0", ] @@ -512,12 +962,33 @@ dependencies = [ "serde_json", ] +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + [[package]] name = "pin-project-lite" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -528,22 +999,110 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.44" +name = "quinn" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ - "proc-macro2", + "bytes", + "cfg_aliases 0.2.2", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", ] [[package]] -name = "radix_trie" -version = "0.2.1" +name = "quinn-proto" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" dependencies = [ - "endian-type", - "nibble_vec", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases 0.2.2", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", ] [[package]] @@ -601,6 +1160,61 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rt-format" version = "0.3.1" @@ -611,6 +1225,20 @@ dependencies = [ "regex", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.11.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -630,6 +1258,41 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustscript" version = "0.1.0" @@ -637,6 +1300,12 @@ dependencies = [ "pd-vm", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "rustyline" version = "14.0.0" @@ -659,6 +1328,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "self_cell" version = "1.2.2" @@ -692,7 +1367,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -708,18 +1383,58 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.117" @@ -731,20 +1446,101 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "target-lexicon" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ + "bytes", + "libc", + "mio", "pin-project-lite", + "socket2", "tokio-macros", + "windows-sys 0.61.2", ] [[package]] @@ -755,9 +1551,102 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.11.0", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -776,12 +1665,131 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.117", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmtime-internal-core" version = "42.0.1" @@ -803,6 +1811,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "windows-link" version = "0.2.1" @@ -900,6 +1937,115 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 5cfc571d..2cda5b84 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,8 @@ name = "vm" [features] default = ["runtime", "cli", "cranelift-jit"] runtime = [] +http-client = ["runtime", "dep:reqwest", "dep:url", "dep:tokio", "dep:futures-util"] +sqlite = ["runtime", "dep:rusqlite"] edge-abi = [ "dep:edge_abi", "edge_abi/console", @@ -60,6 +62,11 @@ cranelift-jit = { version = "0.129.1", optional = true } cranelift-module = { version = "0.129.1", optional = true } cranelift-native = { version = "0.129.1", optional = true } pd-host-function = { path = "./pd-host-function", version = "0.1.0" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "stream"], optional = true } +rusqlite = { version = "0.32", default-features = false, features = ["bundled", "hooks", "limits"], optional = true } +url = { version = "2", optional = true } +futures-util = { version = "0.3", optional = true } +tokio = { version = "1", features = ["rt-multi-thread", "net", "time", "sync"], optional = true } edge_abi = { package = "pd-edge-abi", version = "0.1.1", default-features = false, optional = true } futures-channel = "0.3" paste = "1" @@ -77,6 +84,7 @@ windows-sys = { version = "0.59", features = ["Win32_System_Diagnostics_Debug", libc = "0.2" [dev-dependencies] +futures-util = "0.3" syn = { version = "2", features = ["full"] } tokio = { version = "1", features = ["macros", "rt", "time", "sync"] } @@ -85,5 +93,15 @@ name = "host_binding_generation_tests" path = "tests/host_binding_generation_tests.rs" required-features = ["cranelift-jit"] +[[test]] +name = "http_host_tests" +path = "tests/vm/http_host_tests.rs" +required-features = ["runtime", "http-client"] + +[[test]] +name = "sqlite_host_tests" +path = "tests/vm/sqlite_host_tests.rs" +required-features = ["sqlite"] + [build-dependencies] syn = { version = "2", features = ["full"] } diff --git a/build.rs b/build.rs index 09cfa5ac..64979a9c 100644 --- a/build.rs +++ b/build.rs @@ -149,11 +149,30 @@ fn main() { println!("cargo:rerun-if-changed={}", catalog_path.display()); let catalog = parse_catalog(&catalog_path); - let host_sources = [SourceSpec { - path: "src/builtins/runtime/host.rs".to_string(), - module: "host".to_string(), - category: SourceCategory::DefaultHost, - }]; + let mut host_sources = vec![ + SourceSpec { + path: "src/builtins/runtime/host.rs".to_string(), + module: "host".to_string(), + category: SourceCategory::DefaultHost, + }, + SourceSpec { + path: "src/builtins/runtime/http.rs".to_string(), + module: "http".to_string(), + category: SourceCategory::DefaultHost, + }, + SourceSpec { + path: "src/builtins/runtime/context_host.rs".to_string(), + module: "context_host".to_string(), + category: SourceCategory::DefaultHost, + }, + ]; + if env::var_os("CARGO_FEATURE_SQLITE").is_some() { + host_sources.push(SourceSpec { + path: "src/builtins/runtime/sqlite.rs".to_string(), + module: "sqlite".to_string(), + category: SourceCategory::DefaultHost, + }); + } let builtin_sources = builtin_source_specs(&namespaces); let core_sources = [SourceSpec { path: "src/builtins/runtime/core.rs".to_string(), @@ -934,6 +953,7 @@ fn render_builtin_catalog( writeln!(&mut out, "impl BuiltinFunction {{").unwrap(); render_builtin_name_method(&mut out, &builtin_variant_order, &actual_builtin_by_variant); + render_builtin_capability_method(&mut out, builtin_callables); render_builtin_arity_method(&mut out, &builtin_variant_order, &actual_builtin_by_variant); render_builtin_accepts_arity_method( &mut out, @@ -1075,6 +1095,12 @@ fn render_builtin_runtime_dispatch( ) .unwrap(); } + writeln!( + &mut out, + " registry.mark_runtime_owned_pending({:?});", + callable.name + ) + .unwrap(); } writeln!(&mut out, "}}").unwrap(); writeln!(&mut out).unwrap(); @@ -1091,6 +1117,12 @@ fn render_builtin_runtime_dispatch( .render_bind_static_call(&callable.name, &host_wrapper_adapter_name(callable)); writeln!(&mut out, " {:?} => {{", callable.name).unwrap(); writeln!(&mut out, " {bind_call}").unwrap(); + writeln!( + &mut out, + " vm.mark_runtime_owned_pending_binding({:?});", + callable.name + ) + .unwrap(); writeln!(&mut out, " true").unwrap(); writeln!(&mut out, " }}").unwrap(); } @@ -1408,6 +1440,35 @@ fn render_builtin_name_method( writeln!(out).unwrap(); } +fn render_builtin_capability_method(out: &mut String, builtin_callables: &[CallableDecl]) { + let mut capability_variants = Vec::new(); + for callable in builtin_callables { + let variant = builtin_variant_name(&callable.name); + if !capability_variants.contains(&variant) { + capability_variants.push(variant); + } + } + capability_variants.sort(); + writeln!(out, " #[cfg(feature = \"runtime\")]").unwrap(); + writeln!( + out, + " pub(crate) const fn requires_explicit_host_capability(self) -> bool {{" + ) + .unwrap(); + if capability_variants.is_empty() { + writeln!(out, " false").unwrap(); + } else { + let patterns = capability_variants + .iter() + .map(|variant| format!("BuiltinFunction::{variant}")) + .collect::>() + .join(" | "); + writeln!(out, " matches!(self, {patterns})").unwrap(); + } + writeln!(out, " }}").unwrap(); + writeln!(out).unwrap(); +} + fn render_builtin_arity_method( out: &mut String, builtin_variant_order: &[String], diff --git a/crates/rustscript/Cargo.toml b/crates/rustscript/Cargo.toml index 6279a1e1..8d72371c 100644 --- a/crates/rustscript/Cargo.toml +++ b/crates/rustscript/Cargo.toml @@ -16,6 +16,8 @@ runtime = ["pd_vm_crate/runtime"] edge-abi = ["pd_vm_crate/edge-abi"] cli = ["pd_vm_crate/cli"] cranelift-jit = ["pd_vm_crate/cranelift-jit"] +http-client = ["runtime", "pd_vm_crate/http-client"] +sqlite = ["pd_vm_crate/sqlite"] [dependencies] -pd_vm_crate = { package = "pd-vm", path = "../..", version = ">=0.1.0, <1.0.0" } +pd_vm_crate = { package = "pd-vm", path = "../..", version = "=0.1.0", default-features = false } diff --git a/crates/rustscript/tests/alias_smoke.rs b/crates/rustscript/tests/alias_smoke.rs index c58bb302..1a63ee26 100644 --- a/crates/rustscript/tests/alias_smoke.rs +++ b/crates/rustscript/tests/alias_smoke.rs @@ -21,3 +21,38 @@ fn alias_exports_op_code() { let _ = rustscript::OpCode::Nop; let _ = rustscript::OpCode::Add; } + +#[cfg(feature = "runtime")] +#[test] +fn alias_exports_public_runtime_event_contract() { + fn accept_sink(_sink: S) {} + + struct Sink; + impl rustscript::EventSink for Sink { + fn emit(&mut self, _payload: rustscript::EventPayload) -> rustscript::RuntimeResult<()> { + Ok(()) + } + } + + accept_sink(Sink); +} + +#[cfg(feature = "http-client")] +#[test] +fn alias_http_client_includes_runtime_contract() { + fn accept_runtime_result(_result: rustscript::RuntimeResult<()>) {} + + accept_runtime_result(Ok(())); +} + +#[cfg(feature = "sqlite")] +#[test] +fn alias_exports_public_sqlite_configuration() { + let program = rustscript::compile_source("0;") + .expect("minimal alias SQLite program should compile") + .program; + let mut vm = rustscript::Vm::new(program); + vm.configure_sqlite(rustscript::SqlitePolicy::default()); + let _limits = rustscript::SqliteLimits::default(); + vm.clear_sqlite_configuration(); +} diff --git a/docs/callable-runtime.md b/docs/callable-runtime.md index ed0cfd4b..d71033d8 100644 --- a/docs/callable-runtime.md +++ b/docs/callable-runtime.md @@ -9,7 +9,7 @@ RustScript bytecode format version 11 (VMBC v11) introduces runtime script call - callable environments are bound through the internal builtin call path; callable creation adds no bytecode opcode. - `ret` completes the active script frame. A nested frame leaves exactly one result at the caller segment base, using `null` when the body produced no value. Root `ret` keeps the historical program-result stack behavior. -VMBC v11 is a hard format boundary. Decoders reject all earlier versions (v10 and below) with a deterministic unsupported-version error; there is no compatibility decoder and no old-ID alias. The stream includes script-function entry ranges, callable prototypes, function regions, root callable bindings, and call indices drawn from the static builtin catalog. PDRC v6 recordings and AOT artifacts (format 7, ABI 6) use their corresponding bumped versions and include callable metadata in cache identity. +VMBC v11 is a hard format boundary. Decoders reject all earlier versions (v10 and below) with a deterministic unsupported-version error; there is no compatibility decoder and no old-ID alias. The stream includes script-function entry ranges, callable prototypes, function regions, root callable bindings, and call indices drawn from the static builtin catalog. PDRC v6 recordings and AOT artifacts (format 7, ABI 7) use their corresponding bumped versions and include callable metadata in cache identity. ## Static builtin IDs diff --git a/pd-host-function/src/edge.rs b/pd-host-function/src/edge.rs new file mode 100644 index 00000000..4f6c268a --- /dev/null +++ b/pd-host-function/src/edge.rs @@ -0,0 +1,1162 @@ +use quote::{format_ident, quote}; +use syn::{ + Error, Expr, FnArg, Ident, ItemFn, LitStr, Meta, Pat, PatIdent, ReturnType, Token, Type, + punctuated::Punctuated, +}; + +pub(crate) fn expand_scoped_pd_host_function( + attr: Punctuated, + mut item: ItemFn, +) -> Result { + let edge_attr = parse_edge_host_attr(&attr)?; + let was_async = item.sig.asyncness.is_some(); + let docs = doc_string(&item.attrs); + if docs.trim().is_empty() { + return Err(Error::new_spanned( + &item.sig.ident, + "#[pd_host_function] requires /// doc comments", + )); + } + if edge_attr.scope.is_some() && !edge_attr.bind_params.is_empty() { + return Err(Error::new_spanned( + &item.sig.ident, + "scoped pd_host_function does not support bind(...); scoped registrations must be self-contained", + )); + } + + transform_async_edge_function(&mut item, &edge_attr)?; + validate_edge_bind_names(&item, &edge_attr.bind_params)?; + for input in &item.sig.inputs { + validate_edge_param(input, &edge_attr.bind_params)?; + } + validate_edge_return_type(&item.sig.output)?; + + let (wrapper_name, impl_name) = wrapper_and_impl_names(&item.sig.ident); + if item.sig.ident != impl_name { + item.sig.ident = impl_name.clone(); + } + let wrapper = generate_edge_host_binder(&item, &wrapper_name, &edge_attr)?; + let static_wrapper = + generate_scoped_edge_host_static_wrapper(&item, &wrapper_name, &edge_attr, was_async)?; + let registration = generate_edge_host_registration(&item, &wrapper_name, &edge_attr, &docs)?; + Ok(quote! { + #item + #wrapper + #static_wrapper + #registration + }) +} + +fn doc_string(attrs: &[syn::Attribute]) -> String { + attrs + .iter() + .filter_map(|attr| { + if !attr.path().is_ident("doc") { + return None; + } + match &attr.meta { + Meta::NameValue(name_value) => match &name_value.value { + syn::Expr::Lit(expr_lit) => match &expr_lit.lit { + syn::Lit::Str(value) => Some(value.value().trim().to_string()), + _ => None, + }, + _ => None, + }, + _ => None, + } + }) + .filter(|line| !line.is_empty()) + .collect::>() + .join("\n") +} + +struct EdgeHostAttr { + name: Expr, + scope: Option, + bind_params: Vec, +} + +#[derive(Clone, Copy)] +enum EdgeHostScopeAttr { + Runtime, + Http, + HttpExtension, + Io, + Transport, + Mqtt, + WebSocket, + WebRtc, + Proxy, + Console, +} + +fn parse_edge_host_attr(args: &Punctuated) -> Result { + let mut name = None; + let mut scope = None; + let mut bind_params = Vec::new(); + + for meta in args { + match meta { + Meta::NameValue(name_value) if name_value.path.is_ident("name") => { + if name.is_some() { + return Err(Error::new_spanned( + name_value, + "duplicate name argument in #[pd_host_function(...)]", + )); + } + name = Some(name_value.value.clone()); + } + Meta::NameValue(name_value) if name_value.path.is_ident("scope") => { + if scope.is_some() { + return Err(Error::new_spanned( + name_value, + "duplicate scope argument in #[pd_host_function(...)]", + )); + } + scope = Some(parse_edge_scope(&name_value.value)?); + } + Meta::List(list) if list.path.is_ident("bind") => { + let idents = + list.parse_args_with(Punctuated::::parse_terminated)?; + bind_params.extend(idents.into_iter()); + } + other => { + return Err(Error::new_spanned( + other, + "expected #[pd_host_function(name = ..., scope = ..., bind(...))]", + )); + } + } + } + + let Some(name) = name else { + return Err(Error::new( + proc_macro2::Span::call_site(), + "expected #[pd_host_function(name = ..., scope = ..., bind(...))]", + )); + }; + + Ok(EdgeHostAttr { + name, + scope, + bind_params, + }) +} + +fn parse_edge_scope(value: &Expr) -> Result { + let scope_name = match value { + Expr::Path(path) => { + let Some(segment) = path.path.segments.last() else { + return Err(Error::new_spanned( + value, + "scope must be one of runtime, http, http_extension, io, transport, mqtt, websocket, webrtc, proxy, or console", + )); + }; + if path.path.segments.len() != 1 { + return Err(Error::new_spanned( + value, + "scope must be one of runtime, http, http_extension, io, transport, mqtt, websocket, webrtc, proxy, or console", + )); + } + segment.ident.to_string() + } + Expr::Lit(expr_lit) => match &expr_lit.lit { + syn::Lit::Str(value) => value.value(), + _ => { + return Err(Error::new_spanned( + value, + "scope must be one of runtime, http, http_extension, io, transport, mqtt, websocket, webrtc, proxy, or console", + )); + } + }, + _ => { + return Err(Error::new_spanned( + value, + "scope must be one of runtime, http, http_extension, io, transport, mqtt, websocket, webrtc, proxy, or console", + )); + } + }; + + match scope_name.as_str() { + "runtime" => Ok(EdgeHostScopeAttr::Runtime), + "http" => Ok(EdgeHostScopeAttr::Http), + "http_extension" | "http_extensions" => Ok(EdgeHostScopeAttr::HttpExtension), + "io" | "io_override" | "io_overrides" => Ok(EdgeHostScopeAttr::Io), + "transport" => Ok(EdgeHostScopeAttr::Transport), + "mqtt" => Ok(EdgeHostScopeAttr::Mqtt), + "websocket" => Ok(EdgeHostScopeAttr::WebSocket), + "webrtc" => Ok(EdgeHostScopeAttr::WebRtc), + "proxy" => Ok(EdgeHostScopeAttr::Proxy), + "console" => Ok(EdgeHostScopeAttr::Console), + _ => Err(Error::new_spanned( + value, + "scope must be one of runtime, http, http_extension, io, transport, mqtt, websocket, webrtc, proxy, or console", + )), + } +} + +fn edge_scope_tokens(scope: EdgeHostScopeAttr) -> proc_macro2::TokenStream { + match scope { + EdgeHostScopeAttr::Runtime => { + quote!(crate::abi_impl::registry::EdgeHostScope::Runtime) + } + EdgeHostScopeAttr::Http => quote!(crate::abi_impl::registry::EdgeHostScope::Http), + EdgeHostScopeAttr::HttpExtension => { + quote!(crate::abi_impl::registry::EdgeHostScope::HttpExtension) + } + EdgeHostScopeAttr::Io => quote!(crate::abi_impl::registry::EdgeHostScope::Io), + EdgeHostScopeAttr::Transport => { + quote!(crate::abi_impl::registry::EdgeHostScope::Transport) + } + EdgeHostScopeAttr::Mqtt => quote!(crate::abi_impl::registry::EdgeHostScope::Mqtt), + EdgeHostScopeAttr::WebSocket => { + quote!(crate::abi_impl::registry::EdgeHostScope::WebSocket) + } + EdgeHostScopeAttr::WebRtc => { + quote!(crate::abi_impl::registry::EdgeHostScope::WebRtc) + } + EdgeHostScopeAttr::Proxy => { + quote!(crate::abi_impl::registry::EdgeHostScope::Proxy) + } + EdgeHostScopeAttr::Console => { + quote!(crate::abi_impl::registry::EdgeHostScope::Console) + } + } +} + +fn find_context_param_ident(item: &ItemFn) -> Option { + item.sig.inputs.iter().find_map(|input| { + let FnArg::Typed(pat_type) = input else { + return None; + }; + if !is_edge_context_type(&pat_type.ty) { + return None; + } + match pat_type.pat.as_ref() { + Pat::Ident(PatIdent { ident, .. }) => Some(ident.clone()), + _ => None, + } + }) +} + +fn async_scope_prepare_stmt( + item: &ItemFn, + attr: &EdgeHostAttr, +) -> Result { + let Some(scope) = attr.scope else { + return Ok(quote!()); + }; + let requires_prepare = matches!( + scope, + EdgeHostScopeAttr::Http | EdgeHostScopeAttr::HttpExtension + ); + if !requires_prepare { + return Ok(quote!()); + } + let Some(context_ident) = find_context_param_ident(item) else { + return Err(Error::new_spanned( + &item.sig.ident, + "async scoped http host functions must accept SharedProxyVmContext", + )); + }; + let scope_tokens = edge_scope_tokens(scope); + let name_expr = &attr.name; + Ok(quote! { + crate::abi_impl::prepare_scoped_host_call( + #context_ident.clone(), + #scope_tokens, + #name_expr, + ) + .await?; + }) +} + +fn transform_async_edge_function(item: &mut ItemFn, attr: &EdgeHostAttr) -> Result<(), Error> { + if item.sig.asyncness.is_none() { + return Ok(()); + } + + for input in &item.sig.inputs { + let FnArg::Typed(pat_type) = input else { + return Err(Error::new_spanned(input, "methods are not supported")); + }; + if is_value_slice_type(&pat_type.ty) { + return Err(Error::new_spanned( + &pat_type.ty, + "async edge host functions do not support raw args; use typed parameters instead", + )); + } + if edge_arg_decoder_kind(&pat_type.ty) + .ok() + .is_some_and(edge_arg_decoder_is_borrowed) + { + return Err(Error::new_spanned( + &pat_type.ty, + "async edge host functions do not support borrowed typed parameters; use owned String, Value, or VmMap inputs", + )); + } + } + + let Some(vm_ident) = find_vm_param_ident(item) else { + return Err(Error::new_spanned( + &item.sig.ident, + "async edge host functions must accept a Vm parameter so the macro can schedule the future", + )); + }; + + match edge_output_kind(&item.sig.output)? { + Some(EdgeOutputKind::ResultCallOutcome) => {} + Some(EdgeOutputKind::CallOutcome) => { + return Err(Error::new_spanned( + &item.sig.output, + "async edge host functions must return Result", + )); + } + None => { + return Err(Error::new_spanned( + &item.sig.output, + "edge host functions must return CallOutcome or Result", + )); + } + } + + let original_block = item.block.clone(); + let prepare_stmt = async_scope_prepare_stmt(item, attr)?; + item.sig.asyncness = None; + *item.block = syn::parse2(quote!({ + crate::abi_impl::schedule_current_future_call(#vm_ident, async move { + #prepare_stmt + let __pd_edge_outcome = (async move #original_block).await?; + match __pd_edge_outcome { + ::vm::CallOutcome::Return(values) => Ok(values), + ::vm::CallOutcome::Halt => Err(::vm::VmError::HostError( + "async edge host functions must not return Halt".to_string(), + )), + ::vm::CallOutcome::Yield => Err(::vm::VmError::HostError( + "async edge host functions must not return Yield".to_string(), + )), + ::vm::CallOutcome::Pending(_) => Err(::vm::VmError::HostError( + "async edge host functions must not return Pending".to_string(), + )), + } + }) + }))?; + Ok(()) +} + +fn validate_edge_bind_names(item: &ItemFn, bind_params: &[Ident]) -> Result<(), Error> { + let params = item + .sig + .inputs + .iter() + .filter_map(|input| match input { + FnArg::Typed(pat_type) => match pat_type.pat.as_ref() { + Pat::Ident(ident) => Some(ident.ident.to_string()), + _ => None, + }, + FnArg::Receiver(_) => None, + }) + .collect::>(); + + for bind in bind_params { + if !params.iter().any(|name| name == &bind.to_string()) { + return Err(Error::new_spanned( + bind, + format!( + "bind parameter '{}' does not match any function parameter", + bind + ), + )); + } + } + + Ok(()) +} + +fn validate_edge_param(arg: &FnArg, bind_params: &[Ident]) -> Result<(), Error> { + let FnArg::Typed(pat_type) = arg else { + return Err(Error::new_spanned(arg, "methods are not supported")); + }; + let Pat::Ident(PatIdent { ident, .. }) = pat_type.pat.as_ref() else { + return Err(Error::new_spanned( + &pat_type.pat, + "edge host parameters must use identifier patterns", + )); + }; + + if is_vm_context_type(&pat_type.ty) + || is_edge_async_ops_type(&pat_type.ty) + || is_edge_context_type(&pat_type.ty) + || is_value_slice_type(&pat_type.ty) + { + if bind_params.iter().any(|candidate| candidate == ident) { + return Err(Error::new_spanned( + ident, + "special edge host parameters must not be listed in bind(...)", + )); + } + return Ok(()); + } + + if bind_params.iter().any(|candidate| candidate == ident) { + return Ok(()); + } + + edge_arg_decoder_kind(&pat_type.ty).map(|_| ()) +} + +fn validate_edge_return_type(output: &ReturnType) -> Result<(), Error> { + match edge_output_kind(output)? { + Some(_) => Ok(()), + None => Err(Error::new_spanned( + output, + "edge host functions must return CallOutcome or Result", + )), + } +} + +fn generate_edge_host_binder( + item: &ItemFn, + wrapper_name: &syn::Ident, + attr: &EdgeHostAttr, +) -> Result { + let impl_name = &item.sig.ident; + let vis = &item.vis; + let name_expr = &attr.name; + let mut binder_params = Vec::::new(); + let mut binder_setup = Vec::::new(); + let mut closure_setup = Vec::::new(); + let mut call_args = Vec::::new(); + let mut extract_stmts = Vec::::new(); + let mut arg_index = 0usize; + let mut raw_args = false; + + binder_params.push(quote!(bind_vm: &mut ::vm::Vm)); + binder_params.push(quote!(bind_context: &crate::abi_impl::SharedProxyVmContext)); + binder_params.push(quote!(bind_async_ops: &crate::abi_impl::SharedVmAsyncOps)); + + for input in &item.sig.inputs { + let FnArg::Typed(pat_type) = input else { + return Err(Error::new_spanned(input, "methods are not supported")); + }; + let Pat::Ident(PatIdent { ident, .. }) = pat_type.pat.as_ref() else { + return Err(Error::new_spanned( + &pat_type.pat, + "edge host parameters must use identifier patterns", + )); + }; + let ty = &pat_type.ty; + + if is_vm_context_type(ty) { + call_args.push(quote!(vm)); + continue; + } + + if is_edge_async_ops_type(ty) { + binder_setup.push(quote!(let #ident = bind_async_ops.clone();)); + closure_setup.push(quote!(let #ident = #ident.clone();)); + call_args.push(quote!(#ident)); + continue; + } + + if is_edge_context_type(ty) { + binder_setup.push(quote!(let #ident = bind_context.clone();)); + closure_setup.push(quote!(let #ident = #ident.clone();)); + call_args.push(quote!(#ident)); + continue; + } + + if is_value_slice_type(ty) { + raw_args = true; + call_args.push(quote!(args)); + continue; + } + + if attr.bind_params.iter().any(|candidate| candidate == ident) { + binder_params.push(quote!(#ident: #ty)); + binder_setup.push(quote!(let #ident = #ident.clone();)); + closure_setup.push(quote!(let #ident = #ident.clone();)); + call_args.push(quote!(#ident)); + continue; + } + + let decoder = edge_arg_decoder_kind(ty)?; + extract_stmts.push(edge_extract_stmt(ident, decoder, arg_index, wrapper_name)); + call_args.push(quote!(#ident)); + arg_index += 1; + } + + let arity_check = if raw_args { + None + } else { + Some(quote! { + if args.len() != #arg_index { + return Err(::vm::VmError::HostError(format!( + "expected {} arguments, got {}", + #arg_index, + args.len() + ))); + } + }) + }; + + let call_expr = match edge_output_kind(&item.sig.output)? { + Some(EdgeOutputKind::ResultCallOutcome) => quote!(#impl_name(#(#call_args),*)), + Some(EdgeOutputKind::CallOutcome) => quote!(Ok(#impl_name(#(#call_args),*))), + None => { + return Err(Error::new_spanned( + &item.sig.output, + "edge host functions must return CallOutcome or Result", + )); + } + }; + + Ok(quote! { + #[allow(dead_code)] + #vis fn #wrapper_name(#(#binder_params),*) { + #(#binder_setup)* + crate::abi_impl::bind_async_host_handler(bind_vm, bind_async_ops, #name_expr, move |vm, args| { + #arity_check + #(#closure_setup)* + #(#extract_stmts)* + #call_expr + }); + } + }) +} + +fn generate_scoped_edge_host_static_wrapper( + item: &ItemFn, + wrapper_name: &syn::Ident, + attr: &EdgeHostAttr, + was_async: bool, +) -> Result { + let Some(scope) = attr.scope else { + return Ok(quote!()); + }; + let impl_name = &item.sig.ident; + let static_wrapper_name = format_ident!("__pd_edge_static_{}", wrapper_name); + let uses_vm = scoped_wrapper_uses_vm(item); + let scope_tokens = edge_scope_tokens(scope); + let scope_requires_prepare = matches!( + scope, + EdgeHostScopeAttr::Http | EdgeHostScopeAttr::HttpExtension + ); + let args_only_sync_fast_path = !uses_vm && !was_async; + let prepare_context_ident = format_ident!("__pd_edge_prepare_context"); + let mut setup_stmts = Vec::::new(); + let mut call_args = Vec::::new(); + let mut extract_stmts = Vec::::new(); + let mut arg_index = 0usize; + + for input in &item.sig.inputs { + let FnArg::Typed(pat_type) = input else { + return Err(Error::new_spanned(input, "methods are not supported")); + }; + let Pat::Ident(PatIdent { ident, .. }) = pat_type.pat.as_ref() else { + return Err(Error::new_spanned( + &pat_type.pat, + "edge host parameters must use identifier patterns", + )); + }; + let ty = &pat_type.ty; + + if is_vm_context_type(ty) { + call_args.push(quote!(vm)); + continue; + } + + if is_edge_async_ops_type(ty) { + setup_stmts.push(quote!(let #ident = crate::abi_impl::current_async_ops()?;)); + call_args.push(quote!(#ident)); + continue; + } + + if is_edge_context_type(ty) { + if args_only_sync_fast_path && scope_requires_prepare { + setup_stmts.push(quote!(let #ident = #prepare_context_ident.clone();)); + } else { + setup_stmts.push(quote!(let #ident = crate::abi_impl::current_vm_context()?;)); + } + call_args.push(quote!(#ident)); + continue; + } + + if is_value_slice_type(ty) { + return Err(Error::new_spanned( + ty, + "scoped pd_host_function does not support raw args", + )); + } + + if attr.bind_params.iter().any(|candidate| candidate == ident) { + return Err(Error::new_spanned( + ident, + "scoped pd_host_function does not support bind(...)", + )); + } + + let decoder = edge_arg_decoder_kind(ty)?; + extract_stmts.push(edge_extract_stmt(ident, decoder, arg_index, wrapper_name)); + call_args.push(quote!(#ident)); + arg_index += 1; + } + + let _ = u8::try_from(arg_index).map_err(|_| { + Error::new_spanned( + &item.sig.ident, + "edge host functions must have 255 arguments or fewer", + ) + })?; + let call_expr = match edge_output_kind(&item.sig.output)? { + Some(EdgeOutputKind::ResultCallOutcome) => quote!(#impl_name(#(#call_args),*)), + Some(EdgeOutputKind::CallOutcome) => quote!(Ok(#impl_name(#(#call_args),*))), + None => { + return Err(Error::new_spanned( + &item.sig.output, + "edge host functions must return CallOutcome or Result", + )); + } + }; + + if uses_vm { + Ok(quote! { + fn #static_wrapper_name( + vm: &mut ::vm::Vm, + args: &[::vm::Value], + ) -> Result<::vm::CallOutcome, ::vm::VmError> { + if args.len() != #arg_index { + return Err(::vm::VmError::HostError(format!( + "expected {} arguments, got {}", + #arg_index, + args.len() + ))); + } + #(#setup_stmts)* + let __pd_edge_outcome = { + #(#extract_stmts)* + #call_expr + }?; + Ok(__pd_edge_outcome) + } + }) + } else if args_only_sync_fast_path && scope_requires_prepare { + let name_expr = &attr.name; + Ok(quote! { + fn #static_wrapper_name( + args: &[::vm::Value], + ) -> Result<::vm::CallOutcome, ::vm::VmError> { + if args.len() != #arg_index { + return Err(::vm::VmError::HostError(format!( + "expected {} arguments, got {}", + #arg_index, + args.len() + ))); + } + let #prepare_context_ident = crate::abi_impl::current_vm_context()?; + if !crate::abi_impl::scoped_host_call_can_run_synchronously( + &#prepare_context_ident, + #scope_tokens, + #name_expr, + )? { + return Err(::vm::VmError::HostError(format!( + "synchronous scoped host function {} requires an async signature", + #name_expr, + ))); + } + #(#setup_stmts)* + #(#extract_stmts)* + let __pd_edge_outcome = #call_expr?; + Ok(__pd_edge_outcome) + } + }) + } else { + Ok(quote! { + fn #static_wrapper_name( + args: &[::vm::Value], + ) -> Result<::vm::CallOutcome, ::vm::VmError> { + if args.len() != #arg_index { + return Err(::vm::VmError::HostError(format!( + "expected {} arguments, got {}", + #arg_index, + args.len() + ))); + } + #(#setup_stmts)* + let __pd_edge_outcome = { + #(#extract_stmts)* + #call_expr + }?; + Ok(__pd_edge_outcome) + } + }) + } +} + +fn generate_edge_host_registration( + item: &ItemFn, + wrapper_name: &syn::Ident, + attr: &EdgeHostAttr, + docs: &str, +) -> Result { + let Some(scope) = attr.scope else { + return Ok(quote!()); + }; + + let entry_name = format_ident!("__pd_edge_registration_{}", wrapper_name); + let scope_tokens = edge_scope_tokens(scope); + let static_wrapper_name = format_ident!("__pd_edge_static_{}", wrapper_name); + let function_kind = if scoped_wrapper_uses_vm(item) { + quote!(crate::abi_impl::registry::EdgeHostRegistrationFunction::StackStatic(#static_wrapper_name)) + } else { + quote!(crate::abi_impl::registry::EdgeHostRegistrationFunction::ArgsStatic(#static_wrapper_name)) + }; + let mut arity = 0usize; + + for input in &item.sig.inputs { + let FnArg::Typed(pat_type) = input else { + return Err(Error::new_spanned(input, "methods are not supported")); + }; + if is_vm_context_type(&pat_type.ty) + || is_edge_async_ops_type(&pat_type.ty) + || is_edge_context_type(&pat_type.ty) + { + continue; + } + if is_value_slice_type(&pat_type.ty) { + return Err(Error::new_spanned( + &pat_type.ty, + "scoped pd_host_function does not support raw args", + )); + } + arity += 1; + } + + let arity = u8::try_from(arity).map_err(|_| { + Error::new_spanned( + &item.sig.ident, + "edge host functions must have 255 arguments or fewer", + ) + })?; + let name_expr = &attr.name; + let docs = docs.to_string(); + + Ok(quote! { + #[::linkme::distributed_slice(crate::abi_impl::registry::PD_EDGE_HOST_FUNCTIONS)] + #[allow(non_upper_case_globals)] + static #entry_name: crate::abi_impl::registry::EdgeHostRegistration = + crate::abi_impl::registry::EdgeHostRegistration { + scope: #scope_tokens, + name: #name_expr, + arity: #arity, + docs: #docs, + function: #function_kind, + }; + }) +} + +fn scoped_wrapper_uses_vm(item: &ItemFn) -> bool { + item.sig.inputs.iter().any(|input| match input { + FnArg::Typed(pat_type) => is_vm_context_type(&pat_type.ty), + FnArg::Receiver(_) => false, + }) +} + +fn find_vm_param_ident(item: &ItemFn) -> Option { + item.sig.inputs.iter().find_map(|input| { + let FnArg::Typed(pat_type) = input else { + return None; + }; + if !is_vm_context_type(&pat_type.ty) { + return None; + } + match pat_type.pat.as_ref() { + Pat::Ident(PatIdent { ident, .. }) => Some(ident.clone()), + _ => None, + } + }) +} + +fn wrapper_and_impl_names(name: &syn::Ident) -> (syn::Ident, syn::Ident) { + let original = name.to_string(); + match original.strip_suffix("_impl") { + Some(prefix) => ( + syn::Ident::new(prefix, name.span()), + syn::Ident::new(&original, name.span()), + ), + None => ( + syn::Ident::new(&original, name.span()), + syn::Ident::new(&format!("{original}_impl"), name.span()), + ), + } +} + +#[derive(Clone, Copy)] +enum EdgeArgDecoderKind { + String, + StringRef, + Int, + Bool, + Value, + ValueRef, + Map, + MapRef, +} + +#[derive(Clone, Copy)] +enum EdgeOutputKind { + CallOutcome, + ResultCallOutcome, +} + +fn edge_arg_decoder_kind(ty: &Type) -> Result { + match ty { + Type::Group(group) => edge_arg_decoder_kind(&group.elem), + Type::Paren(paren) => edge_arg_decoder_kind(&paren.elem), + Type::Reference(reference) => { + if reference.mutability.is_some() { + return Err(Error::new_spanned( + ty, + "mutable borrowed edge host argument types are not supported", + )); + } + let Some(inner) = type_path_terminal_ident(reference.elem.as_ref()) else { + return Err(Error::new_spanned( + ty, + "unsupported borrowed edge host argument type", + )); + }; + match inner.as_str() { + "str" => Ok(EdgeArgDecoderKind::StringRef), + "Value" => Ok(EdgeArgDecoderKind::ValueRef), + "VmMap" => Ok(EdgeArgDecoderKind::MapRef), + other => Err(Error::new_spanned( + ty, + format!("unsupported borrowed edge host argument type '&{other}'"), + )), + } + } + Type::Path(path) => { + let Some(segment) = path.path.segments.last() else { + return Err(Error::new_spanned( + ty, + "unsupported edge host argument type", + )); + }; + match segment.ident.to_string().as_str() { + "String" => Ok(EdgeArgDecoderKind::String), + "i8" | "i16" | "i32" | "i64" | "isize" | "u8" | "u16" | "u32" | "u64" | "usize" => { + Ok(EdgeArgDecoderKind::Int) + } + "bool" => Ok(EdgeArgDecoderKind::Bool), + "Value" => Ok(EdgeArgDecoderKind::Value), + "VmMap" => Ok(EdgeArgDecoderKind::Map), + other => Err(Error::new_spanned( + ty, + format!("unsupported edge host argument type '{other}'"), + )), + } + } + _ => Err(Error::new_spanned( + ty, + "unsupported edge host argument type", + )), + } +} + +fn edge_extract_stmt( + ident: &Ident, + decoder: EdgeArgDecoderKind, + arg_index: usize, + wrapper_name: &syn::Ident, +) -> proc_macro2::TokenStream { + let label = LitStr::new( + &format!("{} {}", wrapper_name, ident), + proc_macro2::Span::call_site(), + ); + let index = syn::Index::from(arg_index); + match decoder { + EdgeArgDecoderKind::String => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::String(value)) => value.to_string(), + Some(_) => return Err(::vm::VmError::TypeMismatch("string")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::StringRef => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::String(value)) => value.as_str(), + Some(_) => return Err(::vm::VmError::TypeMismatch("string")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::Int => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::Int(value)) => *value, + Some(_) => return Err(::vm::VmError::TypeMismatch("int")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::Bool => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::Bool(value)) => *value, + Some(_) => return Err(::vm::VmError::TypeMismatch("bool")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::Value => quote! { + let #ident = match args.get(#index) { + Some(value) => value.clone(), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::ValueRef => quote! { + let #ident = match args.get(#index) { + Some(value) => value, + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::Map => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::Map(entries)) => entries.as_ref().clone(), + Some(_) => return Err(::vm::VmError::TypeMismatch("map")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + EdgeArgDecoderKind::MapRef => quote! { + let #ident = match args.get(#index) { + Some(::vm::Value::Map(entries)) => entries.as_ref(), + Some(_) => return Err(::vm::VmError::TypeMismatch("map")), + None => { + return Err(::vm::VmError::HostError(format!( + "missing argument: {}", + #label + ))); + } + }; + }, + } +} + +fn edge_arg_decoder_is_borrowed(decoder: EdgeArgDecoderKind) -> bool { + matches!( + decoder, + EdgeArgDecoderKind::StringRef | EdgeArgDecoderKind::ValueRef | EdgeArgDecoderKind::MapRef + ) +} + +fn type_path_terminal_ident(ty: &Type) -> Option { + match ty { + Type::Group(group) => type_path_terminal_ident(&group.elem), + Type::Paren(paren) => type_path_terminal_ident(&paren.elem), + Type::Path(path) => path + .path + .segments + .last() + .map(|segment| segment.ident.to_string()), + _ => None, + } +} + +fn edge_output_kind(output: &ReturnType) -> Result, Error> { + match output { + ReturnType::Default => Ok(None), + ReturnType::Type(_, ty) => { + if is_call_outcome_type(ty) { + return Ok(Some(EdgeOutputKind::CallOutcome)); + } + if is_host_call_result_type(ty) { + return Ok(Some(EdgeOutputKind::ResultCallOutcome)); + } + if let Some(inner) = unwrap_result_type(ty)? + && is_call_outcome_type(&inner) + { + return Ok(Some(EdgeOutputKind::ResultCallOutcome)); + } + Ok(None) + } + } +} + +fn unwrap_result_type(ty: &Type) -> Result, Error> { + match ty { + Type::Group(group) => unwrap_result_type(&group.elem), + Type::Paren(paren) => unwrap_result_type(&paren.elem), + Type::Reference(reference) => unwrap_result_type(&reference.elem), + Type::Path(path) => { + let Some(segment) = path.path.segments.last() else { + return Ok(None); + }; + if segment.ident != "Result" { + return Ok(None); + } + let syn::PathArguments::AngleBracketed(args) = &segment.arguments else { + return Err(Error::new_spanned( + &segment.arguments, + "Result requires generic arguments", + )); + }; + let Some(syn::GenericArgument::Type(inner)) = args.args.first() else { + return Err(Error::new_spanned( + args, + "Result requires a return type argument", + )); + }; + Ok(Some(inner.clone())) + } + _ => Ok(None), + } +} + +fn is_call_outcome_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_call_outcome_type(&group.elem), + Type::Paren(paren) => is_call_outcome_type(&paren.elem), + Type::Reference(reference) => is_call_outcome_type(&reference.elem), + Type::Path(path) => path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "CallOutcome"), + _ => false, + } +} + +fn is_host_call_result_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_host_call_result_type(&group.elem), + Type::Paren(paren) => is_host_call_result_type(&paren.elem), + Type::Reference(reference) => is_host_call_result_type(&reference.elem), + Type::Path(path) => path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "HostCallResult"), + _ => false, + } +} + +fn is_vm_context_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_vm_context_type(&group.elem), + Type::Paren(paren) => is_vm_context_type(&paren.elem), + Type::Reference(reference) => is_vm_context_type(&reference.elem), + Type::Path(path) => path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "Vm"), + _ => false, + } +} + +fn is_edge_async_ops_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_edge_async_ops_type(&group.elem), + Type::Paren(paren) => is_edge_async_ops_type(&paren.elem), + Type::Reference(reference) => is_edge_async_ops_type(&reference.elem), + Type::Path(path) => path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "SharedVmAsyncOps"), + _ => false, + } +} + +fn is_edge_context_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_edge_context_type(&group.elem), + Type::Paren(paren) => is_edge_context_type(&paren.elem), + Type::Reference(reference) => is_edge_context_type(&reference.elem), + Type::Path(path) => path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "SharedProxyVmContext"), + _ => false, + } +} + +fn is_value_slice_type(ty: &Type) -> bool { + match ty { + Type::Group(group) => is_value_slice_type(&group.elem), + Type::Paren(paren) => is_value_slice_type(&paren.elem), + Type::Reference(reference) => matches!( + reference.elem.as_ref(), + Type::Slice(slice) + if matches!( + slice.elem.as_ref(), + Type::Path(path) + if path + .path + .segments + .last() + .is_some_and(|segment| segment.ident == "Value") + ) + ), + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::expand_scoped_pd_host_function; + use syn::{ItemFn, Meta, Token, parse_quote, punctuated::Punctuated}; + + #[test] + fn preserves_async_scoped_edge_host_expansion() { + let attr: Punctuated = parse_quote!(name = "test::suspend", scope = http); + let item: ItemFn = parse_quote! { + /// Returns a value after a scoped host operation completes. + #[pd_host_function(name = "test::suspend", scope = http)] + async fn suspend( + _vm: &mut Vm, + _context: SharedProxyVmContext, + ) -> Result { + Ok(CallOutcome::Return(vm::CallReturn::none())) + } + }; + + let expanded = expand_scoped_pd_host_function(attr, item) + .expect("async scoped edge host functions should be accepted"); + let rendered = expanded.to_string(); + assert!(rendered.contains("schedule_current_future_call")); + assert!(rendered.contains("PD_EDGE_HOST_FUNCTIONS")); + } +} diff --git a/pd-host-function/src/lib.rs b/pd-host-function/src/lib.rs index 4aa1d3bc..4a878844 100644 --- a/pd-host-function/src/lib.rs +++ b/pd-host-function/src/lib.rs @@ -5,15 +5,35 @@ use syn::{ punctuated::Punctuated, }; +mod edge; + #[proc_macro_attribute] pub fn pd_host_function(attr: TokenStream, item: TokenStream) -> TokenStream { let args = parse_macro_input!(attr with Punctuated::::parse_terminated); - match expand_pd_host_function(args, parse_macro_input!(item as ItemFn)) { + let item = parse_macro_input!(item as ItemFn); + let result = if uses_edge_host_contract(&args, &item) { + edge::expand_scoped_pd_host_function(args, item) + } else { + expand_pd_host_function(args, item) + }; + match result { Ok(tokens) => tokens.into(), Err(err) => err.to_compile_error().into(), } } +fn uses_edge_host_contract(args: &Punctuated, item: &ItemFn) -> bool { + if item.sig.asyncness.is_some() { + return true; + } + + args.iter().any(|meta| match meta { + Meta::NameValue(name_value) if name_value.path.is_ident("scope") => true, + Meta::List(list) if list.path.is_ident("bind") => true, + _ => false, + }) +} + fn expand_pd_host_function( attr: Punctuated, mut item: ItemFn, @@ -482,7 +502,7 @@ fn uses_taken_extractor(ty: &Type) -> bool { #[cfg(test)] mod tests { - use super::expand_pd_host_function; + use super::{expand_pd_host_function, uses_edge_host_contract}; use syn::{ItemFn, Meta, Token, parse_quote, punctuated::Punctuated}; #[test] @@ -533,4 +553,26 @@ mod tests { .expect_err("the pd-host-function macro must not accept an async attribute"); assert!(error.to_string().contains("only supports name")); } + + #[test] + fn native_async_signature_selects_edge_contract() { + let attr: Punctuated = parse_quote!(name = "test::async_call"); + let item: ItemFn = parse_quote! { + async fn async_call() -> VmResult> { + todo!() + } + }; + assert!(uses_edge_host_contract(&attr, &item)); + } + + #[test] + fn name_expression_alone_does_not_select_edge_contract() { + let attr: Punctuated = parse_quote!(name = NAME_PATH); + let item: ItemFn = parse_quote! { + fn sync_call() -> VmResult> { + todo!() + } + }; + assert!(!uses_edge_host_contract(&attr, &item)); + } } diff --git a/pd-vm-wasm/src/analyzer.rs b/pd-vm-wasm/src/analyzer.rs index cb77f169..4c671efc 100644 --- a/pd-vm-wasm/src/analyzer.rs +++ b/pd-vm-wasm/src/analyzer.rs @@ -98,6 +98,15 @@ fn lint_compile_result( LintReport { diagnostics } } } + // Milestone 5+ load/merge errors carry the compilation-wide source + // map; unwrap it so the branch below renders from the owning source. + Err(SourcePathError::SourceWithMap { error, .. }) => lint_compile_result( + source, + flavor, + path, + options, + Err(SourcePathError::Source(error)), + ), Err(SourcePathError::Source(SourceError::Parse(err))) => { let mut diagnostics = lint_trailing_function_return_semicolon_diagnostics(source, flavor); @@ -115,7 +124,11 @@ fn lint_compile_result( source, flavor, path, options, )); let mut source_map = SourceMap::new(); - let source_id = source_map.add_source("", source.to_string()); + // Milestone 5+ compile errors name their owning source; register + // the root text under that name so the rendered snippet resolves + // against the right file. + let source_id = + source_map.add_source(err.source_name().unwrap_or(""), source.to_string()); let line = err.line().unwrap_or(0); let span = err.line().and_then(|value| { let span = source_map.line_span(source_id, value)?; diff --git a/pd-vm-wasm/src/lib.rs b/pd-vm-wasm/src/lib.rs index 9ccc0896..48b31cb6 100644 --- a/pd-vm-wasm/src/lib.rs +++ b/pd-vm-wasm/src/lib.rs @@ -1169,8 +1169,7 @@ mod runtime_tests { diagnostic.message ); assert!( - diagnostic.rendered.contains(":") - && diagnostic.rendered.contains("let value = if true => {"), + diagnostic.rendered.contains("let value = if true => {"), "expected rendered diagnostic snippet, got {:?}", diagnostic.rendered ); diff --git a/plans/2026-08-09_architecture-plan-index.md b/plans/2026-08-09_architecture-plan-index.md new file mode 100644 index 00000000..8c2f798d --- /dev/null +++ b/plans/2026-08-09_architecture-plan-index.md @@ -0,0 +1,88 @@ +# RustScript Architecture Plan Index + +**Goal:** Classify the current VM/compiler/runtime findings into independent implementation plans with explicit dependency order. + +**Architecture:** Correctness fixes are separated from structural refactors. Each plan owns one architectural surface and defines its own boundary and target criteria. Agent-owned behavior remains in `rustscript-agent` plans. + +**Tech Stack:** RustScript compiler, VM, host runtime, interpreter/JIT/AOT/no-std, agent integration contracts. + +--- + +## Classification + +| Category | Problem surface | Independent plan | Depends on | +| --- | --- | --- | --- | +| Wire/ABI | Count-derived builtin indices change existing call IDs | `2026-08-09_static-builtin-id.md` | none | +| Compiler correctness | UTF-8 rewrite, parent-path normalization, nested source diagnostics, public entry-point parity | `2026-08-09_nested-module-correctness.md` | none | +| Compiler architecture | Text rewrite, synthetic preludes, flat global symbols, basename identity, source ownership | `2026-08-09_semantic-module-system.md` | nested correctness | +| VM ownership | Monolithic VM mixes engine/program/instance/run/host state | `2026-08-09_vm-runtime-decomposition.md` | static IDs | +| Host lifecycle | Generic resource/operation code unused; IO/HTTP/SQLite duplicate lifecycle/cancellation | `2026-08-09_unified-host-lifecycle.md` | VM decomposition | +| Execution contract | Return/event ambiguity, buffered-only events, string errors, fragmented terminal state | `2026-08-09_run-outcome-event-error-contract.md` | RunContext; host lifecycle for final cancellation integration | +| Authorization | Builtin fast-path bypass, mutable identity/cache complexity, Edge macro leakage | `2026-08-09_capability-profile-host-binding.md` | static IDs | +| Async host transport | Core macro contains Edge scope knowledge; HTTP owns a synchronous scheduler; IO lacks feature-selected blocking/async bindings | `2026-08-09_http-transport-security-executor.md` | capability profile and host lifecycle | +| Structured concurrency | One waiting slot, no generic multi-operation/child-program supervisor | `2026-08-09_structured-task-supervisor.md` | VM decomposition, host lifecycle, capability profile | +| Backend architecture | Repeated semantics across interpreter/JIT/AOT/native/no-std | `2026-08-09_backend-semantic-convergence.md` | static IDs, VM decomposition | + +## Agent-owned plans + +| Category | Plan | +| --- | --- | +| Canonical product/framework roadmap | `rustscript-agent/plans/2026-07-30_rustscript-agent-gateway-api.md` | +| Run admission, structured input, result/events, timeout/cancellation, live delivery | `rustscript-agent/plans/2026-08-09_agent-run-lifecycle-events.md` | +| Transactional RSS storage, retention, restart, replay, idempotency | `rustscript-agent/plans/2026-08-09_agent-durable-state.md` | + +## Implementation route + +### Wave 0: Immediate correctness and identity + +Can run independently: + +1. Static builtin IDs. +2. Nested module correctness. + +Exit gate: static IDs no longer depend on catalog length; nested UTF-8/path/diagnostic regressions have executable coverage. + +### Wave 1: Foundational ownership and authorization + +Can run in parallel after relevant Wave 0 gates: + +1. VM runtime decomposition after static IDs. +2. Capability profile/host binding after static IDs. +3. Semantic module system after nested correctness. + +Exit gate: module semantics have explicit identities; VM has explicit ownership layers; every privileged call uses one authorization path. + +### Wave 2: Unified execution lifecycle + +1. Unified host resource/operation/cancellation lifecycle after VM ownership exists. +2. RunOutcome/event/error implementation on RunContext, integrating lifecycle cancellation as it becomes available. + +Exit gate: production host subsystems use one lifecycle, and every run has one structured terminal outcome with live bounded events. + +### Wave 3: Specialized consumers + +Can run in parallel after their dependencies: + +1. Generic host-driven async ABI, IO dual implementation, and async-only HTTP transport security. +2. Structured task supervisor. +3. Backend semantic convergence. +4. Agent run lifecycle and durable state integration. + +## Scope boundary + +- No plan adds compatibility decoding for pre-static-ID VMBC. +- No plan adds agent/provider/platform policy to `rustscript`. +- No agent plan defines VM internal implementation. +- Correctness plans do not wait for structural refactors. +- Structural plans remove superseded transitional paths after migration; they do not retain dual long-term architectures. +- New generic host functions require their own implementation plans; this index covers the architecture findings already identified. +- Async host futures are driven by the embedding host. VM, HTTP, and IO do not own a private executor or synchronous polling scheduler. +- Core host macros contain no pd-edge scopes, context types, registry generation, or downstream module paths. + +## Target criteria + +- Every finding from the architecture/current-worktree review maps to one owning plan. +- Cross-plan dependencies are explicit and acyclic. +- Each plan has implementation route, scope boundary, and target criteria. +- Core and agent ownership do not overlap. +- Obsolete review-finding and mixed historical plans are removed after their live requirements are represented here. diff --git a/plans/2026-08-09_backend-semantic-convergence.md b/plans/2026-08-09_backend-semantic-convergence.md new file mode 100644 index 00000000..9b280532 --- /dev/null +++ b/plans/2026-08-09_backend-semantic-convergence.md @@ -0,0 +1,136 @@ +# Backend Semantic Convergence Plan + +**Goal:** Reduce semantic duplication across interpreter, Trace JIT, AOT, native bridge, and no-std execution by introducing one canonical instruction/operation contract and differential verification. + +**Architecture:** Bytecode semantics, builtin signatures, ownership rules, traps, and deoptimization outcomes are defined once. Each backend lowers or interprets the same contract. Generated coverage tables and differential fixtures detect missing or divergent implementations. + +**Tech Stack:** Rust 2024, interpreter, Trace JIT, Cranelift AOT, native bridge, `pd-vm-nostd`, property/differential tests. + +--- + +## Independence and dependency + +- Independent of agent framework and module loading. +- Static builtin IDs should land first. +- VM decomposition should define Engine/Program ownership before large backend file moves. +- This plan does not block immediate correctness plans. + +## Scope boundary + +### In scope + +- One canonical semantic description for opcodes and builtins. +- Generated backend coverage checks. +- Shared ownership/trap/helper contracts. +- Differential interpreter/JIT/AOT/no-std tests. +- Incremental removal of duplicated lowering logic. + +### Out of scope + +- New optimization targets or benchmark promises. +- New bytecode opcodes solely to simplify one backend. +- A complete JIT rewrite in one milestone. +- Agent, HTTP, SQLite, or gateway behavior. + +## Implementation route + +### Milestone 1: Build a backend coverage inventory + +**Files:** +- Create backend coverage tests/tools under `tests/` or `src/backend/` +- Read interpreter, JIT recorder/lowerer, AOT IR/lowerer, no-std dispatch + +Generate a matrix for every opcode/builtin: + +```text +semantic definition +interpreter +trace recorder +JIT lowering +AOT lowering +no-std +fallback/deopt rule +``` + +Fail CI when a newly added operation lacks an explicit backend disposition. + +### Milestone 2: Define canonical operation semantics + +**Files:** +- Create: `src/semantics/` or equivalent +- Modify opcode/builtin metadata generation + +Represent: + +- operand/result types and stack effect; +- ownership/borrow/clone/drop behavior; +- trap/error conditions; +- side-effect and suspension classification; +- interpreter helper and native helper ABI; +- deopt/fallback permission. + +Keep explicit Rust implementation hooks where declarative metadata is insufficient. + +### Milestone 3: Generate shared dispatch metadata + +1. Generate interpreter validation/stack-effect tables. +2. Generate JIT/AOT eligibility and helper IDs. +3. Generate no-std support/fallback declarations. +4. Key builtins by static ID. +5. Reject mismatched arity/type/ownership metadata at build time. + +### Milestone 4: Consolidate native helper contracts + +**Files:** +- Modify native bridge/helper modules +- Modify JIT/AOT lowerers + +1. Define one helper ABI for tagged/scalar/heap operands. +2. Centralize owned temporary and Arc/raw-pointer rules. +3. Centralize trap/status routing. +4. Remove backend-specific reinterpretation of the same helper payload. + +### Milestone 5: Add differential execution harness + +For generated and curated programs, compare: + +- return value and structured error; +- side-effect/event sequence; +- ownership/drop counters where observable; +- fuel/deadline behavior; +- interpreter, JIT, AOT, and no-std supported subsets. + +Include arrays/maps/bytes, calls/closures, branches/loops, host-call boundaries, traps, and deopt cases. + +### Milestone 6: Migrate one semantic family at a time + +Recommended order: + +1. scalar arithmetic/comparison; +2. stack/local/frame operations; +3. collection access/mutation; +4. calls/closures; +5. builtin/native helper calls; +6. suspension/deopt/terminal outcomes. + +Each family removes superseded duplicate tables after differential parity passes. + +### Milestone 7: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --workspace --all-features +cargo test --locked -p pd-vm-nostd +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- Every opcode and builtin has one canonical semantic entry. +- Every backend declares implement/fallback/unsupported explicitly. +- New operations cannot compile without a complete backend disposition. +- Interpreter/JIT/AOT/no-std differential fixtures agree on the supported subset. +- Native ownership and trap ABI is shared by JIT and AOT. +- Backend-specific large files lose duplicated semantic policy over incremental milestones. +- Performance changes are measured separately from semantic convergence. diff --git a/plans/2026-08-09_capability-profile-host-binding.md b/plans/2026-08-09_capability-profile-host-binding.md new file mode 100644 index 00000000..578bd203 --- /dev/null +++ b/plans/2026-08-09_capability-profile-host-binding.md @@ -0,0 +1,137 @@ +# Capability Profile and Host Binding Contract Plan + +**Goal:** Authorize every privileged builtin and host import through one immutable, auditable capability profile keyed by static callable identity. + +**Architecture:** Compilation records required callable IDs; binding validates them against a `CapabilityProfile` with a stable fingerprint. Builtin fast paths, registry functions, overrides, cached plans, and cloned registries all use the same authorization decision before dispatch. + +**Tech Stack:** Rust 2024, generated builtin catalog, host function registry, proc macro, binding-plan tests. + +**Status:** Completed + +--- + +## Independence and dependency + +- Depends on static builtin IDs for durable callable identity. +- Can be implemented alongside VM decomposition; the VM-owned part is callable authorization only. +- HTTP/IO/SQLite configuration remains state owned by each host implementation and is not embedded in the profile. + +## Scope boundary + +### In scope + +- One authorization path for privileged builtins and registry host imports. +- Immutable capability profiles and stable fingerprints. +- A clean boundary between callable authorization and host-owned path/network/database/process limits. +- Binding-plan cache correctness under clone/mutation/generation changes. +- Explicit ordinary versus Edge proc-macro contracts. + +### Out of scope + +- Adding agent/provider-specific capabilities. +- Implementing filesystem/process/task functions. +- Backward-compatible acceptance of implicit broad profiles. +- HTTP DNS/SSRF implementation details. +- pd-edge release workflow changes. + +## Target model + +```text +CallableId = static builtin ID or explicit host import identity +CapabilityProfile + allowed callables + delegation limits + fingerprint + +HostFunctionState (type-erased by the VM, typed by each host module) + HTTP configuration and in-flight permits + IO path/process/size policy + SQLite root and resource limits + +BindingPlan + program requirements + resolved call targets + capability fingerprint + registry generation/identity +``` + +## Implementation route + +### Milestone 1: Add authorization regression tests + +Required failing cases: + +- empty profile rejects privileged builtin fast paths; +- empty profile rejects registry hosts and overrides; +- language-pure builtins remain available under the documented baseline profile; +- cached/uncached plans make identical decisions; +- sibling registry mutation invalidates or rejects stale plans; +- clone identity and structural independence follow one documented rule; +- parameter policy cannot be widened by script input. + +### Milestone 2: Define immutable profiles + +**Files:** +- Modify: `src/vm/host.rs` +- Create: `src/vm/capability.rs` +- Modify: `src/lib.rs` + +1. Separate pure language builtins from privileged host capabilities. +2. Key builtin permissions by explicit static ID. +3. Key external hosts by a stable import identity, not a mutable vector slot alone. +4. Keep HTTP/SQLite/IO policy types out of the VM capability layer. +5. Compute a deterministic fingerprint from callable authorization only. + +### Milestone 3: Put authorization before every dispatch path + +**Files:** +- Modify: `src/vm/host.rs` +- Modify builtin dispatch and fast-path entry points + +1. Resolve callable identity. +2. Authorize before builtin fast path, override, registry call, or native continuation. +3. Ensure a plan cannot grant a capability absent from the current profile. +4. Reject undeclared imports during preflight before side effects. + +### Milestone 4: Simplify plan cache identity + +1. Bind plans to program identity, registry identity/generation, and capability fingerprint. +2. Remove Arc-token combinations that encode overlapping identity concepts. +3. Define clone behavior explicitly and test source/sibling mutation. +4. Make stale-plan errors deterministic. + +### Milestone 5: Separate proc-macro contracts + +**Files:** +- Modify: `pd-host-function/src/lib.rs` +- Modify: `pd-host-function/src/edge.rs` +- Add independent consumer compile fixtures + +1. Require an explicit Edge marker/scope for Edge-only expansion. +2. Ordinary async functions must receive an ordinary supported expansion or a direct compile-time diagnostic. +3. Apply one scoped HTTP routing rule to VM-aware and args-only forms. +4. Hide Edge implementation paths from ordinary downstream consumers. +5. Test from a separate crate that has no pd-edge internals. + +### Milestone 6: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked -p pd-host-function +cargo test --locked --test runtime_host_tests +cargo test --locked --test http_host_tests --features http-client +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- No privileged builtin bypasses the capability decision. +- Empty means deny for every privileged path. +- Capability identity does not depend on catalog order or registry slot alone. +- Binding-plan cache keys contain one stable capability fingerprint. +- Clones and stale plans have tested deterministic semantics. +- Ordinary proc-macro consumers never receive Edge-private symbols. +- Parameter policies remain host-owned native upper bounds that script data cannot widen. +- `HostRuntime`, `CapabilityProfile`, and inherent `Vm` APIs do not name HTTP/IO/SQLite policy types. diff --git a/plans/2026-08-09_http-transport-security-executor.md b/plans/2026-08-09_http-transport-security-executor.md new file mode 100644 index 00000000..75ec902e --- /dev/null +++ b/plans/2026-08-09_http-transport-security-executor.md @@ -0,0 +1,182 @@ +# Async Host Transport Security Plan + +**Goal:** Make HTTP a host-driven async host function, provide feature-selected blocking/async IO implementations, and preserve transport security without any HTTP-owned scheduler. + +**Architecture:** `#[pd_host_function] async fn` produces a generic async host factory. The VM allocates an operation ID and hands the resulting `'static` future to the embedding's `HostAsyncBridge`; the host owns submission, waking, polling, cancellation, and reactor/executor integration. HTTP contains only policy, request construction, transport, redirect, deadline, and response decoding logic. Edge scopes and `SharedProxyVmContext` expansion remain owned by `pd-edge`. + +**Tech Stack:** Rust 2024, proc macros, `Future`, `HostAsyncBridge`, Reqwest/Tokio under the `async` feature, local HTTP fixtures. + +--- + +## Independence and dependency + +- Depends on the capability profile and unified host lifecycle. +- Adds a generic async host ABI before migrating IO or HTTP. +- Requires a coordinated `pd-edge` adapter migration because the current core proc-macro contains Edge-specific expansion. +- Independent of agent provider protocols and source-language `async`/`await` syntax. + +## Hard ownership rules + +1. The VM owns script suspension state and operation-ID allocation. +2. The embedding host owns future storage, waking, polling, cancellation, and executor/reactor integration. +3. A subsystem async host function owns only its future body and typed policy/context snapshot. +4. HTTP must not create a thread, Tokio runtime, oneshot completion scheduler, private pending map, private poller, or independent operation-ID namespace. +5. Core `pd-host-function` must not contain Edge scope enums, `SharedProxyVmContext`, `crate::abi_impl` paths, or pd-edge registry generation. +6. With the `async` feature disabled, IO keeps its blocking implementation and HTTP is absent from callable metadata and runtime registration. +7. With the `async` feature enabled, IO binds its async implementation and HTTP is available only through the async host ABI. +8. Async operation driving code lives in dedicated folders: core contracts/lifecycle under `src/vm/async_host/`, and the pd-edge driver under `pd-edge/src/async_host/`. It must not accumulate in `host.rs`, `abi_impl/mod.rs`, HTTP, or IO modules. + +## Scope boundary + +### In scope + +- Generic async `#[pd_host_function(name = "...")]` expansion using owned arguments. +- Host-driven future submission/poll/cancel ABI. +- Migration of pd-edge scope expansion to a pd-edge-owned proc-macro adapter. +- Feature-selected blocking/async IO implementations. +- Async-only generic HTTP host. +- Complete IPv4/IPv6 special-use classification. +- Async DNS, total/connect/first-byte/idle deadlines, destination pinning, redirect revalidation, and body limits. +- Cancellation/reset/drop tests across host-driver and transport phases. + +### Out of scope + +- A VM-owned Tokio runtime or process executor. +- HTTP-specific scheduling infrastructure. +- Provider JSON, retries, model selection, SSE semantic parsing, or agent loops. +- Ambient proxy support by default. +- Script-controlled policy relaxation. +- Source-language futures or `await` syntax. + +## Implementation route + +### Milestone 1: Freeze generic async host semantics with RED tests + +**Files:** +- Modify: `pd-host-function/src/lib.rs` +- Modify: `tests/host_binding_generation_tests.rs` +- Modify: VM host lifecycle tests + +Cover: + +- ordinary owned-argument async signatures are accepted; +- borrowed typed parameters and raw borrowed args are rejected for async hosts; +- generated wrappers submit exactly one `'static` future to the installed host driver; +- missing driver fails before a pending operation becomes visible; +- completion returns exactly one terminal result; +- cancellation/reset/drop cancel the driver operation exactly once; +- rejected submission retires the operation ID and leaves no waiting state; +- interpreter/JIT/AOT all suspend through the same `CallOutcome::Pending` boundary. + +### Milestone 2: Add the host-driven async ABI + +**Files:** +- Create: `src/vm/async_host/mod.rs` +- Create supporting files under `src/vm/async_host/` for lifecycle/bridge concerns when needed +- Modify: `src/vm/host.rs` only to remove superseded inline async lifecycle code +- Modify: `src/vm/host_runtime.rs` +- Modify: `src/vm/mod.rs` +- Modify: `build.rs` +- Modify: `pd-host-function/src/lib.rs` + +1. Define the boxed `'static` host future output contract. +2. Extend or replace `HostAsyncBridge` with explicit submit, poll, and cancel operations. +3. Allocate IDs through the shared operation registry before submission and retire them on every fallible path. +4. Generate an async host adapter that takes owned script arguments, constructs a future, and submits it through the VM boundary. +5. Classify async generated hosts as suspension-capable and exclude them from non-yielding native fast paths. +6. Keep cached registry binding and direct VM binding equivalent. + +### Milestone 3: Return Edge scope ownership to pd-edge + +**Core files:** +- Remove: `pd-host-function/src/edge.rs` +- Modify: `pd-host-function/src/lib.rs` + +**pd-edge files:** +- Create a pd-edge-owned proc-macro adapter crate or equivalent owned macro module. +- Create: `src/async_host/mod.rs` and focused driver/operation files beneath that folder. +- Migrate `scope = runtime/http/http_extension/transport`, bind parameters, registry generation, and `SharedProxyVmContext` preparation. +- Adapt `VmAsyncOpBridge` to the generic host submit/poll/cancel contract. +- Remove future storage, operation allocation, reactor entry, and bridge polling logic from `src/abi_impl/mod.rs`. + +The core proc-macro must retain only name-based generic sync/async host expansion. + +### Milestone 4: Provide blocking and async IO implementations + +**Files:** +- Modify: IO runtime modules, build generation, and IO tests + +1. Keep blocking IO available without the `async` feature. +2. Under `async`, bind the same script-facing IO API to async host functions with owned parameters. +3. Make build-time callable discovery honor the active feature so only one implementation enters metadata/registration. +4. Route async IO futures through the embedding host driver. +5. Preserve canonical path, process permission, byte, line, and handle policies in both variants. + +### Milestone 5: Migrate HTTP to an async host function + +**Files:** +- Modify: `Cargo.toml` +- Modify: `src/builtins/runtime/http.rs` +- Modify: `src/builtins/runtime/mod.rs` +- Modify: `tests/vm/http_host_tests.rs` + +1. Gate HTTP dependencies, callable metadata, configuration, and tests on `async`. +2. Convert `http::client::request` to a true async host function with an owned request and immutable policy/context snapshot. +3. Delete `schedule_request`, `HttpCompletion`, `HttpRequestResource`, oneshot completion, per-request threads/runtimes, and HTTP-specific pending polling. +4. Resolve DNS inside the submitted future and count it against the total deadline. +5. Validate and pin every resolved address before connection while preserving TLS hostname/SNI verification. +6. Repeat policy, credential stripping, pinning, and deadline checks for every redirect. +7. Enforce request/header/body, connect, first-byte, idle, total, and response-byte limits. +8. Disable ambient environment proxies unless the embedding supplies explicit policy. + +### Milestone 6: Cancellation and lifecycle convergence + +1. Propagate run cancellation/deadline/reset/drop to `HostAsyncBridge::cancel_op_with_reason`. +2. Ensure driver completion after a terminal run cannot re-enter the VM. +3. Verify cancellation during DNS, connect, response headers, and body streaming. +4. Confirm no HTTP or IO async subsystem resource remains after completion/cancellation. +5. Remove runtime owner-poller routing that became obsolete after host-driver migration. + +### Milestone 7: Verification + +Core: + +```bash +cargo fmt --all -- --check +cargo test --locked -p pd-host-function +cargo test --locked --test host_binding_generation_tests --all-features +cargo test --locked --test io_builtin_edge_tests --all-features +cargo test --locked --test http_host_tests --features async +cargo test --locked --workspace --all-features +cargo test --locked --workspace --no-default-features --tests --no-run +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +RUSTDOCFLAGS='-D warnings' cargo doc --locked --workspace --all-features --no-deps +git diff --check +``` + +pd-edge: + +```bash +cargo fmt --all -- --check +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +All temporary and target directories must be under `/mnt/TEMP/rustscript/` and removed after final review. + +## Target criteria + +- A generic ordinary async host function compiles and suspends through the host driver. +- The host driver, not VM/HTTP/IO, stores and drives submitted futures. +- Async driving code is isolated in the dedicated core and pd-edge async-host folders. +- Core proc-macro code has no pd-edge scope or context knowledge. +- IO has blocking and async implementations selected by `async`. +- HTTP is unavailable without `async` and uses no private scheduler. +- DNS time counts against the total deadline. +- Every connected address is validated and pinned before connection. +- Special-use IPv4/IPv6 ranges are denied by default. +- Redirects repeat destination and credential validation. +- Cancellation during every transport phase reaches the driver and finishes within the documented bound. +- Reset/drop leave no future, operation, permit, stream, or response body live. +- HTTP host code contains no provider or agent policy. diff --git a/plans/2026-08-09_nested-module-correctness.md b/plans/2026-08-09_nested-module-correctness.md new file mode 100644 index 00000000..a19f2f46 --- /dev/null +++ b/plans/2026-08-09_nested-module-correctness.md @@ -0,0 +1,127 @@ +# Nested Module Correctness Implementation Plan + +**Goal:** Correct the current nested RSS module loader without expanding it into a new module architecture. + +**Architecture:** Keep the existing source-loader, import rewrite, and public `SourcePathError` shape, while repairing UTF-8 preservation, lexical path normalization, source ownership in diagnostics, and parity across public compile entry points. This is the short corrective plan; semantic module replacement is covered separately. + +**Tech Stack:** Rust 2024, RustScript compiler/source loader, `SourceMap`, CLI diagnostics, Cargo integration tests. + +--- + +## Independence and dependency + +- Independent of VM runtime, builtin IDs, agent gateway, HTTP, and persistence. +- Must complete before the semantic module-system plan begins. +- The agent repository only supplies a composition fixture; compiler behavior remains owned by `rustscript`. + +## Scope boundary + +### In scope + +- UTF-8-safe call-site rewrite. +- Correct preservation of unmatched relative `..` components. +- Canonical identity for equivalent disk module paths. +- Nested parse and strict-type diagnostics rendered from the real module source. +- Equivalent behavior for file, source-with-options, and source-at-path public entry points. +- Regression coverage for import/export/cycle behavior already changed in the worktree. + +### Out of scope + +- New public error enum variants. +- A compatibility wrapper for the removed `SourceAt` experiment. +- Semantic symbol resolution or a new IR. +- Agent storage migration or gateway changes. +- VM-visible opcodes or host capabilities. + +## Implementation route + +### Milestone 1: Add failing UTF-8 rewrite tests + +**Files:** +- Modify: `tests/compiler/module_import_tests.rs` +- Modify: unit tests in `src/compiler/source_loader/rewrite.rs` + +Add nested modules that trigger namespace and named-import rewriting while containing: + +- non-ASCII string literals; +- line and block comments; +- non-ASCII source outside rewritten spans where syntax permits it. + +Assert byte-for-byte preservation of untouched source and runtime preservation of values such as `"猫"`. + +### Milestone 2: Make scanners copy source slices + +**Files:** +- Modify: `src/compiler/source_loader/rewrite.rs` + +1. Stop appending UTF-8 bytes with `byte as char`. +2. Advance by valid UTF-8 scalar boundaries or copy untouched ranges as source slices. +3. Keep token recognition ASCII-specific where the grammar requires ASCII identifiers and separators. +4. Preserve comments, strings, escapes, and line counts exactly. + +### Milestone 3: Correct path normalization and identity + +**Files:** +- Modify: `src/compiler/source_loader/imports.rs` +- Modify: `src/compiler/source_loader/graph.rs` +- Test: `tests/compiler/module_import_tests.rs` + +1. Pop `ParentDir` only when the previous normalized component is `Normal`. +2. Never cancel an unmatched `ParentDir` with a later `ParentDir`. +3. Preserve root semantics for absolute paths. +4. Use canonical disk identity for files that exist; use a normalized explicit virtual identity for source overrides. +5. Key `seen`, `visiting`, exports, and overrides with the same module identity. +6. Test consecutive `super::`, absolute above-root input rejection/normalization policy, path aliases, cycle aliases, and duplicate import aliases. + +### Milestone 4: Carry nested source context to diagnostics + +**Files:** +- Modify: `src/compiler/source_loader.rs` +- Modify: `src/compiler/source_loader/graph.rs` +- Modify: `src/compiler/pipeline.rs` +- Modify: `src/cli.rs` +- Test: `tests/compiler/module_import_tests.rs` +- Add or modify CLI diagnostic integration tests + +1. Keep `SourcePathError` public enum shape unchanged. +2. Carry internal `{ path, source text, SourceId/span }` context through compilation. +3. Render nested parse and strict-type errors against the nested source, not the root source map. +4. Apply the same path/source enrichment to: + - `compile_source_file`; + - `compile_source_with_flavor_and_options`; + - `compile_source_at_path_with_flavor_and_options`. +5. Test path, line, code frame, underline, and source override content, not message text alone. + +### Milestone 5: Preserve import/export behavior + +Add regression cases for: + +- nested namespace aliases; +- nested named imports; +- public-only exports; +- no transitive re-export; +- same-directory `self::` and parent-directory `super::`; +- missing modules and normalized cycles; +- root and nested host namespace imports. + +### Milestone 6: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --test compiler_tests module_import +cargo test --locked --test compiler_tests +cargo test --locked --workspace --all-features +git diff --check +``` + +Run the CLI diagnostic fixture and assert that the rendered path and highlighted line both belong to the nested source. + +## Target criteria + +- Rewriting never changes untouched UTF-8 bytes. +- Consecutive unmatched parent components retain their lexical meaning. +- Equivalent disk paths resolve to one module identity. +- Nested diagnostics display the actual nested source line and underline. +- All public module-capable compile entry points identify the failing module. +- Existing public error enum shape remains unchanged. +- Import, export, and cycle tests pass without agent-specific compiler behavior. diff --git a/plans/2026-08-09_run-outcome-event-error-contract.md b/plans/2026-08-09_run-outcome-event-error-contract.md new file mode 100644 index 00000000..c0222507 --- /dev/null +++ b/plans/2026-08-09_run-outcome-event-error-contract.md @@ -0,0 +1,140 @@ +# Run Outcome, Event Stream, and Runtime Error Contract Plan + +**Goal:** Define one structured execution result that keeps return values, events, usage, cancellation, and errors separate and machine-readable. + +**Architecture:** A run produces a terminal `RunOutcome`; events flow during execution through a bounded sink/channel and never replace the function return value. Runtime and host failures retain structured codes and context through the VM embedding boundary. + +**Tech Stack:** Rust 2024, VM embedding API, runtime context/events, host errors, agent runner integration tests. + +--- + +## Independence and dependency + +- Contract design can start independently. +- Implementation depends on RunContext ownership from the VM decomposition plan. +- Operation cancellation details depend on the unified host-lifecycle plan. +- The agent run-lifecycle plan consumes this API. + +## Scope boundary + +### In scope + +- `RunOutcome`, terminal reason, usage, return value, and structured error. +- Bounded event emission during execution. +- Event receipt/sequence semantics at the VM boundary. +- Structured runtime/host error propagation. +- Removal of stack-top/event-last inference in embedding code. + +### Out of scope + +- Agent event names, provider protocols, SSE framing, or Telegram rendering. +- Durable event persistence. +- Source-language concurrency syntax. +- Compatibility wrappers for ambiguous prior return behavior. + +## Target contracts + +```text +RunOutcome + return_value: optional Value + termination: completed | cancelled | failed | budget_exhausted + error: optional RuntimeError + usage: RunUsage + last_event_sequence + +RuntimeEvent + sequence + value + payload_bytes + +RuntimeError + code + message + subsystem + operation/resource context + retryability where meaningful + source error where meaningful +``` + +## Implementation route + +### Milestone 1: Add contract tests + +Add tests proving: + +- a script may emit events and return a different value; +- zero events does not alter the return value; +- event order is monotonic; +- sink rejection/backpressure has a documented terminal behavior; +- cancellation reason survives the public VM API; +- host/runtime codes survive without string equality checks; +- usage is finalized for success, error, cancellation, and budget exhaustion. + +### Milestone 2: Define terminal and usage types + +**Files:** +- Modify: `src/lib.rs` +- Create: `src/vm/outcome.rs` +- Modify runtime error modules + +1. Define `RunOutcome`, `RunTermination`, and `RunUsage`. +2. Make halt/failure/cancellation paths produce exactly one terminal outcome. +3. Stop requiring embedders to inspect stack top, yield reason, and side channels to infer completion. + +### Milestone 3: Make events live and bounded + +**Files:** +- Modify: `src/builtins/runtime/context.rs` +- Modify: `src/builtins/runtime/event.rs` +- Modify: `src/builtins/runtime/context_host.rs` +- Modify: RunContext + +1. Define a bounded event sink contract. +2. Emit each accepted event during execution. +3. Allocate sequence numbers once at the run boundary. +4. Define overflow policy explicitly: block/yield, return a typed limit error, or drop only where configured with a receipt. Silent loss is prohibited. +5. Keep event values independent from function return storage. + +### Milestone 4: Preserve structured errors + +**Files:** +- Modify: runtime error types +- Modify: `src/vm/host.rs` +- Modify: public VM error surface + +1. Carry `RuntimeErrorCode` through host completion and `RunOutcome`. +2. Include structured cancellation/deadline/resource/operation context. +3. Remove embedding logic that compares error strings such as `"cancelled"`. +4. Define rendering separately from machine-readable fields. + +### Milestone 5: Migrate embedders and remove ambiguous APIs + +**Files:** +- Modify examples and tests in `rustscript` +- Coordinate later changes in `rustscript-agent/src/lib.rs` + +1. Consume `RunOutcome.return_value` directly. +2. Subscribe to events through the sink/channel. +3. Remove event-last and stack-last fallback behavior. +4. Remove superseded internal return APIs after migration; no dual long-term contract. + +### Milestone 6: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --test runtime_context_tests +cargo test --locked --test runtime_host_tests +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- Emitting an event never changes the function return value. +- Events are observable before run completion through a bounded contract. +- Every run produces one structured terminal outcome. +- Cancellation, deadline, resource, and host errors retain machine-readable codes. +- Embedders do not infer results from stack/event ordering. +- String equality is absent from cancellation/error control flow. +- Usage and event sequence metadata are finalized for every terminal path. diff --git a/plans/2026-08-09_semantic-module-system.md b/plans/2026-08-09_semantic-module-system.md new file mode 100644 index 00000000..96fb4040 --- /dev/null +++ b/plans/2026-08-09_semantic-module-system.md @@ -0,0 +1,154 @@ +# Semantic Module System Implementation Plan + +**Status (2026-08-09):** Milestones 1-7 complete, committed in b3ef8a7. +The semantic module graph is the sole file-module path: `rewrite.rs` and +`line_map.rs` are deleted, the synthetic imported-function prelude is gone, +and module sources are parsed verbatim with implicit-extern fallback and +resolved by `SymbolId` in the source loader (see +`src/compiler/source_loader.rs` module docs). Verification: `compiler_tests` +(215 tests incl. `semantic_module_m6_tests`), workspace all-features tests, +fmt, clippy (no new warnings), and `git diff --check` are green. + +**Goal:** Replace textual import rewriting and synthetic declarations with a semantic module graph and symbol resolution model. + +**Architecture:** Parse imports as syntax, assign every source a `ModuleId` and `SourceId`, resolve declarations to `SymbolId`, and link by resolved identity. Module namespaces, visibility, private helpers, and diagnostics become first-class compiler data instead of rewritten text and parallel metadata arrays. + +**Tech Stack:** Rust 2024, RustScript parser, frontend IR, compiler pipeline, linker, `SourceMap`. + +--- + +## Independence and dependency + +- Depends on completion of `2026-08-09_nested-module-correctness.md` so the current branch has a verified baseline. +- Independent of VM execution, host capabilities, HTTP, agent gateway, and backend optimization. +- May proceed in compiler-only milestones with bytecode output held behaviorally constant. + +## Scope boundary + +### In scope + +- `ModuleId`, `SourceId`, `SymbolId`, import edges, export tables, and visibility. +- Semantic namespace and named-import resolution. +- Private function identity across modules. +- Source-owned diagnostics after graph merge. +- Removal of synthetic function preludes and call-site text rewriting for file modules. + +### Out of scope + +- Package manager, remote modules, registry resolution, or dependency downloads. +- Dynamic module loading at VM runtime. +- New bytecode opcodes solely for module names. +- Host namespace redesign. +- Agent-specific storage or provider modules. + +## Implementation route + +### Milestone 1: Define compiler-owned identities + +**Files:** +- Create: `src/compiler/modules.rs` +- Modify: `src/compiler/source_loader.rs` +- Modify: `src/compiler/pipeline.rs` +- Test: compiler module tests + +Define: + +```text +ModuleId +SourceId +SymbolId +ModuleGraph +ModuleNode { source, imports, declarations, exports } +ResolvedImport +``` + +IDs are deterministic within one compilation and never derived only from a file stem. + +### Milestone 2: Parse import syntax into AST/IR + +**Files:** +- Modify RustScript parser/frontend import nodes +- Modify source-loader import discovery +- Test parser and module fixtures + +1. Stop using line-prefix stripping as the authoritative import parser. +2. Preserve import spans and clauses in the parsed unit. +3. Resolve `self::`, `super::`, namespace aliases, and named imports from structured nodes. +4. Keep host namespace imports on their existing dedicated resolution path. + +### Milestone 3: Build declarations and export tables + +**Files:** +- Modify frontend IR declaration metadata +- Modify `src/compiler/source_loader/graph.rs` +- Modify linker symbol collection + +1. Assign each declaration a symbol owned by its module. +2. Mark public exports explicitly. +3. Keep imported symbols separate from local declarations. +4. Prevent implicit transitive re-export. +5. Permit different modules to have private or public functions with the same source name. + +### Milestone 4: Resolve calls by symbol identity + +**Files:** +- Modify expression/call IR +- Modify `src/compiler/linker.rs` +- Modify lowering consumers + +1. Resolve local, named-import, and namespace calls to `SymbolId` before merge. +2. Replace string-based global function matching with symbol lookup. +3. Use deterministic internal mangling only at the final flat bytecode boundary if required. +4. Remove basename-only scope prefixes. + +### Milestone 5: Preserve source ownership through merge + +**Files:** +- Modify `src/compiler/pipeline.rs` +- Modify diagnostic/source-map structures +- Test rendered diagnostics + +1. Every span retains its source identity. +2. Merging units cannot reinterpret one module's offset in another source. +3. Parse, typing, duplicate symbol, visibility, and unresolved import errors render from the owning source. +4. Remove parallel `stmt_sources`, `function_sources`, and ad hoc prelude line remapping where replaced by source-owned IR. + +### Milestone 6: Remove textual compatibility machinery + +**Files:** +- Remove obsolete paths in `src/compiler/source_loader/rewrite.rs` +- Remove synthetic prelude generation and related line maps +- Update tests and compiler docs + +Do not retain a second module pipeline after semantic resolution reaches parity. + +### Milestone 7: Verification + +Required cases: + +- two directories containing modules with the same stem; +- two namespaces exporting the same function name; +- same-named private helpers in multiple modules; +- visibility errors and no transitive re-export; +- cycles through path aliases; +- in-memory overrides mixed with disk modules; +- deterministic output independent of import discovery order; +- source-correct diagnostics for every module. + +```bash +cargo fmt --all -- --check +cargo test --locked --test compiler_tests +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- File-module calls are resolved by `SymbolId`, not rewritten source text. +- Module identity never depends only on a basename. +- Same-named declarations in independent modules coexist. +- Public/private and re-export rules are represented in compiler data. +- Synthetic imported-function preludes are removed from the module path. +- Every diagnostic span retains its owning source after linking. +- No agent-specific compiler rule is introduced. diff --git a/plans/2026-08-09_static-builtin-id.md b/plans/2026-08-09_static-builtin-id.md new file mode 100644 index 00000000..6296ecd7 --- /dev/null +++ b/plans/2026-08-09_static-builtin-id.md @@ -0,0 +1,107 @@ +# Static Builtin ID Implementation Plan + +**Goal:** Replace count-derived builtin call indices with explicit static IDs that remain fixed after assignment. + +**Architecture:** Every VM-visible builtin receives an explicit `u16` ID in one authoritative catalog. `build.rs`, the interpreter, compiler, wire encoder/decoder, and `pd-vm-nostd` consume generated tables from that catalog. This migration may break existing VMBC once; the implementation will bump the bytecode ABI and reject the previous format instead of carrying an old-ID decoder. + +**Tech Stack:** Rust 2024, `build.rs` code generation, VMBC wire format, `pd-vm`, `pd-vm-nostd`. + +--- + +## Independence and dependency + +- Independent of agent framework, module loading, HTTP behavior, and JIT refactoring. +- Must land before more builtins are added. +- Later capability plans may key permissions by the static builtin ID. + +## Scope boundary + +### In scope + +- Explicit IDs for ordinary, internal, and special-call builtins. +- One authoritative catalog and generated forward/reverse lookup. +- A one-time VMBC ABI version bump. +- Compile-time duplicate/range validation. +- Shared std/no-std ID generation. + +### Out of scope + +- Compatibility decoding for prior VMBC versions. +- Aliases from old IDs to new IDs. +- New builtin behavior or host capabilities. +- Changes to source-language names. + +## Implementation route + +### Milestone 1: Freeze the ID contract with failing tests + +**Files:** +- Modify: `tests/wire/wire_tests.rs` +- Modify: `src/bytecode.rs` +- Add fixture/catalog tests under `tests/wire/` + +1. Add assertions for explicit IDs of representative ordinary, internal, and special builtins. +2. Add a uniqueness test over the complete catalog. +3. Add range tests proving static IDs do not overlap opcodes or reserved sentinels. +4. Add a test that appending a synthetic catalog entry does not change existing IDs. + +**RED command:** + +```bash +cargo test --locked --test wire_tests builtin +``` + +### Milestone 2: Introduce the authoritative catalog + +**Files:** +- Modify: `build.rs` +- Modify: `src/builtins/mod.rs` or create `src/builtins/catalog.rs` +- Modify generated builtin metadata consumers + +1. Define each entry as `{ id, source_name, Rust variant, class, feature gate }`. +2. Remove `BUILTIN_CALL_BASE` arithmetic from ID assignment. +3. Generate `BuiltinFunction::call_index`, reverse lookup, dispatch tables, and catalog iteration from explicit IDs. +4. Fail the build on duplicate IDs, duplicate names, out-of-range IDs, or a missing explicit ID. +5. Reserve documented ID blocks for ordinary, internal, and future extension entries without deriving IDs from catalog length. + +### Milestone 3: Share IDs with no-std + +**Files:** +- Modify: `pd-vm-nostd/src/vm.rs` +- Modify: `pd-vm-nostd/build.rs` or generate a shared checked-in artifact +- Modify: no-std wire tests + +1. Remove the duplicated `BUILTIN_BASE` constant. +2. Generate or import the same explicit ID table without requiring std-only dependencies. +3. Verify std compiler output executes under `pd-vm-nostd` with identical builtin dispatch. + +### Milestone 4: Declare the format break + +**Files:** +- Modify: `src/bytecode.rs` +- Modify: VMBC format tests and documentation + +1. Increment `BYTECODE_ABI_VERSION` once. +2. Reject the previous version with a deterministic unsupported-version error. +3. Do not add migration, dual decoding, or legacy aliases. +4. Regenerate only current-version fixtures. + +### Milestone 5: Full verification + +```bash +cargo fmt --all -- --check +cargo test --locked --test wire_tests +cargo test --locked -p pd-vm-nostd +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- Every VM-visible builtin has one explicit static ID. +- Adding or reordering catalog entries leaves all prior explicit IDs unchanged. +- Duplicate or missing IDs fail during generation. +- std and no-std use the same IDs without manually mirrored base arithmetic. +- VMBC declares the one-time incompatible format change and rejects the old version. +- No compatibility decoder or old-ID alias remains. diff --git a/plans/2026-08-09_structured-task-supervisor.md b/plans/2026-08-09_structured-task-supervisor.md new file mode 100644 index 00000000..40ae0db9 --- /dev/null +++ b/plans/2026-08-09_structured-task-supervisor.md @@ -0,0 +1,134 @@ +# Structured Task Supervisor Implementation Plan + +**Goal:** Add implementation-independent structured concurrency for multiple pending operations and isolated child program runs without exposing Rust futures, threads, or executor handles to scripts. + +**Architecture:** A run-scoped `TaskSupervisor` owns child operation/program tasks, concurrency permits, cancellation tree, result ordering, and cleanup. Tasks are descriptors validated against delegated capability profiles. Parent completion cannot leave active descendants. + +**Tech Stack:** Rust 2024, RunContext, HostRuntime operation registry, isolated VM instances, bounded executor. + +--- + +## Independence and dependency + +- Depends on VM decomposition and unified host lifecycle. +- Consumes static capability identity/profile delegation. +- Independent of agent tool/subagent policy; agent RSS may wrap it later. + +## Scope boundary + +### In scope + +- Multiple active operation/task records per run. +- Bounded `all`, `pool`, `race`, fail-fast, and isolated program fanout semantics. +- Parent/child cancellation and resource budgets. +- Ordered result collection and event association. +- Removal of the one-waiting-slot architectural limitation for structured tasks. + +### Out of scope + +- Source-language `async`, `await`, arbitrary futures, or shared-memory threads. +- Agent-specific tool, provider, or subagent descriptors. +- Mutable resource sharing between child VMs. +- Distributed execution or durable background jobs. + +## Target contracts + +```text +TaskSupervisor + spawn(descriptor, delegated_profile) + all(task_ids) + pool(descriptors, max_concurrency, fail_fast) + race(task_ids) + cancel(task_id/reason) + cancel_all(reason) + +TaskDescriptor + host operation + isolated program + input + +TaskResult + index + terminal status + return value or structured error + usage +``` + +## Implementation route + +### Milestone 1: Freeze structured semantics with tests + +Cover: + +- ordered all/pool results despite completion order; +- race returns first success and cancels remaining tasks; +- fail-fast cancellation; +- collect-all partial failures; +- parent cancellation reaches every descendant; +- child cancellation does not affect siblings by default; +- depth/fanout/active/time/fuel/operation limits; +- no child result/event after parent terminal state; +- isolated stacks/resources/capability profiles. + +### Milestone 2: Add TaskSupervisor to RunContext/HostRuntime + +**Files:** +- Create: `src/builtins/runtime/task.rs` +- Modify: RunContext and HostRuntime component files +- Modify: operation registry integration + +1. Store task state outside Instance's single wait marker. +2. Register every task/child operation in the shared operation registry. +3. Allocate permits before spawn. +4. Create child cancellation tokens under the run token. +5. Associate result/event/usage with task and parent run identity. + +### Milestone 3: Support isolated child programs + +1. Spawn a fresh Instance and RunContext from immutable Program/Engine references. +2. Delegate only a subset of the parent capability profile. +3. Prohibit mutable resource-handle transfer. +4. Bound child input/output/event bytes. +5. Finalize child outcome before collection. + +### Milestone 4: Add generic task host surface + +Expose implementation-independent operations such as: + +```text +task::all +task::pool +task::race +task::run_program +task::run_program_many +``` + +Descriptors contain only generic host-operation or program references. They cannot contain agent tool/provider names. + +### Milestone 5: Integrate scheduler wake/resume + +1. Permit multiple active task operations while Instance waits on one structured join/select result. +2. Wake the instance when the requested aggregate condition is met. +3. Keep remaining task state under supervisor ownership. +4. Cancel and clean all descendants before parent terminal completion. + +### Milestone 6: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --test runtime_task_tests +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +Include stress tests for permit exhaustion, cancellation races, deterministic ordering, nested depth, and cleanup counters. + +## Target criteria + +- One run can own multiple active tasks under explicit bounds. +- Structured joins/races have deterministic documented semantics. +- Parent terminal state implies zero active descendants. +- Child VMs share immutable program/engine data only. +- Capability delegation can only narrow access. +- Mutable resource handles never cross child boundaries. +- Agent/provider concepts do not appear in the core task descriptors. diff --git a/plans/2026-08-09_unified-host-lifecycle.md b/plans/2026-08-09_unified-host-lifecycle.md new file mode 100644 index 00000000..5a28066e --- /dev/null +++ b/plans/2026-08-09_unified-host-lifecycle.md @@ -0,0 +1,160 @@ +# Unified Host Resource, Operation, and Cancellation Plan + +**Goal:** Make every privileged runtime subsystem use one resource arena, operation registry, cancellation model, and cleanup lifecycle. + +**Architecture:** `HostRuntime` owns typed opaque resources, operation identity, and cancellation state. Blocking IO and SQLite resources use the shared arena/registry. Async host futures are submitted to and driven by the embedding `HostAsyncBridge`; the VM retains only script suspension and lifecycle identity. Cancellation carries a structured reason and propagates from run to bridge operation and resource cleanup. + +**Tech Stack:** Rust 2024, existing runtime hosts, `HostOpId`, HTTP client, SQLite, VM reset/drop tests. + +**Status:** Completed + +**Async correction:** The original HTTP/IO migration used subsystem pollers and, for HTTP, a per-request thread/runtime. Those transitional paths are superseded by `2026-08-09_http-transport-security-executor.md`. Completion of this plan does not authorize VM-, HTTP-, or IO-owned async executors. + +**Directory correction:** Generic async host contracts and VM lifecycle glue belong under `src/vm/async_host/`; each embedding's concrete driver belongs in its own dedicated async-host folder. `host.rs` and subsystem modules must remain binding/business-logic surfaces rather than async driver containers. + +--- + +## Independence and dependency + +- Depends on the HostRuntime/RunContext ownership contract from `2026-08-09_vm-runtime-decomposition.md`. +- Capability authorization can be implemented in parallel if it targets the same HostRuntime boundary. +- Agent lifecycle consumes this contract but is not implemented here. + +## Scope boundary + +### In scope + +- One opaque resource handle format and typed resource validation. +- One operation-ID/cancellation registry, with async future dispatch delegated to the embedding host bridge. +- Cancellation tree/reasons, deadlines, cleanup, and terminal state. +- Migration of blocking IO and SQLite state plus lifecycle identity for host-driven async operations. +- Removal of unused generic substrate and subsystem-specific duplicate registries. + +### Out of scope + +- Agent subagent semantics or provider retries. +- New filesystem/process/task host APIs. +- Source-language futures, `async`, or `await` syntax. +- Sharing mutable resources across VMs. + +## Target contracts + +```text +ResourceArena + insert(type, value, cleanup) + get(handle, expected_type) + close(handle, reason) + close_all(reason) + +OperationRegistry + start(owner, cancellation, cleanup) + complete(id, result) + cancel(id, reason) + cancel_all(reason) + +HostAsyncBridge + submit(id, future) + poll(id, waker) + cancel(id, reason) + +CancellationToken + parent + reason + deadline + child tokens +``` + +Handles must encode enough table/generation/type identity to reject stale, forged, cross-type, and cross-VM use. + +## Implementation route + +### Milestone 1: Freeze lifecycle semantics with tests + +Add tests for: + +- stale handle after close; +- handle reuse with generation change; +- wrong resource type; +- cross-VM handle rejection; +- operation completion/cancel race; +- reset/drop cleanup exactly once; +- parent cancellation propagation; +- timeout, user stop, resource close, and VM reset reasons. + +### Milestone 2: Replace the unused generic substrate + +**Files:** +- Modify: `src/builtins/runtime/resource.rs` +- Modify: `src/builtins/runtime/cancellation.rs` +- Modify: `src/vm/host_runtime.rs` + +1. Store opaque host resources, not only language `Value` objects. +2. Define a resource type identifier and cleanup contract. +3. Make operation owner/poll/cancel routing data-driven. +4. Remove APIs that remain unused after the contract is fixed. + +### Milestone 3: Migrate SQLite + +**Files:** +- Modify: `src/builtins/runtime/sqlite.rs` +- Modify: SQLite tests + +1. Replace SQLite-local handle counters and connection maps with ResourceArena handles. +2. Replace SQLite-local pending-op maps/signals with OperationRegistry. +3. Register `InterruptHandle` cancellation cleanup. +4. Ensure reset/drop waits only for the bounded documented grace period and no operation can re-enter a completed run. +5. Preserve path, SQL, row, byte, transaction, and authorizer limits. + +### Milestone 4: Establish HTTP lifecycle identity + +**Files:** +- Modify: `src/builtins/runtime/http.rs` +- Modify: HTTP tests + +1. Allocate HTTP-visible wait identities through the shared operation registry. +2. Connect run cancellation/reset/drop to the host async bridge. +3. Do not store or poll HTTP futures through HTTP-specific resources. +4. Remove HTTP-specific pending dispatch from `runtime/mod.rs` during the async host migration. + +### Milestone 5: Migrate IO and other existing resources + +**Files:** +- Modify IO runtime modules and VM host polling + +Keep blocking file/iterator/callback resources in the shared arena/registry. Async IO futures use the embedding host bridge under the `async` feature. Delete subsystem counters/maps and pollers after migration. + +### Milestone 6: Centralize wait/poll/cancel + +**Files:** +- Modify: `src/builtins/runtime/mod.rs` +- Modify: `src/vm/host.rs` +- Modify: `src/vm/mod.rs` or new component files + +1. Replace subsystem `if` chains with one bridge dispatch for async host operations. +2. Let Instance wait on an operation ID while HostRuntime owns lifecycle state and the embedding owns the future. +3. Route run cancellation, deadline, resource close, reset, and drop through one cancellation API. +4. Guarantee one terminal transition and one cleanup execution. + +### Milestone 7: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --test runtime_context_tests +cargo test --locked --test runtime_host_tests +cargo test --locked --test http_host_tests --features http-client +cargo test --locked --test sqlite_host_tests --features sqlite +cargo test --locked --workspace --all-features +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +## Target criteria + +- Production blocking IO/SQLite paths use the shared arena and registry; async IO/HTTP use shared lifecycle IDs plus the embedding host bridge. +- No subsystem owns an independent public operation-ID namespace. +- Cancellation reasons remain structured from run through cleanup. +- Close/reset/drop invoke cleanup once and reject stale handles afterward. +- A terminal run cannot receive a late operation result. +- Generic resource/operation code has production callers and no broad dead-code warnings. +- Per-subsystem async polling chains are removed. +- No HTTP/IO path creates a private thread, runtime, oneshot completion scheduler, or executor. diff --git a/plans/2026-08-09_vm-runtime-decomposition.md b/plans/2026-08-09_vm-runtime-decomposition.md new file mode 100644 index 00000000..2c80b101 --- /dev/null +++ b/plans/2026-08-09_vm-runtime-decomposition.md @@ -0,0 +1,150 @@ +# VM Runtime Decomposition Implementation Plan + +**Goal:** Split the current monolithic `Vm` state into explicit engine, program, instance, run-context, and host-runtime ownership layers. + +**Architecture:** Immutable compiled artifacts and backend caches live outside per-run execution state. An `Instance` owns interpreter state, a `RunContext` owns one execution's input/budgets/events/cancellation, and `HostRuntime` owns capabilities/resources/operations. The migration preserves observable execution behavior while removing subsystem-specific fields from the central VM object. + +**Tech Stack:** Rust 2024, `pd-vm` interpreter/JIT/AOT integration, existing compiler and runtime tests. + +--- + +## Independence and dependency + +- Static builtin IDs should land first so decomposition does not move an unstable wire catalog. +- Defines ownership required by the unified host-lifecycle and RunOutcome plans. +- Independent of agent providers, gateway routes, module semantics, and new host functions. + +## Scope boundary + +### In scope + +- Ownership split for program, backend cache, interpreter instance, run-scoped context, and host runtime. +- Explicit reset/drop semantics for each layer. +- Removal of subsystem fields from the top-level VM facade. +- Migration of embedding entry points to the new ownership model. + +### Out of scope + +- New language syntax or bytecode operations. +- New host capabilities. +- Compatibility adapters for every prior internal API. +- JIT/AOT optimization redesign. +- Agent-specific execution policy. + +## Target model + +```text +Engine + backend configuration + decoded/JIT/AOT caches + code-generation telemetry + +Program + immutable bytecode + constants and metadata + import requirements + +Instance + instruction pointer + stack, locals, frames, captures + yield/wait state + +RunContext + input + event channel + fuel/deadline/cancellation + usage accounting + +HostRuntime + capability profile + resources + operations + executor +``` + +The public facade may be renamed or retained, but ownership must follow this model. + +## Implementation route + +### Milestone 1: Add ownership tests + +**Files:** +- Add focused tests under `tests/vm/` +- Modify reset/reuse tests + +Prove: + +- one immutable program can create multiple isolated instances; +- run input/events/budgets never leak between runs; +- backend cache may be shared without sharing stacks/resources; +- reset closes run-scoped state and retains only documented reusable state. + +### Milestone 2: Extract immutable Program and Engine state + +**Files:** +- Modify: `src/vm/mod.rs` +- Create: `src/vm/engine.rs` +- Create or refine: `src/vm/program.rs` +- Move backend cache ownership from VM fields + +1. Remove raw program pointer/cache duplication from per-run state. +2. Give Engine explicit cache keys and invalidation rules. +3. Keep Program immutable after validation/binding metadata construction. +4. Test program sharing across interpreter-only, JIT, and AOT configurations. + +### Milestone 3: Extract Instance state + +**Files:** +- Create: `src/vm/instance.rs` +- Modify interpreter dispatch and frame helpers + +Move IP, stack, locals, frames, captures, callbacks, waiting/yield state, and instance-only counters. Define one lifecycle from new to halted/failed/cancelled. + +### Milestone 4: Introduce RunContext + +**Files:** +- Create: `src/vm/run_context.rs` +- Move runtime input, event sink, fuel, epoch/deadline, cancellation, and usage state + +1. Create a fresh RunContext per execution. +2. Make cancellation and deadline mandatory run-owned data, with explicit unlimited settings where allowed. +3. Remove source injection and embedding-global event ownership from execution paths. +4. Make run completion consume/finalize the context. + +### Milestone 5: Extract HostRuntime shell + +**Files:** +- Create: `src/vm/host_runtime.rs` +- Modify: `src/vm/host.rs` +- Modify: `src/builtins/runtime/mod.rs` + +Move capability profile, host bindings, resource tables, operation registry, and executor references behind HostRuntime. Subsystem migration proceeds in the separate host-lifecycle plan. + +### Milestone 6: Remove duplicate lifecycle paths + +1. Replace central constructor/reset/drop field lists with component lifecycle methods. +2. Remove fields that exist only as transitional mirrors. +3. Remove old internal APIs once all callers move; no long-lived compatibility layer. +4. Document thread-safety and clone semantics for Engine, Program, Instance, RunContext, and HostRuntime. + +### Milestone 7: Verification + +```bash +cargo fmt --all -- --check +cargo test --locked --workspace --all-features +cargo test --locked -p pd-vm-nostd +cargo clippy --locked --workspace --all-targets --all-features -- -D warnings +git diff --check +``` + +Add behavioral comparison fixtures that run the same program before and after each extraction milestone. + +## Target criteria + +- Immutable Program data and backend caches are not owned by per-run state. +- Stack/frame/wait state is isolated in Instance. +- Input/events/budget/cancellation are isolated in RunContext. +- Capabilities/resources/operations are isolated in HostRuntime. +- Reset and drop no longer enumerate every runtime subsystem in one central method. +- Multiple instances from one program cannot share mutable run or host resources. +- Existing interpreter/JIT/AOT/no-std behavior tests remain passing. diff --git a/src/builtins/runtime/cancellation.rs b/src/builtins/runtime/cancellation.rs new file mode 100644 index 00000000..ef8dbc67 --- /dev/null +++ b/src/builtins/runtime/cancellation.rs @@ -0,0 +1,1006 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Instant; + +use super::error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; +use super::resource::ResourceHandle; + +pub const DEFAULT_MAX_PENDING_OPERATIONS: usize = 64; +const TERMINAL_BIT: u8 = 0x80; +const REASON_MASK: u8 = !TERMINAL_BIT; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct OperationId(u64); + +impl OperationId { + pub fn from_raw(raw: u64) -> RuntimeResult { + if raw == 0 { + return Err(RuntimeError::new( + RuntimeErrorCode::OperationIdExhausted, + "runtime::operation", + "operation id zero is reserved", + )); + } + Ok(Self(raw)) + } + + pub const fn raw(self) -> u64 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum OperationOwner { + HostBridge, + Io, + Http, + #[cfg(feature = "sqlite")] + Sqlite, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum CancellationReason { + Requested = 1, + Deadline = 2, + VmReset = 3, + Parent = 4, + ResourceClosed = 5, +} + +impl CancellationReason { + pub const fn as_str(self) -> &'static str { + match self { + Self::Requested => "requested", + Self::Deadline => "deadline", + Self::VmReset => "vm_reset", + Self::Parent => "parent", + Self::ResourceClosed => "resource_closed", + } + } + + fn from_raw(raw: u8) -> Option { + match raw { + 1 => Some(Self::Requested), + 2 => Some(Self::Deadline), + 3 => Some(Self::VmReset), + 4 => Some(Self::Parent), + 5 => Some(Self::ResourceClosed), + _ => None, + } + } +} + +struct CancellationSignal { + state: AtomicU8, + deadline: Option, + children: Mutex>>, + propagation_error: Mutex>, +} + +impl CancellationSignal { + fn mark_cancelled(&self, reason: CancellationReason) -> bool { + self.state + .compare_exchange(0, reason as u8, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + fn cancel(&self, reason: CancellationReason) -> (bool, Option) { + let transitioned = self.mark_cancelled(reason); + let mut first_error = None; + if transitioned { + let children = self + .children + .lock() + .expect("cancellation children lock should not be poisoned") + .iter() + .filter_map(Weak::upgrade) + .collect::>(); + for child in children { + if let Err(error) = child.cancel(reason) + && first_error.is_none() + { + first_error = Some(error); + } + } + } + (transitioned, first_error) + } + + fn store_propagation_error(&self, error: Option) { + if let Some(error) = error { + let mut stored = self + .propagation_error + .lock() + .expect("cancellation propagation error lock should not be poisoned"); + if stored.is_none() { + *stored = Some(error); + } + } + } + + fn take_propagation_error(&self) -> Option { + self.propagation_error + .lock() + .expect("cancellation propagation error lock should not be poisoned") + .take() + } + + fn reason(&self) -> Option { + let state = self.state.load(Ordering::Acquire); + if state == 0 + && self + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + { + let (_, error) = self.cancel(CancellationReason::Deadline); + self.store_propagation_error(error); + } + CancellationReason::from_raw(self.state.load(Ordering::Acquire) & REASON_MASK) + } + + fn finish_success(&self) -> bool { + self.state + .compare_exchange(0, TERMINAL_BIT, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } + + fn finish_cancelled(&self, requested: CancellationReason) -> CancellationReason { + loop { + let state = self.state.load(Ordering::Acquire); + let reason = CancellationReason::from_raw(state & REASON_MASK).unwrap_or(requested); + if state & TERMINAL_BIT != 0 { + return reason; + } + let terminal = TERMINAL_BIT | reason as u8; + if self + .state + .compare_exchange(state, terminal, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return reason; + } + } + } +} + +#[derive(Clone)] +pub struct CancellationToken { + id: OperationId, + signal: Arc, +} + +impl CancellationToken { + pub(crate) fn root() -> Self { + Self { + id: OperationId(u64::MAX), + signal: Arc::new(CancellationSignal { + state: AtomicU8::new(0), + deadline: None, + children: Mutex::new(Vec::new()), + propagation_error: Mutex::new(None), + }), + } + } + + pub fn is_cancelled(&self) -> bool { + self.reason().is_some() + } + + pub fn reason(&self) -> Option { + self.signal.reason() + } + + pub fn cancel(&self, reason: CancellationReason) -> bool { + let (transitioned, error) = self.signal.cancel(reason); + self.signal.store_propagation_error(error); + transitioned + } + + pub(crate) fn take_propagation_error(&self) -> Option { + self.signal.take_propagation_error() + } + + pub(crate) fn mark_cancelled(&self, reason: CancellationReason) -> bool { + self.signal.mark_cancelled(reason) + } + + pub fn check(&self) -> RuntimeResult<()> { + let Some(reason) = self.reason() else { + return Ok(()); + }; + Err(RuntimeError::new( + RuntimeErrorCode::OperationCancelled, + "runtime::operation", + format!( + "operation {} was cancelled ({})", + self.id.raw(), + reason.as_str() + ), + ) + .with_value(self.id.raw())) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OperationStatus { + Pending, + Completed, + Cancelled(CancellationReason), + Failed(RuntimeError), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum OperationEnd { + Completed, + Cancelled(CancellationReason), + Failed(RuntimeError), +} + +pub type OperationCleanup = Box RuntimeResult<()> + Send + 'static>; + +struct OperationInner { + status: OperationStatus, + cleanup: Option, + payload: Option, + resource: Option, +} + +struct RegistryInner { + operations: Mutex>, +} + +struct OperationCore { + id: OperationId, + owner: OperationOwner, + token: CancellationToken, + inner: Mutex, +} + +impl OperationCore { + fn status(&self) -> OperationStatus { + self.inner + .lock() + .expect("operation state lock should not be poisoned") + .status + .clone() + } + + fn cancel(&self, reason: CancellationReason) -> RuntimeResult { + let _ = self.token.reason(); + let (_, child_error) = self.token.signal.cancel(reason); + let child_error = child_error.or_else(|| self.token.signal.take_propagation_error()); + let cleanup = { + let mut inner = self + .inner + .lock() + .expect("operation state lock should not be poisoned"); + if !matches!(inner.status, OperationStatus::Pending) { + return Ok(false); + } + let reason = self.token.reason().unwrap_or(reason); + let reason = self.token.signal.finish_cancelled(reason); + inner.status = OperationStatus::Cancelled(reason); + (inner.cleanup.take(), reason) + }; + let cleanup_result = if let (Some(cleanup), reason) = cleanup { + cleanup(OperationEnd::Cancelled(reason)).map_err(|error| { + RuntimeError::new( + RuntimeErrorCode::OperationCleanupFailed, + "runtime::operation", + error.to_string(), + ) + .with_value(self.id.raw()) + }) + } else { + Ok(()) + }; + match (child_error, cleanup_result) { + (Some(error), _) => Err(error), + (None, Err(error)) => Err(error), + (None, Ok(())) => Ok(true), + } + } + + fn complete(&self) -> RuntimeResult { + if let Some(reason) = self.token.reason() { + return self.cancel(reason); + } + self.finish(OperationStatus::Completed, OperationEnd::Completed) + } + + fn fail(&self, error: RuntimeError) -> RuntimeResult { + if let Some(reason) = self.token.reason() { + return self.cancel(reason); + } + self.finish( + OperationStatus::Failed(error.clone()), + OperationEnd::Failed(error), + ) + } + + fn finish(&self, status: OperationStatus, end: OperationEnd) -> RuntimeResult { + let (cleanup, end) = { + let mut inner = self + .inner + .lock() + .expect("operation state lock should not be poisoned"); + if !matches!(inner.status, OperationStatus::Pending) { + return Ok(false); + } + let end = if self.token.signal.finish_success() { + inner.status = status; + end + } else { + let reason = self + .token + .reason() + .expect("a failed success transition must carry cancellation"); + let reason = self.token.signal.finish_cancelled(reason); + inner.status = OperationStatus::Cancelled(reason); + OperationEnd::Cancelled(reason) + }; + (inner.cleanup.take(), end) + }; + let cleanup_result = if let Some(cleanup) = cleanup { + cleanup(end).map_err(|error| { + RuntimeError::new( + RuntimeErrorCode::OperationCleanupFailed, + "runtime::operation", + error.to_string(), + ) + .with_value(self.id.raw()) + }) + } else { + Ok(()) + }; + cleanup_result?; + Ok(true) + } + + fn attach_parent(self: &Arc, parent: &CancellationToken) -> RuntimeResult<()> { + { + let mut children = parent + .signal + .children + .lock() + .expect("cancellation children lock should not be poisoned"); + children.retain(|child| child.strong_count() > 0); + children.push(Arc::downgrade(self)); + } + if parent.is_cancelled() { + self.cancel(parent.reason().unwrap_or(CancellationReason::Parent))?; + } + Ok(()) + } +} + +#[derive(Clone)] +pub struct OperationState { + core: Arc, +} + +impl fmt::Debug for OperationState { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OperationState") + .field("id", &self.id()) + .field("owner", &self.owner()) + .field("status", &self.status()) + .finish() + } +} + +impl OperationState { + pub fn id(&self) -> OperationId { + self.core.id + } + + pub fn owner(&self) -> OperationOwner { + self.core.owner + } + + pub fn token(&self) -> CancellationToken { + self.core.token.clone() + } + + pub fn status(&self) -> OperationStatus { + self.core.status() + } + + pub fn set_payload(&self, payload: ResourceHandle) { + self.core + .inner + .lock() + .expect("operation state lock should not be poisoned") + .payload = Some(payload); + } + + #[cfg(feature = "sqlite")] + pub(crate) fn set_cleanup(&self, cleanup: OperationCleanup) -> RuntimeResult<()> { + let mut inner = self + .core + .inner + .lock() + .expect("operation state lock should not be poisoned"); + if !matches!(inner.status, OperationStatus::Pending) { + return Err(RuntimeError::new( + RuntimeErrorCode::OperationAlreadyFinished, + "runtime::operation", + "cannot attach cleanup to a terminal operation", + ) + .with_value(self.id().raw())); + } + if inner.cleanup.is_some() { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::operation", + "operation cleanup is already configured", + ) + .with_value(self.id().raw())); + } + inner.cleanup = Some(cleanup); + Ok(()) + } + + pub fn payload(&self) -> Option { + self.core + .inner + .lock() + .expect("operation state lock should not be poisoned") + .payload + } + + pub fn set_resource(&self, resource: ResourceHandle) { + self.core + .inner + .lock() + .expect("operation state lock should not be poisoned") + .resource = Some(resource); + } + + pub fn resource(&self) -> Option { + self.core + .inner + .lock() + .expect("operation state lock should not be poisoned") + .resource + } + + pub fn cancel(&self, reason: CancellationReason) -> RuntimeResult { + self.core.cancel(reason) + } + + pub fn complete(&self) -> RuntimeResult { + self.core.complete() + } + + pub fn fail(&self, error: RuntimeError) -> RuntimeResult { + self.core.fail(error) + } + + fn build( + id: OperationId, + owner: OperationOwner, + deadline: Option, + cleanup: Option, + ) -> Self { + let token = CancellationToken { + id, + signal: Arc::new(CancellationSignal { + state: AtomicU8::new(0), + deadline, + children: Mutex::new(Vec::new()), + propagation_error: Mutex::new(None), + }), + }; + Self { + core: Arc::new(OperationCore { + id, + owner, + token, + inner: Mutex::new(OperationInner { + status: OperationStatus::Pending, + cleanup, + payload: None, + resource: None, + }), + }), + } + } +} + +pub struct OperationRegistry { + max_pending: usize, + next_id: u64, + last_external_id: u64, + inner: Arc, +} + +impl OperationRegistry { + pub fn with_limit(max_pending: usize) -> RuntimeResult { + if max_pending == 0 { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::operation", + "operation registry capacity must be positive", + )); + } + Ok(Self { + max_pending, + next_id: 1, + last_external_id: 0, + inner: Arc::new(RegistryInner { + operations: Mutex::new(HashMap::new()), + }), + }) + } + + pub fn active_count(&self) -> usize { + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .values() + .filter(|operation| !matches!(operation.status(), OperationStatus::Cancelled(_))) + .count() + } + + pub(crate) fn allocate_id(&mut self) -> RuntimeResult { + let id = OperationId::from_raw(self.next_id)?; + self.next_id = self.next_id.checked_add(1).ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::OperationIdExhausted, + "runtime::operation", + "operation id space exhausted", + ) + })?; + Ok(id) + } + + pub fn start_owned( + &mut self, + owner: OperationOwner, + parent: Option<&CancellationToken>, + deadline: Option, + cleanup: Option, + ) -> RuntimeResult { + if self.active_count() >= self.max_pending { + return Err(RuntimeError::new( + RuntimeErrorCode::OperationLimitExceeded, + "runtime::operation", + "pending operation capacity has been reached", + ) + .with_limit(self.max_pending)); + } + let id = self.allocate_id()?; + let operation = OperationState::build(id, owner, deadline, cleanup); + if let Some(parent) = parent { + operation.core.attach_parent(parent)?; + } + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .insert(id, operation.clone()); + Ok(operation) + } + + #[cfg(test)] + pub fn register_external( + &mut self, + id: OperationId, + owner: OperationOwner, + parent: Option<&CancellationToken>, + deadline: Option, + cleanup: Option, + ) -> RuntimeResult { + self.retire_external_id(id)?; + self.register_retired_external(id, owner, parent, deadline, cleanup) + } + + pub(crate) fn retire_external_id(&mut self, id: OperationId) -> RuntimeResult<()> { + if id.raw() <= self.last_external_id { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::operation", + format!( + "external operation {} is not newer than the last external operation {}", + id.raw(), + self.last_external_id + ), + ) + .with_value(id.raw())); + } + let next_id = id.raw().checked_add(1).ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::OperationIdExhausted, + "runtime::operation", + "operation id space exhausted", + ) + })?; + self.last_external_id = id.raw(); + self.next_id = self.next_id.max(next_id); + Ok(()) + } + + pub(crate) fn register_retired_external( + &mut self, + id: OperationId, + owner: OperationOwner, + parent: Option<&CancellationToken>, + deadline: Option, + cleanup: Option, + ) -> RuntimeResult { + if id.raw() != self.last_external_id { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::operation", + format!("external operation {} has not just been retired", id.raw()), + ) + .with_value(id.raw())); + } + if self.active_count() >= self.max_pending { + return Err(RuntimeError::new( + RuntimeErrorCode::OperationLimitExceeded, + "runtime::operation", + "pending operation capacity has been reached", + ) + .with_limit(self.max_pending)); + } + let registered = self + .inner + .operations + .lock() + .expect("operation registry lock should not be poisoned"); + if registered.contains_key(&id) { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::operation", + format!("operation {} is already registered", id.raw()), + ) + .with_value(id.raw())); + } + let operation = OperationState::build(id, owner, deadline, cleanup); + drop(registered); + if let Some(parent) = parent { + operation.core.attach_parent(parent)?; + } + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .insert(id, operation.clone()); + Ok(operation) + } + + pub fn get(&self, id: OperationId) -> RuntimeResult { + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .get(&id) + .cloned() + .ok_or_else(|| operation_not_found(id)) + } + + pub fn operations_by_owner(&self, owner: OperationOwner) -> Vec { + let operations = self.registered_operations(); + operations + .into_iter() + .filter(|operation| operation.owner() == owner) + .collect() + } + + pub fn operations_for_resource(&self, resource: ResourceHandle) -> Vec { + let operations = self.registered_operations(); + operations + .into_iter() + .filter(|operation| operation.resource() == Some(resource)) + .collect() + } + + pub fn cancel(&mut self, id: OperationId, reason: CancellationReason) -> RuntimeResult { + self.take_operation(id)?.cancel(reason) + } + + pub fn complete(&mut self, id: OperationId) -> RuntimeResult { + self.take_operation(id)?.complete() + } + + pub fn fail(&mut self, id: OperationId, error: RuntimeError) -> RuntimeResult { + self.take_operation(id)?.fail(error) + } + + pub fn cancel_all(&mut self, reason: CancellationReason) -> RuntimeResult { + let operations = { + let mut registered = self + .inner + .operations + .lock() + .expect("operation registry lock should not be poisoned"); + std::mem::take(&mut *registered) + }; + let operations = operations.into_values().collect::>(); + for operation in &operations { + operation.token().mark_cancelled(reason); + } + let mut first_error = None; + for operation in &operations { + if let Err(error) = operation.cancel(reason) { + first_error.get_or_insert(error); + } + } + match first_error { + Some(error) => Err(error), + None => Ok(operations + .iter() + .filter(|operation| matches!(operation.status(), OperationStatus::Cancelled(_))) + .count()), + } + } + + fn registered_operations(&self) -> Vec { + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .values() + .cloned() + .collect() + } + + fn take_operation(&mut self, id: OperationId) -> RuntimeResult { + self.inner + .operations + .lock() + .expect("operation registry lock should not be poisoned") + .remove(&id) + .ok_or_else(|| operation_not_found(id)) + } +} + +impl Default for OperationRegistry { + fn default() -> Self { + Self::with_limit(DEFAULT_MAX_PENDING_OPERATIONS) + .expect("default operation registry configuration should be valid") + } +} + +impl Drop for OperationRegistry { + fn drop(&mut self) { + let _ = self.cancel_all(CancellationReason::VmReset); + } +} + +fn operation_not_found(id: OperationId) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::OperationNotFound, + "runtime::operation", + format!("operation {} is not registered", id.raw()), + ) + .with_value(id.raw()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, Instant}; + + use super::super::error::{RuntimeError, RuntimeErrorCode}; + use super::{ + CancellationReason, OperationId, OperationOwner, OperationRegistry, OperationStatus, + }; + + #[test] + fn token_reports_the_first_cancellation_reason() { + let mut registry = OperationRegistry::with_limit(4).expect("registry should be valid"); + let operation = registry + .start_owned(OperationOwner::Io, None, None, None) + .expect("operation should start"); + let token = operation.token(); + assert!(token.cancel(CancellationReason::Deadline)); + assert!(!token.cancel(CancellationReason::Parent)); + assert_eq!(token.reason(), Some(CancellationReason::Deadline)); + assert_eq!(operation.status(), OperationStatus::Pending); + } + + #[test] + fn parent_cancellation_propagates_and_deadline_is_structured() { + let mut registry = OperationRegistry::with_limit(4).expect("registry should be valid"); + let parent = registry + .start_owned(OperationOwner::Http, None, None, None) + .expect("parent should start"); + let child = registry + .start_owned(OperationOwner::Http, Some(&parent.token()), None, None) + .expect("child should start"); + assert!( + parent + .cancel(CancellationReason::Requested) + .expect("parent cancellation should succeed") + ); + assert_eq!(child.token().reason(), Some(CancellationReason::Requested)); + + let deadline_parent = registry + .start_owned(OperationOwner::Io, None, None, None) + .expect("deadline parent should start"); + let expired = registry + .start_owned( + OperationOwner::Io, + Some(&deadline_parent.token()), + Some(Instant::now() - Duration::from_millis(1)), + None, + ) + .expect("deadline child should start"); + assert_eq!(expired.token().reason(), Some(CancellationReason::Deadline)); + } + + #[test] + fn cancel_all_counts_children_cancelled_by_parent_propagation() { + let mut registry = OperationRegistry::with_limit(2).expect("registry should be valid"); + let parent = registry + .start_owned(OperationOwner::Http, None, None, None) + .expect("parent should start"); + let child = registry + .start_owned(OperationOwner::Io, Some(&parent.token()), None, None) + .expect("child should start"); + + assert_eq!( + registry + .cancel_all(CancellationReason::VmReset) + .expect("all operations should cancel"), + 2 + ); + assert_eq!( + parent.status(), + OperationStatus::Cancelled(CancellationReason::VmReset) + ); + assert!(matches!( + child.status(), + OperationStatus::Cancelled(CancellationReason::Parent | CancellationReason::VmReset) + )); + assert_eq!(registry.active_count(), 0); + } + + #[test] + fn parent_cancellation_finishes_registered_children_and_releases_capacity() { + let child_cleanup_count = Arc::new(AtomicUsize::new(0)); + let cleanup_count = Arc::clone(&child_cleanup_count); + let mut registry = OperationRegistry::with_limit(2).expect("registry should be valid"); + let parent = registry + .start_owned(OperationOwner::Http, None, None, None) + .expect("parent should start"); + let child = registry + .start_owned( + OperationOwner::Io, + Some(&parent.token()), + None, + Some(Box::new(move |end| { + assert_eq!( + end, + super::OperationEnd::Cancelled(CancellationReason::Requested) + ); + cleanup_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + })), + ) + .expect("child should start"); + + assert!( + parent + .cancel(CancellationReason::Requested) + .expect("parent should cancel") + ); + + assert_eq!( + child.status(), + OperationStatus::Cancelled(CancellationReason::Requested) + ); + assert_eq!(child_cleanup_count.load(Ordering::SeqCst), 1); + assert_eq!(registry.active_count(), 0); + assert!(registry.get(child.id()).is_ok()); + registry + .start_owned(OperationOwner::Io, None, None, None) + .expect("parent cancellation should release registry capacity"); + assert!( + !child + .cancel(CancellationReason::Requested) + .expect("child cancellation should remain idempotent") + ); + assert_eq!(child_cleanup_count.load(Ordering::SeqCst), 1); + } + + #[test] + fn parent_cancellation_propagates_child_cleanup_failure() { + let mut registry = OperationRegistry::with_limit(2).expect("registry should be valid"); + let parent = registry + .start_owned(OperationOwner::Http, None, None, None) + .expect("parent should start"); + registry + .start_owned( + OperationOwner::Io, + Some(&parent.token()), + None, + Some(Box::new(|_| { + Err(RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "test::cleanup", + "child cleanup failed", + )) + })), + ) + .expect("child should start"); + + let error = parent + .cancel(CancellationReason::Requested) + .expect_err("child cleanup failure should propagate"); + assert_eq!(error.code(), RuntimeErrorCode::OperationCleanupFailed); + } + + #[test] + fn completed_external_operation_ids_cannot_be_reused() { + let mut registry = OperationRegistry::with_limit(2).expect("registry should be valid"); + let id = OperationId::from_raw(7).expect("operation id should be valid"); + registry + .register_external(id, OperationOwner::HostBridge, None, None, None) + .expect("first external operation should register"); + registry + .complete(id) + .expect("external operation should complete"); + + let error = registry + .register_external(id, OperationOwner::HostBridge, None, None, None) + .expect_err("completed external operation id must remain retired"); + assert_eq!(error.code(), RuntimeErrorCode::InvalidConfiguration); + } + + #[test] + fn rejected_external_operation_ids_are_retired() { + let mut registry = OperationRegistry::with_limit(1).expect("registry should be valid"); + let active = registry + .start_owned(OperationOwner::Io, None, None, None) + .expect("capacity should be occupied"); + let id = OperationId::from_raw(7).expect("operation id should be valid"); + let error = registry + .register_external(id, OperationOwner::HostBridge, None, None, None) + .expect_err("external operation should exceed capacity"); + assert_eq!(error.code(), RuntimeErrorCode::OperationLimitExceeded); + registry + .complete(active.id()) + .expect("capacity should be released"); + + let error = registry + .register_external(id, OperationOwner::HostBridge, None, None, None) + .expect_err("rejected external operation id must remain retired"); + assert_eq!(error.code(), RuntimeErrorCode::InvalidConfiguration); + } + + #[test] + fn attaching_children_prunes_completed_parent_links() { + let mut registry = OperationRegistry::with_limit(2).expect("registry should be valid"); + let parent = registry + .start_owned(OperationOwner::Http, None, None, None) + .expect("parent should start"); + + for _ in 0..32 { + let child = registry + .start_owned(OperationOwner::Io, Some(&parent.token()), None, None) + .expect("child should start"); + registry + .complete(child.id()) + .expect("child should complete"); + } + + let live_links = parent + .token() + .signal + .children + .lock() + .expect("children lock") + .len(); + assert!(live_links <= 1, "completed child links should be pruned"); + } +} diff --git a/src/builtins/runtime/context.rs b/src/builtins/runtime/context.rs new file mode 100644 index 00000000..2588e23b --- /dev/null +++ b/src/builtins/runtime/context.rs @@ -0,0 +1,137 @@ +use super::error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; +use super::event::{EventEmitter, EventLimits, EventReceipt, EventSink}; +use crate::vm::{Value, VmResult}; + +pub const RUNTIME_INPUT_NAME: &str = "runtime::input"; +#[allow(dead_code)] +pub const RUNTIME_EMIT_NAME: &str = "runtime::emit"; + +#[allow(dead_code)] +pub type RuntimeEventSink = dyn EventSink; + +/// Configuration for one VM/run-scoped generic runtime context. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RuntimeContextConfig { + event_limits: EventLimits, +} + +impl RuntimeContextConfig { + pub const fn new(event_limits: EventLimits) -> Self { + Self { event_limits } + } + + pub const fn event_limits(self) -> EventLimits { + self.event_limits + } +} + +impl Default for RuntimeContextConfig { + fn default() -> Self { + Self::new(EventLimits::default()) + } +} + +/// Run-scoped input and generic event transport hooks. +/// +/// The context stores values as VM [`Value`]s and delegates event persistence/delivery to the +/// embedding. It has no knowledge of sessions, providers, platforms, or event names. +pub struct RuntimeContext { + input: Option, + events: EventEmitter, +} + +#[allow(dead_code)] +impl RuntimeContext { + pub fn with_config(config: RuntimeContextConfig) -> RuntimeResult { + Ok(Self { + input: None, + events: EventEmitter::new(config.event_limits()), + }) + } + + pub fn config(&self) -> RuntimeContextConfig { + RuntimeContextConfig::new(self.events.limits()) + } + + pub fn set_input(&mut self, value: Value) -> RuntimeResult<()> { + self.input = Some(value); + Ok(()) + } + + pub fn clear_input(&mut self) { + self.input = None; + } + + pub fn reset_for_reuse(&mut self) { + self.input = None; + self.events.reset_for_reuse(); + } + + pub fn input(&self) -> RuntimeResult { + self.input.clone().ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::InputUnavailable, + RUNTIME_INPUT_NAME, + "run input has not been configured", + ) + }) + } + + pub fn set_event_sink(&mut self, sink: S) -> RuntimeResult<()> + where + S: EventSink + 'static, + { + self.events.set_sink(sink); + Ok(()) + } + + pub fn clear_event_sink(&mut self) { + self.events.clear_sink(); + } + + pub fn emit(&mut self, value: Value) -> RuntimeResult { + self.events.emit(value) + } + + pub fn emitted_events(&self) -> u64 { + self.events.emitted_events() + } + + pub fn event_limits(&self) -> EventLimits { + self.events.limits() + } +} + +impl Default for RuntimeContext { + fn default() -> Self { + Self::with_config(RuntimeContextConfig::default()) + .expect("default runtime context configuration should be valid") + } +} + +/// Parent registration helper for the zero-argument `runtime::input()` host function. +pub fn runtime_input(context: &RuntimeContext) -> VmResult { + context + .input() + .map_err(|error| crate::vm::VmError::HostError(error.to_string())) +} + +/// Parent registration helper for the one-argument `runtime::emit(value)` host function. +pub fn runtime_emit(context: &mut RuntimeContext, value: Value) -> VmResult<()> { + context + .emit(value) + .map(|_| ()) + .map_err(|error| crate::vm::VmError::HostError(error.to_string())) +} + +#[cfg(test)] +mod tests { + use super::{RUNTIME_EMIT_NAME, RUNTIME_INPUT_NAME, RuntimeContext}; + + #[test] + fn host_names_are_generic_and_stable() { + assert_eq!(RUNTIME_INPUT_NAME, "runtime::input"); + assert_eq!(RUNTIME_EMIT_NAME, "runtime::emit"); + assert!(std::mem::size_of::() > 0); + } +} diff --git a/src/builtins/runtime/context_host.rs b/src/builtins/runtime/context_host.rs new file mode 100644 index 00000000..d4c32b0f --- /dev/null +++ b/src/builtins/runtime/context_host.rs @@ -0,0 +1,29 @@ +use pd_host_function::pd_host_function; + +use super::AnyValue; +use crate::vm::{Value, Vm, VmResult}; + +/// Returns the embedding-provided input for the current run. +#[pd_host_function(name = "runtime::input")] +fn runtime_input_impl(vm: &mut Vm) -> VmResult { + vm.runtime_input_value() +} + +/// Returns the run-scoped input encoded with the runtime's strict JSON contract. +#[pd_host_function(name = "runtime::input_json")] +fn runtime_input_json_impl(vm: &mut Vm) -> VmResult { + let value = vm.runtime_input_value()?; + super::json::encode_value_to_string(&value) +} + +/// Emits one bounded event without changing the script return value. +#[pd_host_function(name = "runtime::emit")] +fn runtime_emit_impl(vm: &mut Vm, value: AnyValue) -> VmResult<()> { + vm.emit_runtime_event(value) +} + +/// Emits one JSON text event for strict RSS boundary adapters. +#[pd_host_function(name = "runtime::emit_json")] +fn runtime_emit_json_impl(vm: &mut Vm, value: &str) -> VmResult<()> { + vm.emit_runtime_event(Value::string(value)) +} diff --git a/src/builtins/runtime/error.rs b/src/builtins/runtime/error.rs new file mode 100644 index 00000000..506ccf52 --- /dev/null +++ b/src/builtins/runtime/error.rs @@ -0,0 +1,165 @@ +use std::fmt; + +/// Result type used by the generic runtime support modules. +pub type RuntimeResult = Result; + +/// Stable machine-readable categories for runtime capability failures. +#[allow(dead_code)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RuntimeErrorCode { + InvalidConfiguration, + InputUnavailable, + EventSinkUnavailable, + EventPayloadTooLarge, + EventDepthExceeded, + EventSequenceExhausted, + EventSinkRejected, + ResourceLimitExceeded, + InvalidResourceHandle, + ResourceHandleWrongTable, + ResourceTypeMismatch, + ResourceStale, + ResourceAlreadyClosed, + ResourceIdExhausted, + ResourceCleanupFailed, + OperationLimitExceeded, + OperationNotFound, + OperationAlreadyFinished, + OperationCancelled, + OperationFailed, + OperationIdExhausted, + OperationCleanupFailed, +} + +impl RuntimeErrorCode { + pub const fn as_str(self) -> &'static str { + match self { + Self::InvalidConfiguration => "invalid_configuration", + Self::InputUnavailable => "input_unavailable", + Self::EventSinkUnavailable => "event_sink_unavailable", + Self::EventPayloadTooLarge => "event_payload_too_large", + Self::EventDepthExceeded => "event_depth_exceeded", + Self::EventSequenceExhausted => "event_sequence_exhausted", + Self::EventSinkRejected => "event_sink_rejected", + Self::ResourceLimitExceeded => "resource_limit_exceeded", + Self::InvalidResourceHandle => "invalid_resource_handle", + Self::ResourceHandleWrongTable => "resource_handle_wrong_table", + Self::ResourceTypeMismatch => "resource_type_mismatch", + Self::ResourceStale => "resource_stale", + Self::ResourceAlreadyClosed => "resource_already_closed", + Self::ResourceIdExhausted => "resource_id_exhausted", + Self::ResourceCleanupFailed => "resource_cleanup_failed", + Self::OperationLimitExceeded => "operation_limit_exceeded", + Self::OperationNotFound => "operation_not_found", + Self::OperationAlreadyFinished => "operation_already_finished", + Self::OperationCancelled => "operation_cancelled", + Self::OperationFailed => "operation_failed", + Self::OperationIdExhausted => "operation_id_exhausted", + Self::OperationCleanupFailed => "operation_cleanup_failed", + } + } +} + +/// Structured error returned by the generic runtime support modules. +/// +/// The core VM currently exposes `VmError::HostError` as the extension point for host failures. +/// Runtime code keeps the stable category and fields until the parent wiring maps it into that +/// existing VM error variant. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RuntimeError { + code: RuntimeErrorCode, + operation: &'static str, + message: String, + limit: Option, + value: Option, +} + +#[allow(dead_code)] +impl RuntimeError { + pub fn new( + code: RuntimeErrorCode, + operation: &'static str, + message: impl Into, + ) -> Self { + Self { + code, + operation, + message: message.into(), + limit: None, + value: None, + } + } + + pub fn code(&self) -> RuntimeErrorCode { + self.code + } + + pub fn operation(&self) -> &'static str { + self.operation + } + + pub fn message(&self) -> &str { + &self.message + } + + pub fn limit(&self) -> Option { + self.limit + } + + pub fn value(&self) -> Option { + self.value + } + + pub fn with_limit(mut self, limit: usize) -> Self { + self.limit = Some(limit); + self + } + + pub fn with_value(mut self, value: u64) -> Self { + self.value = Some(value); + self + } +} + +impl fmt::Display for RuntimeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "runtime error [{}] in {}: {}", + self.code.as_str(), + self.operation, + self.message + )?; + if let Some(limit) = self.limit { + write!(formatter, " (limit: {limit})")?; + } + if let Some(value) = self.value { + write!(formatter, " (value: {value})")?; + } + Ok(()) + } +} + +impl std::error::Error for RuntimeError {} + +#[cfg(test)] +mod tests { + use super::{RuntimeError, RuntimeErrorCode}; + + #[test] + fn structured_error_preserves_code_and_fields() { + let error = RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "event payload exceeds the configured bound", + ) + .with_limit(32) + .with_value(64); + + assert_eq!(error.code(), RuntimeErrorCode::EventPayloadTooLarge); + assert_eq!(error.operation(), "runtime::emit"); + assert_eq!(error.limit(), Some(32)); + assert_eq!(error.value(), Some(64)); + assert!(error.to_string().contains("event_payload_too_large")); + } +} diff --git a/src/builtins/runtime/event.rs b/src/builtins/runtime/event.rs new file mode 100644 index 00000000..cc4a3e94 --- /dev/null +++ b/src/builtins/runtime/event.rs @@ -0,0 +1,345 @@ +use crate::vm::Value; + +use super::error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; + +pub const DEFAULT_MAX_EVENT_PAYLOAD_BYTES: usize = 64 * 1024; +pub const DEFAULT_MAX_EVENT_DEPTH: usize = 64; +pub const DEFAULT_MAX_EVENTS: u64 = 1_024; +pub const DEFAULT_MAX_EVENT_BYTES: usize = 16 * 1024 * 1024; + +/// Bounds applied before an event is handed to an embedding-owned sink. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EventLimits { + max_payload_bytes: usize, + max_depth: usize, + max_events: u64, + max_total_bytes: usize, +} + +impl EventLimits { + pub fn new(max_payload_bytes: usize, max_depth: usize) -> RuntimeResult { + Self::with_budget( + max_payload_bytes, + max_depth, + DEFAULT_MAX_EVENTS, + DEFAULT_MAX_EVENT_BYTES, + ) + } + + pub fn with_budget( + max_payload_bytes: usize, + max_depth: usize, + max_events: u64, + max_total_bytes: usize, + ) -> RuntimeResult { + if max_payload_bytes == 0 || max_depth == 0 || max_events == 0 || max_total_bytes == 0 { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "runtime::emit", + "event payload and depth limits must be positive", + )); + } + Ok(Self { + max_payload_bytes, + max_depth, + max_events, + max_total_bytes, + }) + } + + pub const fn max_payload_bytes(self) -> usize { + self.max_payload_bytes + } + + pub const fn max_depth(self) -> usize { + self.max_depth + } + + pub const fn max_events(self) -> u64 { + self.max_events + } + + pub const fn max_total_bytes(self) -> usize { + self.max_total_bytes + } +} + +impl Default for EventLimits { + fn default() -> Self { + Self { + max_payload_bytes: DEFAULT_MAX_EVENT_PAYLOAD_BYTES, + max_depth: DEFAULT_MAX_EVENT_DEPTH, + max_events: DEFAULT_MAX_EVENTS, + max_total_bytes: DEFAULT_MAX_EVENT_BYTES, + } + } +} + +/// An event value whose size and nesting have already been checked. +#[derive(Clone, Debug, PartialEq)] +pub struct EventPayload { + value: Value, + size_bytes: usize, +} + +impl EventPayload { + pub fn try_new(value: Value, limits: EventLimits) -> RuntimeResult { + let size_bytes = estimate_value_size(&value, limits)?; + Ok(Self { value, size_bytes }) + } + + pub fn value(&self) -> &Value { + &self.value + } + + pub fn size_bytes(&self) -> usize { + self.size_bytes + } + + pub fn into_value(self) -> Value { + self.value + } +} + +/// Embedding-owned transport hook for bounded runtime events. +pub trait EventSink: Send { + fn emit(&mut self, payload: EventPayload) -> RuntimeResult<()>; +} + +impl EventSink for F +where + F: FnMut(EventPayload) -> RuntimeResult<()> + Send + 'static, +{ + fn emit(&mut self, payload: EventPayload) -> RuntimeResult<()> { + self(payload) + } +} + +/// Validates and forwards generic values without attaching agent or platform semantics. +pub struct EventEmitter { + limits: EventLimits, + sink: Option>, + emitted_events: u64, + emitted_bytes: usize, +} + +#[allow(dead_code)] +impl EventEmitter { + pub fn new(limits: EventLimits) -> Self { + Self { + limits, + sink: None, + emitted_events: 0, + emitted_bytes: 0, + } + } + + pub fn limits(&self) -> EventLimits { + self.limits + } + + pub fn set_sink(&mut self, sink: S) + where + S: EventSink + 'static, + { + self.sink = Some(Box::new(sink)); + } + + pub fn clear_sink(&mut self) { + self.sink = None; + } + + pub fn reset_for_reuse(&mut self) { + self.sink = None; + self.emitted_events = 0; + self.emitted_bytes = 0; + } + + pub fn emitted_events(&self) -> u64 { + self.emitted_events + } + + pub fn emit(&mut self, value: Value) -> RuntimeResult { + let payload = EventPayload::try_new(value, self.limits)?; + if self.emitted_events >= self.limits.max_events { + return Err(RuntimeError::new( + RuntimeErrorCode::EventSequenceExhausted, + "runtime::emit", + "event count exceeds the configured bound", + ) + .with_limit(self.limits.max_events.min(usize::MAX as u64) as usize) + .with_value(self.emitted_events)); + } + let total_bytes = self + .emitted_bytes + .checked_add(payload.size_bytes()) + .ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "cumulative event bytes overflowed", + ) + })?; + if total_bytes > self.limits.max_total_bytes { + return Err(RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "cumulative event bytes exceed the configured bound", + ) + .with_limit(self.limits.max_total_bytes) + .with_value(total_bytes as u64)); + } + let sequence = self.emitted_events.checked_add(1).ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::EventSequenceExhausted, + "runtime::emit", + "event sequence exhausted", + ) + })?; + let sink = self.sink.as_mut().ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::EventSinkUnavailable, + "runtime::emit", + "an event sink has not been configured", + ) + })?; + sink.emit(payload.clone()).map_err(|error| { + RuntimeError::new( + RuntimeErrorCode::EventSinkRejected, + "runtime::emit", + error.to_string(), + ) + })?; + self.emitted_events = sequence; + self.emitted_bytes = total_bytes; + Ok(EventReceipt { + sequence, + payload_bytes: payload.size_bytes(), + }) + } +} + +impl Default for EventEmitter { + fn default() -> Self { + Self::new(EventLimits::default()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EventReceipt { + sequence: u64, + payload_bytes: usize, +} + +#[allow(dead_code)] +impl EventReceipt { + pub const fn sequence(self) -> u64 { + self.sequence + } + + pub const fn payload_bytes(self) -> usize { + self.payload_bytes + } +} + +/// Estimates the bounded representation size used by [`EventPayload`]. +/// +/// The estimate is deliberately independent of serialization formats. It counts scalar tags, +/// container headers, string/byte contents, and recursively contained values. The host transport +/// can apply a stricter byte limit when it serializes the validated value. +pub fn estimate_value_size(value: &Value, limits: EventLimits) -> RuntimeResult { + measure_value(value, 0, limits) +} + +fn measure_value(value: &Value, depth: usize, limits: EventLimits) -> RuntimeResult { + if depth > limits.max_depth { + return Err(RuntimeError::new( + RuntimeErrorCode::EventDepthExceeded, + "runtime::emit", + "event payload nesting exceeds the configured bound", + ) + .with_limit(limits.max_depth) + .with_value(depth as u64)); + } + + let size = match value { + Value::Null | Value::Bool(_) => 1, + Value::Int(_) | Value::Float(_) => 9, + Value::String(text) => 1usize.saturating_add(text.len()), + Value::Bytes(bytes) => 1usize.saturating_add(bytes.len()), + Value::Callable(_) => 17, + Value::Array(values) => { + let mut size = 5usize; + for child in values.iter() { + size = checked_payload_add(size, measure_value(child, depth + 1, limits)?, limits)?; + } + size + } + Value::Map(entries) => { + let mut size = 5usize; + for (key, child) in entries.iter() { + size = checked_payload_add(size, measure_value(key, depth + 1, limits)?, limits)?; + size = checked_payload_add(size, measure_value(child, depth + 1, limits)?, limits)?; + } + size + } + }; + + if size > limits.max_payload_bytes { + return Err(RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "event payload exceeds the configured byte bound", + ) + .with_limit(limits.max_payload_bytes) + .with_value(size as u64)); + } + Ok(size) +} + +fn checked_payload_add( + current: usize, + additional: usize, + limits: EventLimits, +) -> RuntimeResult { + let total = current.checked_add(additional).ok_or_else(|| { + RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "event payload size overflowed", + ) + .with_limit(limits.max_payload_bytes) + })?; + if total > limits.max_payload_bytes { + return Err(RuntimeError::new( + RuntimeErrorCode::EventPayloadTooLarge, + "runtime::emit", + "event payload exceeds the configured byte bound", + ) + .with_limit(limits.max_payload_bytes) + .with_value(total as u64)); + } + Ok(total) +} + +#[cfg(test)] +mod tests { + use super::{EventEmitter, EventLimits, EventPayload}; + use crate::vm::Value; + + #[test] + fn payload_size_and_sequence_are_exposed_after_validation() { + let limits = EventLimits::new(128, 4).expect("limits should be valid"); + let payload = + EventPayload::try_new(Value::string("event"), limits).expect("payload should fit"); + assert!(payload.size_bytes() >= 5); + + let mut emitter = EventEmitter::new(limits); + emitter.set_sink(|_| Ok(())); + let receipt = emitter + .emit(Value::string("event")) + .expect("event should be emitted"); + assert_eq!(receipt.sequence(), 1); + assert_eq!(emitter.emitted_events(), 1); + } +} diff --git a/src/builtins/runtime/http.rs b/src/builtins/runtime/http.rs new file mode 100644 index 00000000..c39ef87f --- /dev/null +++ b/src/builtins/runtime/http.rs @@ -0,0 +1,969 @@ +use std::task::{Context, Poll}; + +#[cfg(feature = "http-client")] +use futures_util::StreamExt; +#[cfg(feature = "http-client")] +use futures_util::future::{AbortHandle, Abortable}; + +use pd_host_function::pd_host_function; + +use super::{HostCallResult, Vm, VmMap, VmResult}; +#[cfg(feature = "http-client")] +use crate::builtins::runtime::cancellation::{ + CancellationReason, CancellationToken, OperationId, OperationOwner, +}; +#[cfg(feature = "http-client")] +use crate::builtins::runtime::error::{RuntimeError, RuntimeErrorCode}; +#[cfg(feature = "http-client")] +use crate::builtins::runtime::resource::ResourceTypeId; +#[cfg(feature = "http-client")] +use crate::vm::Value; +use crate::vm::{CallReturn, HostOpId, VmError}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct HttpConfig { + pub allowed_schemes: Vec, + pub allowed_hosts: Vec, + pub allowed_ports: Vec, + pub max_redirects: usize, + pub max_request_body_bytes: usize, + pub max_response_body_bytes: usize, + pub connect_timeout: std::time::Duration, + pub request_timeout: std::time::Duration, + pub allow_private_ips: bool, +} + +impl Default for HttpConfig { + fn default() -> Self { + Self { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: Vec::new(), + allowed_ports: Vec::new(), + max_redirects: 5, + max_request_body_bytes: 1024 * 1024, + max_response_body_bytes: 8 * 1024 * 1024, + connect_timeout: std::time::Duration::from_secs(10), + request_timeout: std::time::Duration::from_secs(30), + allow_private_ips: false, + } + } +} + +#[cfg(feature = "http-client")] +struct HttpCompletion { + result: VmResult, +} + +#[cfg(feature = "http-client")] +struct HttpRequestResource { + receiver: futures_channel::oneshot::Receiver, +} + +pub(crate) struct HttpState { + #[cfg(feature = "http-client")] + config: Option, + pub(crate) max_in_flight: usize, +} + +impl Default for HttpState { + fn default() -> Self { + Self { + #[cfg(feature = "http-client")] + config: None, + max_in_flight: crate::builtins::runtime::cancellation::DEFAULT_MAX_PENDING_OPERATIONS, + } + } +} + +impl HttpState { + pub(crate) fn reset_for_reuse(&mut self) {} + + pub(crate) fn configure(&mut self, config: HttpConfig) { + #[cfg(feature = "http-client")] + { + self.config = Some(config); + } + #[cfg(not(feature = "http-client"))] + let _ = config; + } + + pub(crate) fn clear_configuration(&mut self) { + #[cfg(feature = "http-client")] + { + self.config = None; + } + } + + #[cfg(all(test, feature = "http-client"))] + pub(crate) fn configuration(&self) -> Option<&HttpConfig> { + self.config.as_ref() + } + + pub(crate) fn is_configured(&self) -> bool { + #[cfg(feature = "http-client")] + { + self.config.is_some() + } + #[cfg(not(feature = "http-client"))] + false + } +} + +#[cfg(feature = "http-client")] +fn schedule_request(vm: &mut Vm, config: HttpConfig, request: HttpRequest) -> VmResult { + let max_in_flight = vm.host.http_state.max_in_flight; + if vm + .host + .runtime_operations + .operations_by_owner(OperationOwner::Http) + .len() + >= max_in_flight + { + return Err(VmError::HostError(format!( + "HTTP in-flight request limit of {} has been reached", + max_in_flight + ))); + } + + let deadline = std::time::Instant::now() + config.request_timeout; + let (sender, receiver) = futures_channel::oneshot::channel(); + let (abort_handle, abort_registration) = AbortHandle::new_pair(); + let operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Http, + Some(&vm.run_ctx.cancellation), + Some(deadline), + Some(Box::new(move |_| { + abort_handle.abort(); + Ok(()) + })), + ) + .map_err(runtime_host_error)?; + let operation_id = operation.id(); + let op_id = operation_id.raw(); + let token = operation.token(); + let worker_operation = operation.clone(); + let resource = match vm.host.runtime_resources.insert( + ResourceTypeId::HTTP_REQUEST, + HttpRequestResource { receiver }, + ) { + Ok(resource) => resource, + Err(error) => { + let _ = vm + .host + .runtime_operations + .cancel(operation_id, CancellationReason::ResourceClosed); + return Err(runtime_host_error(error)); + } + }; + operation.set_payload(resource); + + let thread_name = format!("rustscript-http-{op_id}"); + if let Err(error) = std::thread::Builder::new() + .name(thread_name) + .spawn(move || { + let result = match tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime.block_on(async move { + match Abortable::new( + execute_request(&config, &request, &token, deadline), + abort_registration, + ) + .await + { + Ok(result) => result, + Err(_) => cancellation_error(&token), + } + }), + Err(error) => Err(VmError::HostError(format!( + "failed to create HTTP runtime: {error}" + ))), + }; + match &result { + Ok(_) => { + let _ = worker_operation.complete(); + } + Err(error) => { + let _ = worker_operation.fail( + RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "http::request", + error.to_string(), + ) + .with_value(op_id), + ); + } + } + let _ = sender.send(HttpCompletion { result }); + }) + { + super::cancel_runtime_operation(vm, operation_id, CancellationReason::ResourceClosed); + return Err(VmError::HostError(format!( + "failed to start HTTP request: {error}" + ))); + } + + Ok(op_id) +} + +#[cfg(feature = "http-client")] +fn close_request_resource(vm: &mut Vm, op_id: HostOpId, reason: CancellationReason) { + let Ok(operation_id) = OperationId::from_raw(op_id) else { + return; + }; + let Ok(operation) = vm.host.runtime_operations.get(operation_id) else { + return; + }; + let Some(resource) = operation.payload() else { + return; + }; + let _ = super::close_runtime_resource(vm, resource, reason); +} + +#[cfg(feature = "http-client")] +fn runtime_host_error(error: impl std::fmt::Display) -> VmError { + VmError::HostError(error.to_string()) +} + +#[cfg(feature = "http-client")] +fn cancellation_vm_error(token: &CancellationToken) -> VmError { + token + .check() + .map(|()| VmError::HostError("HTTP request was cancelled".to_string())) + .unwrap_or_else(runtime_host_error) +} + +#[cfg(feature = "http-client")] +fn cancellation_error(token: &CancellationToken) -> VmResult { + Err(cancellation_vm_error(token)) +} + +/// Starts an HTTP request under the VM's configured network policy. +/// +/// The request map accepts `method`, `url`, optional `headers`, and optional `body`. +/// The response map contains `status`, `headers`, `body`, and the final `url`. +#[pd_host_function(name = "http::client::request")] +pub(super) fn builtin_http_client_request( + vm: &mut Vm, + request: &VmMap, +) -> VmResult> { + #[cfg(not(feature = "http-client"))] + { + let _ = (vm, request); + Err(VmError::HostError( + "HTTP client support is disabled; enable the http-client feature".to_string(), + )) + } + + #[cfg(feature = "http-client")] + { + let config = vm + .host + .http_state + .config + .clone() + .ok_or_else(|| VmError::HostError("HTTP host is not configured".to_string()))?; + let request = parse_request(request, &config)?; + let op_id = schedule_request(vm, config, request)?; + Ok(HostCallResult::Pending(op_id)) + } +} + +pub(super) fn poll_pending_op( + vm: &mut Vm, + op_id: HostOpId, + cx: &mut Context<'_>, +) -> Poll> { + #[cfg(feature = "http-client")] + { + use std::pin::Pin; + + let operation_id = match OperationId::from_raw(op_id) { + Ok(operation_id) => operation_id, + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), + }; + let operation = match vm.host.runtime_operations.get(operation_id) { + Ok(operation) => operation, + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), + }; + let Some(resource) = operation.payload() else { + return Poll::Ready(Err(VmError::HostError(format!( + "HTTP op {op_id} has no completion payload", + )))); + }; + let poll_result = match vm + .host + .runtime_resources + .get_mut::(resource, ResourceTypeId::HTTP_REQUEST) + { + Ok(request) => Pin::new(&mut request.receiver).poll(cx), + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), + }; + match poll_result { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(completion)) => { + close_request_resource(vm, op_id, CancellationReason::ResourceClosed); + Poll::Ready(completion.result) + } + Poll::Ready(Err(_)) => { + close_request_resource(vm, op_id, CancellationReason::ResourceClosed); + Poll::Ready(Err(VmError::HostError(format!( + "HTTP op {op_id} was cancelled", + )))) + } + } + } + + #[cfg(not(feature = "http-client"))] + { + let _ = (vm, cx); + Poll::Ready(Err(VmError::HostError(format!( + "HTTP support is disabled for op {op_id}", + )))) + } +} + +#[cfg(feature = "http-client")] +struct HttpRequest { + method: reqwest::Method, + url: url::Url, + headers: Vec<(reqwest::header::HeaderName, reqwest::header::HeaderValue)>, + body: Option>, +} + +#[cfg(feature = "http-client")] +fn parse_request(map: &VmMap, config: &HttpConfig) -> VmResult { + let method = map_string(map, "method")?.to_ascii_uppercase(); + if !matches!( + method.as_str(), + "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" + ) { + return Err(VmError::HostError(format!( + "HTTP method '{method}' is not allowed" + ))); + } + let method = reqwest::Method::from_bytes(method.as_bytes()) + .map_err(|_| VmError::HostError("invalid HTTP method".to_string()))?; + let url = map_string(map, "url")? + .parse::() + .map_err(|error| VmError::HostError(format!("invalid HTTP URL: {error}")))?; + if !url.username().is_empty() || url.password().is_some() { + return Err(VmError::HostError( + "HTTP URL userinfo is not allowed".to_string(), + )); + } + let body = match map.get(&Value::string("body")) { + None | Some(Value::Null) => None, + Some(Value::Bytes(bytes)) => { + if bytes.len() > config.max_request_body_bytes { + return Err(VmError::HostError( + "HTTP request body exceeds limit".to_string(), + )); + } + Some(bytes.as_ref().clone()) + } + Some(Value::String(text)) => { + if text.len() > config.max_request_body_bytes { + return Err(VmError::HostError( + "HTTP request body exceeds limit".to_string(), + )); + } + Some(text.as_bytes().to_vec()) + } + Some(_) => return Err(VmError::TypeMismatch("HTTP request body")), + }; + + let mut headers = Vec::new(); + if let Some(Value::Map(header_map)) = map.get(&Value::string("headers")) { + for (key, value) in header_map.iter() { + let Value::String(key) = key else { + return Err(VmError::TypeMismatch("HTTP header name")); + }; + let Value::String(value) = value else { + return Err(VmError::TypeMismatch("HTTP header value")); + }; + if matches!( + key.to_ascii_lowercase().as_str(), + "host" | "content-length" | "transfer-encoding" | "connection" + ) { + return Err(VmError::HostError(format!( + "HTTP header '{key}' is managed by the client", + ))); + } + let name = reqwest::header::HeaderName::from_bytes(key.as_bytes()) + .map_err(|_| VmError::HostError(format!("invalid HTTP header name '{key}'")))?; + let value = reqwest::header::HeaderValue::from_str(value).map_err(|_| { + VmError::HostError(format!("invalid HTTP header value for '{key}'")) + })?; + headers.push((name, value)); + } + } else if map.get(&Value::string("headers")).is_some() { + return Err(VmError::TypeMismatch("HTTP headers")); + } + + Ok(HttpRequest { + method, + url, + headers, + body, + }) +} + +#[cfg(feature = "http-client")] +fn map_string(map: &VmMap, key: &str) -> VmResult { + match map.get(&Value::string(key)) { + Some(Value::String(value)) => Ok(value.as_ref().clone()), + Some(_) => Err(VmError::TypeMismatch("HTTP request string field")), + None => Err(VmError::HostError(format!( + "missing HTTP request field '{key}'" + ))), + } +} + +#[cfg(feature = "http-client")] +fn validate_url_policy<'a>(config: &HttpConfig, url: &'a url::Url) -> VmResult<(&'a str, u16)> { + let scheme = url.scheme().to_ascii_lowercase(); + if !config + .allowed_schemes + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(&scheme)) + { + return Err(VmError::HostError(format!( + "HTTP URL scheme '{scheme}' is not allowed", + ))); + } + let host = url + .host_str() + .ok_or_else(|| VmError::HostError("HTTP URL has no host".to_string()))?; + if !config + .allowed_hosts + .iter() + .any(|allowed| allowed.eq_ignore_ascii_case(host)) + { + return Err(VmError::HostError( + "HTTP target host is not allowed".to_string(), + )); + } + let port = url + .port_or_known_default() + .ok_or_else(|| VmError::HostError("HTTP URL has no known port".to_string()))?; + if !config.allowed_ports.contains(&port) { + return Err(VmError::HostError(format!( + "HTTP target port {port} is not allowed", + ))); + } + Ok((host, port)) +} + +#[cfg(all(feature = "http-client", test))] +fn validate_url(config: &HttpConfig, url: &url::Url) -> VmResult> { + let (host, port) = validate_url_policy(config, url)?; + if config.allow_private_ips { + return Ok(None); + } + + if let Ok(host_ip) = host.parse::() { + validate_resolved_addresses(config, &[std::net::SocketAddr::new(host_ip, port)])?; + return Ok(None); + } + + use std::net::ToSocketAddrs; + let addresses = (host, port) + .to_socket_addrs() + .map_err(|error| VmError::HostError(format!("HTTP host resolution failed: {error}")))? + .collect::>(); + validate_resolved_addresses(config, &addresses)?; + Ok(addresses.first().copied()) +} + +#[cfg(feature = "http-client")] +async fn resolve_url( + config: &HttpConfig, + url: &url::Url, + token: &CancellationToken, + deadline: std::time::Instant, +) -> VmResult> { + token.check().map_err(runtime_host_error)?; + let (host, port) = validate_url_policy(config, url)?; + if let Ok(host_ip) = host.parse::() { + let address = std::net::SocketAddr::new(host_ip, port); + validate_resolved_addresses(config, &[address])?; + return Ok(Some(address)); + } + + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + token.cancel(CancellationReason::Deadline); + return Err(cancellation_vm_error(token)); + } + let addresses = tokio::time::timeout(remaining, tokio::net::lookup_host((host, port))) + .await + .map_err(|_| { + token.cancel(CancellationReason::Deadline); + cancellation_vm_error(token) + })? + .map_err(|error| VmError::HostError(format!("HTTP host resolution failed: {error}")))? + .collect::>(); + token.check().map_err(runtime_host_error)?; + validate_resolved_addresses(config, &addresses)?; + addresses + .first() + .copied() + .map(Some) + .ok_or_else(|| VmError::HostError("HTTP target resolves to a restricted IP".to_string())) +} + +#[cfg(feature = "http-client")] +fn validate_resolved_addresses( + config: &HttpConfig, + addresses: &[std::net::SocketAddr], +) -> VmResult<()> { + if addresses.is_empty() + || (!config.allow_private_ips + && addresses + .iter() + .any(|address| is_restricted_ip(address.ip()))) + { + return Err(VmError::HostError( + "HTTP target resolves to a restricted IP".to_string(), + )); + } + Ok(()) +} + +#[cfg(feature = "http-client")] +fn is_restricted_ip(ip: std::net::IpAddr) -> bool { + match ip { + std::net::IpAddr::V4(ip) => { + let octets = ip.octets(); + matches!(octets[0], 0 | 10 | 127) + || (octets[0] == 100 && (64..=127).contains(&octets[1])) + || (octets[0] == 169 && octets[1] == 254) + || (octets[0] == 172 && (16..=31).contains(&octets[1])) + || (octets[0] == 192 + && matches!( + (octets[1], octets[2]), + (0, 0) | (0, 2) | (31, 196) | (52, 193) | (88, 99) | (168, _) | (175, 48) + )) + || (octets[0] == 198 + && ((18..=19).contains(&octets[1]) || (octets[1] == 51 && octets[2] == 100))) + || (octets[0] == 203 && octets[1] == 0 && octets[2] == 113) + || octets[0] >= 224 + } + std::net::IpAddr::V6(ip) => { + if let Some(mapped) = ip.to_ipv4_mapped() { + return is_restricted_ip(std::net::IpAddr::V4(mapped)); + } + let segments = ip.segments(); + let outside_global_unicast = segments[0] & 0xe000 != 0x2000; + let protocol_assignments = segments[0] == 0x2001 && segments[1] <= 0x01ff; + let documentation = (segments[0] == 0x2001 && segments[1] == 0x0db8) + || (segments[0] == 0x3fff && segments[1] & 0xf000 == 0); + let six_to_four = segments[0] == 0x2002; + let direct_delegation_as112 = + segments[0] == 0x2620 && segments[1] == 0x004f && segments[2] == 0x8000; + outside_global_unicast + || protocol_assignments + || documentation + || six_to_four + || direct_delegation_as112 + } + } +} + +#[cfg(feature = "http-client")] +async fn execute_request( + config: &HttpConfig, + request: &HttpRequest, + token: &CancellationToken, + deadline: std::time::Instant, +) -> VmResult { + token.check().map_err(runtime_host_error)?; + let mut method = request.method.clone(); + let mut url = request.url.clone(); + let mut body = request.body.clone(); + let mut headers = request.headers.clone(); + + for redirect_index in 0..=config.max_redirects { + token.check().map_err(runtime_host_error)?; + let resolved_address = resolve_url(config, &url, token, deadline).await?; + let host = url + .host_str() + .ok_or_else(|| VmError::HostError("HTTP URL has no host".to_string()))?; + let mut client_builder = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .connect_timeout(config.connect_timeout); + if let Some(address) = resolved_address { + client_builder = client_builder.resolve(host, address); + } + let client = client_builder + .build() + .map_err(|error| VmError::HostError(format!("HTTP client setup failed: {error}")))?; + let origin = request.url.origin(); + let mut builder = client.request(method.clone(), url.clone()); + for (name, value) in &headers { + builder = builder.header(name, value); + } + if let Some(body) = &body { + builder = builder.body(body.clone()); + } + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + token.cancel(CancellationReason::Deadline); + return Err(cancellation_vm_error(token)); + } + let response = tokio::time::timeout(remaining, builder.send()) + .await + .map_err(|_| { + token.cancel(CancellationReason::Deadline); + cancellation_vm_error(token) + })? + .map_err(|error| { + if error.is_timeout() { + token.cancel(CancellationReason::Deadline); + cancellation_vm_error(token) + } else { + VmError::HostError(format!("HTTP request failed: {error}")) + } + })?; + token.check().map_err(runtime_host_error)?; + if response.status().is_redirection() { + if redirect_index == config.max_redirects { + return Err(VmError::HostError( + "HTTP redirect limit exceeded".to_string(), + )); + } + let location = response + .headers() + .get(reqwest::header::LOCATION) + .ok_or_else(|| VmError::HostError("HTTP redirect has no location".to_string()))? + .to_str() + .map_err(|_| VmError::HostError("HTTP redirect location is invalid".to_string()))? + .to_string(); + let next_url = url + .join(&location) + .map_err(|error| VmError::HostError(format!("invalid HTTP redirect: {error}")))?; + if next_url.origin() != origin { + headers.retain(|(name, _)| { + name != reqwest::header::AUTHORIZATION && name != reqwest::header::COOKIE + }); + } + if response.status() == reqwest::StatusCode::SEE_OTHER + || ((response.status() == reqwest::StatusCode::MOVED_PERMANENTLY + || response.status() == reqwest::StatusCode::FOUND) + && method != reqwest::Method::GET + && method != reqwest::Method::HEAD) + { + method = reqwest::Method::GET; + body = None; + } + url = next_url; + continue; + } + + let response_headers = response + .headers() + .iter() + .map(|(name, value)| { + let value = value + .to_str() + .map(Value::string) + .unwrap_or_else(|_| Value::bytes(value.as_bytes().to_vec())); + (Value::string(name.as_str()), value) + }) + .collect::>(); + let status = response.status(); + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = { + let remaining = deadline.saturating_duration_since(std::time::Instant::now()); + if remaining.is_zero() { + token.cancel(CancellationReason::Deadline); + return Err(cancellation_vm_error(token)); + } + tokio::time::timeout(remaining, stream.next()) + .await + .map_err(|_| { + token.cancel(CancellationReason::Deadline); + cancellation_vm_error(token) + })? + } { + token.check().map_err(runtime_host_error)?; + let chunk = chunk.map_err(|error| { + VmError::HostError(format!("HTTP response read failed: {error}")) + })?; + if bytes.len().saturating_add(chunk.len()) > config.max_response_body_bytes { + return Err(VmError::HostError( + "HTTP response body exceeds limit".to_string(), + )); + } + bytes.extend_from_slice(&chunk); + } + let response_map = VmMap::from_entries(vec![ + ( + Value::string("status"), + Value::Int(i64::from(status.as_u16())), + ), + ( + Value::string("headers"), + Value::Map(std::sync::Arc::new(VmMap::from_entries(response_headers))), + ), + (Value::string("body"), Value::bytes(bytes)), + (Value::string("url"), Value::string(url.as_str())), + ]); + return Ok(CallReturn::one(Value::Map(std::sync::Arc::new( + response_map, + )))); + } + + Err(VmError::HostError( + "HTTP redirect processing failed".to_string(), + )) +} + +#[cfg(test)] +mod tests { + use super::HttpConfig; + #[cfg(feature = "http-client")] + use super::{ + CancellationReason, HttpRequest, HttpRequestResource, OperationOwner, ResourceTypeId, + execute_request, is_restricted_ip, schedule_request, validate_resolved_addresses, + validate_url, + }; + #[cfg(feature = "http-client")] + use crate::builtins::runtime::cancellation::OperationId; + + #[test] + fn default_http_policy_denies_all_hosts() { + let config = HttpConfig::default(); + assert_eq!(config.allowed_schemes, ["https"]); + assert!(config.allowed_hosts.is_empty()); + assert!(config.allowed_ports.is_empty()); + assert!(!config.allow_private_ips); + } + + #[cfg(feature = "http-client")] + #[test] + fn request_uses_shared_operation_and_resource_lifecycle() { + let mut vm = crate::vm::Vm::new(crate::vm::Program::new(Vec::new(), Vec::new())); + vm.set_http_max_in_flight(1); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![1], + allow_private_ips: true, + ..HttpConfig::default() + }; + let request = HttpRequest { + method: reqwest::Method::GET, + url: "http://127.0.0.1:1/".parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + + let op_id = schedule_request(&mut vm, config, request).expect("request should schedule"); + let operation_id = OperationId::from_raw(op_id).expect("operation id should be valid"); + assert_eq!( + vm.host + .runtime_operations + .get(operation_id) + .expect("operation should be registered") + .owner(), + OperationOwner::Http + ); + let operation = vm + .host + .runtime_operations + .get(operation_id) + .expect("request should remain registered"); + let resource = operation + .payload() + .expect("operation should reference the request resource"); + assert_eq!(resource.resource_type(), ResourceTypeId::HTTP_REQUEST); + assert!( + vm.host + .runtime_resources + .get::(resource, ResourceTypeId::HTTP_REQUEST) + .is_ok() + ); + + let token = operation.token(); + vm.clear_http_configuration(); + assert_eq!(token.reason(), Some(CancellationReason::Requested)); + assert!( + vm.host + .runtime_resources + .get::(resource, ResourceTypeId::HTTP_REQUEST) + .is_err() + ); + assert!(vm.host.runtime_operations.get(operation_id).is_err()); + } + + #[cfg(feature = "http-client")] + #[test] + fn production_request_timeout_sets_structured_deadline_reason() { + use std::time::{Duration, Instant}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (_socket, _) = listener.accept().expect("request should connect"); + std::thread::sleep(Duration::from_millis(100)); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_millis(50), + request_timeout: Duration::from_millis(20), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: reqwest::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let token = crate::builtins::runtime::cancellation::CancellationToken::root(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + runtime + .block_on(execute_request( + &config, + &request, + &token, + Instant::now() + config.request_timeout, + )) + .expect_err("hanging server should time out"); + assert_eq!(token.reason(), Some(CancellationReason::Deadline)); + server.join().expect("server should exit"); + } + + #[cfg(feature = "http-client")] + #[test] + fn response_body_timeout_sets_structured_deadline_reason() { + use std::io::{Read, Write}; + use std::time::{Duration, Instant}; + + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("listener should bind"); + let address = listener.local_addr().expect("listener should have address"); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().expect("request should connect"); + let mut request = [0u8; 1024]; + let _ = socket + .read(&mut request) + .expect("request should be readable"); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\n") + .expect("headers should be written"); + socket.flush().expect("headers should flush"); + std::thread::sleep(Duration::from_millis(100)); + }); + let config = HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![address.port()], + allow_private_ips: true, + connect_timeout: Duration::from_millis(50), + request_timeout: Duration::from_millis(20), + ..HttpConfig::default() + }; + let request = HttpRequest { + method: reqwest::Method::GET, + url: format!("http://{address}/").parse().expect("valid URL"), + headers: Vec::new(), + body: None, + }; + let token = crate::builtins::runtime::cancellation::CancellationToken::root(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("runtime should build"); + + runtime + .block_on(execute_request( + &config, + &request, + &token, + Instant::now() + config.request_timeout, + )) + .expect_err("stalled response body should time out"); + assert_eq!(token.reason(), Some(CancellationReason::Deadline)); + server.join().expect("server should exit"); + } + + #[cfg(feature = "http-client")] + #[test] + fn empty_port_allowlist_rejects_explicit_and_default_ports() { + let config = HttpConfig { + allowed_schemes: vec!["https".to_string()], + allowed_hosts: vec!["example.com".to_string()], + ..HttpConfig::default() + }; + let explicit = "https://example.com:443/".parse().expect("valid URL"); + let default_port = "https://example.com/".parse().expect("valid URL"); + assert!(validate_url(&config, &explicit).is_err()); + assert!(validate_url(&config, &default_port).is_err()); + } + + #[cfg(feature = "http-client")] + #[test] + fn special_use_networks_and_mixed_dns_answers_are_restricted() { + for address in [ + "0.1.2.3", + "100.64.0.1", + "192.0.0.8", + "192.0.2.1", + "192.31.196.1", + "192.52.193.1", + "192.88.99.1", + "192.175.48.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "240.0.0.1", + "100::1", + "2001::1", + "2001:db8::1", + "2002::1", + "2620:4f:8000::1", + "3fff::1", + "fc00::1", + ] { + assert!( + is_restricted_ip(address.parse().expect("valid IP")), + "{address} must be restricted" + ); + } + for address in ["8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"] { + assert!( + !is_restricted_ip(address.parse().expect("valid IP")), + "{address} must remain globally routable" + ); + } + + let config = HttpConfig::default(); + let addresses = [ + "8.8.8.8:443".parse().expect("valid socket address"), + "100.64.0.1:443".parse().expect("valid socket address"), + ]; + assert!(validate_resolved_addresses(&config, &addresses).is_err()); + } + + #[cfg(feature = "http-client")] + #[test] + fn ipv4_mapped_ipv6_loopback_is_restricted() { + assert!(is_restricted_ip( + "::ffff:127.0.0.1".parse().expect("valid IP") + )); + } +} diff --git a/src/builtins/runtime/io.rs b/src/builtins/runtime/io.rs index b9ef5340..ee9e56cd 100644 --- a/src/builtins/runtime/io.rs +++ b/src/builtins/runtime/io.rs @@ -1,46 +1,137 @@ -use std::collections::HashMap; use std::fs::OpenOptions; use std::future::Future; use std::io::{Read, Write}; use std::pin::Pin; use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex, TryLockError}; use std::task::{Context, Poll}; +use std::time::{Duration, Instant}; + +#[cfg(unix)] +use std::os::unix::process::CommandExt; use futures_channel::oneshot; use pd_host_function::pd_host_function; use super::HostCallResult; +use super::cancellation::{CancellationReason, OperationId, OperationOwner}; +use super::error::{RuntimeError, RuntimeErrorCode}; +use super::resource::{ResourceHandle, ResourceTypeId}; use crate::vm::{CallReturn, HostOpId, Value, Vm, VmError, VmResult}; -pub(crate) struct IoState { - pub(super) next_handle: i64, - pub(super) handles: HashMap, - pending_ops: HashMap>, +pub(super) enum IoHandle { + File(std::fs::File), + PopenRead { child: Child }, + PopenWrite { child: Child }, +} + +struct IoResource { + handle: Mutex>, + process_id: AtomicU32, } -impl Default for IoState { - fn default() -> Self { +impl IoResource { + fn new(handle: IoHandle) -> Self { + let process_id = match &handle { + IoHandle::PopenRead { child } | IoHandle::PopenWrite { child } => Some(child.id()), + IoHandle::File(_) => None, + }; Self { - next_handle: 1, - handles: HashMap::new(), - pending_ops: HashMap::new(), + handle: Mutex::new(Some(handle)), + process_id: AtomicU32::new(process_id.unwrap_or(0)), + } + } + + fn with_handle_mut(&self, apply: impl FnOnce(&mut IoHandle) -> VmResult) -> VmResult { + let mut handle = self + .handle + .lock() + .map_err(|_| VmError::HostError("io resource lock was poisoned".to_string()))?; + let handle = handle + .as_mut() + .ok_or_else(|| VmError::HostError("io resource is already closing".to_string()))?; + apply(handle) + } + + fn take_handle(&self) -> VmResult { + self.handle + .lock() + .map_err(|_| VmError::HostError("io resource lock was poisoned".to_string()))? + .take() + .ok_or_else(|| VmError::HostError("io resource is already closing".to_string())) + } + + fn close(&self, reason: CancellationReason) -> VmResult<()> { + let process_id = self.process_id.swap(0, Ordering::AcqRel); + let termination_error = if reason != CancellationReason::ResourceClosed && process_id != 0 { + terminate_process_tree(process_id).err() + } else { + None + }; + + let deadline = Instant::now() + Duration::from_millis(500); + loop { + match self.handle.try_lock() { + Ok(mut handle) => { + let close_result = match handle.take() { + Some(handle) => close_io_handle(handle, reason), + None => Ok(()), + }; + return match close_result { + Err(error) => Err(error), + Ok(()) => termination_error.map_or(Ok(()), Err), + }; + } + Err(TryLockError::Poisoned(_)) => { + return Err(VmError::HostError( + "io resource lock was poisoned".to_string(), + )); + } + Err(TryLockError::WouldBlock) if Instant::now() >= deadline => { + let termination_detail = termination_error + .as_ref() + .map(|error| format!("; process termination failed: {error}")) + .unwrap_or_default(); + return Err(VmError::HostError(format!( + "timed out interrupting pending io operation{termination_detail}" + ))); + } + Err(TryLockError::WouldBlock) => std::thread::sleep(Duration::from_millis(5)), + } } } } -pub(super) enum IoHandle { - File(std::fs::File), - PopenRead { child: Child }, - PopenWrite { child: Child }, +impl Drop for IoResource { + fn drop(&mut self) { + let _ = self.close(CancellationReason::VmReset); + } } struct IoAsyncCompletion { - restored_handle: Option<(i64, IoHandle)>, + opened_handle: Option, + closed_handle: Option, result: VmResult, } -pub(super) fn cancel_pending_op(vm: &mut Vm, op_id: HostOpId) { - vm.io_state.pending_ops.remove(&op_id); +impl IoAsyncCompletion { + fn result(result: VmResult) -> Self { + Self { + opened_handle: None, + closed_handle: None, + result, + } + } +} + +impl Drop for IoAsyncCompletion { + fn drop(&mut self) { + let Some(handle) = self.opened_handle.take() else { + return; + }; + let _ = IoResource::new(handle).close(CancellationReason::VmReset); + } } pub(super) fn poll_builtin_io_op( @@ -48,29 +139,57 @@ pub(super) fn poll_builtin_io_op( op_id: HostOpId, cx: &mut Context<'_>, ) -> Poll> { + let operation_id = match OperationId::from_raw(op_id) { + Ok(operation_id) => operation_id, + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), + }; + let operation = match vm.host.runtime_operations.get(operation_id) { + Ok(operation) => operation, + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), + }; + let Some(callback) = operation.payload() else { + return Poll::Ready(Err(VmError::HostError(format!( + "builtin io op {op_id} has no completion payload", + )))); + }; let poll_result = { - let receiver = match vm.io_state.pending_ops.get_mut(&op_id) { - Some(receiver) => receiver, - None => { - return Poll::Ready(Err(VmError::HostError(format!( - "unknown builtin io op {op_id}", - )))); - } + let receiver = match vm + .host + .runtime_resources + .get_mut::>(callback, ResourceTypeId::CALLBACK) + { + Ok(receiver) => receiver, + Err(error) => return Poll::Ready(Err(runtime_host_error(error))), }; Pin::new(receiver).poll(cx) }; match poll_result { Poll::Pending => Poll::Pending, - Poll::Ready(Ok(completion)) => { - vm.io_state.pending_ops.remove(&op_id); - if let Some((handle_id, handle)) = completion.restored_handle { - vm.io_state.handles.insert(handle_id, handle); + Poll::Ready(Ok(mut completion)) => { + let _ = super::close_runtime_resource(vm, callback, CancellationReason::ResourceClosed); + + if let Some(closed_handle) = completion.closed_handle + && let Err(error) = super::close_runtime_resource( + vm, + closed_handle, + CancellationReason::ResourceClosed, + ) + { + completion.result = Err(runtime_host_error(error)); } - Poll::Ready(completion.result) + if let Some(opened_handle) = completion.opened_handle.take() { + let result = insert_io_resource(vm, opened_handle) + .map(|handle| CallReturn::one(handle.as_value())); + completion.result = result; + } + Poll::Ready(std::mem::replace( + &mut completion.result, + Ok(CallReturn::none()), + )) } Poll::Ready(Err(_)) => { - vm.io_state.pending_ops.remove(&op_id); + let _ = super::close_runtime_resource(vm, callback, CancellationReason::Requested); Poll::Ready(Err(VmError::HostError(format!( "builtin io op {op_id} was cancelled", )))) @@ -78,13 +197,6 @@ pub(super) fn poll_builtin_io_op( } } -pub(super) fn close_all_handles(vm: &mut Vm) { - let handles = std::mem::take(&mut vm.io_state.handles); - for (_, handle) in handles { - let _ = close_io_handle(handle); - } -} - /// Opens a file handle for runtime I/O. #[pd_host_function(name = "io::open")] pub(super) fn builtin_io_open( @@ -92,10 +204,9 @@ pub(super) fn builtin_io_open( path: &str, mode: &str, ) -> VmResult> { - let reserved_id = io_reserve_handle_id(vm); let path = path.to_string(); let mode = mode.to_string(); - let op_id = schedule_io_task(vm, move || { + let op_id = schedule_io_task(vm, None, move || { let mut options = OpenOptions::new(); match mode.as_str() { "r" => { @@ -117,24 +228,21 @@ pub(super) fn builtin_io_open( options.read(true).write(true).create(true).append(true); } other => { - return IoAsyncCompletion { - restored_handle: None, - result: Err(VmError::HostError(format!( - "unsupported io_open mode '{other}', expected r/w/a/r+/w+/a+", - ))), - }; + return IoAsyncCompletion::result(Err(VmError::HostError(format!( + "unsupported io_open mode '{other}', expected r/w/a/r+/w+/a+", + )))); } } match options.open(path) { Ok(file) => IoAsyncCompletion { - restored_handle: Some((reserved_id, IoHandle::File(file))), - result: Ok(CallReturn::one(Value::Int(reserved_id))), - }, - Err(err) => IoAsyncCompletion { - restored_handle: None, - result: Err(VmError::HostError(format!("io_open failed: {err}"))), + opened_handle: Some(IoHandle::File(file)), + closed_handle: None, + result: Ok(CallReturn::none()), }, + Err(err) => { + IoAsyncCompletion::result(Err(VmError::HostError(format!("io_open failed: {err}")))) + } } })?; Ok(HostCallResult::Pending(op_id)) @@ -152,47 +260,36 @@ pub(super) fn builtin_io_popen( "unsupported io_popen mode '{mode}', expected r or w" ))); } - let reserved_id = io_reserve_handle_id(vm); let command = command.to_string(); let mode = mode.to_string(); - let op_id = schedule_io_task(vm, move || { + let op_id = schedule_io_task(vm, None, move || { let child = match spawn_shell_command(command.as_str(), mode.as_str()) { Ok(child) => child, - Err(err) => { - return IoAsyncCompletion { - restored_handle: None, - result: Err(err), - }; - } + Err(err) => return IoAsyncCompletion::result(Err(err)), }; let handle = match mode.as_str() { "r" => { if child.stdout.is_none() { - return IoAsyncCompletion { - restored_handle: None, - result: Err(VmError::HostError( - "io_popen('r') did not provide stdout pipe".to_string(), - )), - }; + return IoAsyncCompletion::result(Err(VmError::HostError( + "io_popen('r') did not provide stdout pipe".to_string(), + ))); } IoHandle::PopenRead { child } } "w" => { if child.stdin.is_none() { - return IoAsyncCompletion { - restored_handle: None, - result: Err(VmError::HostError( - "io_popen('w') did not provide stdin pipe".to_string(), - )), - }; + return IoAsyncCompletion::result(Err(VmError::HostError( + "io_popen('w') did not provide stdin pipe".to_string(), + ))); } IoHandle::PopenWrite { child } } _ => unreachable!("mode validated above"), }; IoAsyncCompletion { - restored_handle: Some((reserved_id, handle)), - result: Ok(CallReturn::one(Value::Int(reserved_id))), + opened_handle: Some(handle), + closed_handle: None, + result: Ok(CallReturn::none()), } })?; Ok(HostCallResult::Pending(op_id)) @@ -201,40 +298,32 @@ pub(super) fn builtin_io_popen( /// Reads all remaining text from an I/O handle. #[pd_host_function(name = "io::read_all")] pub(super) fn builtin_io_read_all(vm: &mut Vm, handle_id: i64) -> VmResult> { - let handle = io_take_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, move || { - let mut handle = handle; - let mut out = String::new(); - let result = match &mut handle { - IoHandle::File(file) => file - .read_to_string(&mut out) - .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}"))) - .map(|_| CallReturn::one(Value::string(out))), - IoHandle::PopenRead { child } => { - let stdout = match child.stdout.as_mut() { - Some(stdout) => stdout, - None => { - return IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result: Err(VmError::HostError( - "io_read_all popen handle missing stdout".to_string(), - )), - }; - } - }; - stdout + let handle = resource_handle(handle_id)?; + let resource = io_resource_for_handle(vm, handle)?; + let op_id = schedule_io_task(vm, Some(handle), move || { + let result = resource.with_handle_mut(|handle| { + let mut out = String::new(); + match handle { + IoHandle::File(file) => file .read_to_string(&mut out) - .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}"))) - .map(|_| CallReturn::one(Value::string(out))) - } - IoHandle::PopenWrite { .. } => Err(VmError::HostError( - "io_read_all requires a readable handle".to_string(), - )), - }; - IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result, - } + .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}")))?, + IoHandle::PopenRead { child } => child + .stdout + .as_mut() + .ok_or_else(|| { + VmError::HostError("io_read_all popen handle missing stdout".to_string()) + })? + .read_to_string(&mut out) + .map_err(|err| VmError::HostError(format!("io_read_all failed: {err}")))?, + IoHandle::PopenWrite { .. } => { + return Err(VmError::HostError( + "io_read_all requires a readable handle".to_string(), + )); + } + }; + Ok(CallReturn::one(Value::string(out))) + }); + IoAsyncCompletion::result(result) })?; Ok(HostCallResult::Pending(op_id)) } @@ -245,35 +334,26 @@ pub(super) fn builtin_io_read_line( vm: &mut Vm, handle_id: i64, ) -> VmResult> { - let handle = io_take_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, move || { - let mut handle = handle; - let result = match &mut handle { - IoHandle::File(file) => { - read_line_from_reader(file).map(|line| CallReturn::one(Value::string(line))) - } - IoHandle::PopenRead { child } => { - let stdout = match child.stdout.as_mut() { - Some(stdout) => stdout, - None => { - return IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result: Err(VmError::HostError( - "io_read_line popen handle missing stdout".to_string(), - )), - }; - } - }; - read_line_from_reader(stdout).map(|line| CallReturn::one(Value::string(line))) - } - IoHandle::PopenWrite { .. } => Err(VmError::HostError( - "io_read_line requires a readable handle".to_string(), - )), - }; - IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result, - } + let handle = resource_handle(handle_id)?; + let resource = io_resource_for_handle(vm, handle)?; + let op_id = schedule_io_task(vm, Some(handle), move || { + let result = resource.with_handle_mut(|handle| { + let line = match handle { + IoHandle::File(file) => read_line_from_reader(file)?, + IoHandle::PopenRead { child } => { + read_line_from_reader(child.stdout.as_mut().ok_or_else(|| { + VmError::HostError("io_read_line popen handle missing stdout".to_string()) + })?)? + } + IoHandle::PopenWrite { .. } => { + return Err(VmError::HostError( + "io_read_line requires a readable handle".to_string(), + )); + } + }; + Ok(CallReturn::one(Value::string(line))) + }); + IoAsyncCompletion::result(result) })?; Ok(HostCallResult::Pending(op_id)) } @@ -286,39 +366,31 @@ pub(super) fn builtin_io_write( text: &str, ) -> VmResult> { let bytes = text.as_bytes().to_vec(); - let handle = io_take_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, move || { - let mut handle = handle; - let result = match &mut handle { - IoHandle::File(file) => file - .write(&bytes) - .map_err(|err| VmError::HostError(format!("io_write failed: {err}"))) - .map(|written| CallReturn::one(Value::Int(written as i64))), - IoHandle::PopenWrite { child } => { - let stdin = match child.stdin.as_mut() { - Some(stdin) => stdin, - None => { - return IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result: Err(VmError::HostError( - "io_write popen handle missing stdin".to_string(), - )), - }; - } - }; - stdin + let handle = resource_handle(handle_id)?; + let resource = io_resource_for_handle(vm, handle)?; + let op_id = schedule_io_task(vm, Some(handle), move || { + let result = resource.with_handle_mut(|handle| { + let written = match handle { + IoHandle::File(file) => file .write(&bytes) - .map_err(|err| VmError::HostError(format!("io_write failed: {err}"))) - .map(|written| CallReturn::one(Value::Int(written as i64))) - } - IoHandle::PopenRead { .. } => Err(VmError::HostError( - "io_write requires a writable handle".to_string(), - )), - }; - IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result, - } + .map_err(|err| VmError::HostError(format!("io_write failed: {err}")))?, + IoHandle::PopenWrite { child } => child + .stdin + .as_mut() + .ok_or_else(|| { + VmError::HostError("io_write popen handle missing stdin".to_string()) + })? + .write(&bytes) + .map_err(|err| VmError::HostError(format!("io_write failed: {err}")))?, + IoHandle::PopenRead { .. } => { + return Err(VmError::HostError( + "io_write requires a writable handle".to_string(), + )); + } + }; + Ok(CallReturn::one(Value::Int(written as i64))) + }); + IoAsyncCompletion::result(result) })?; Ok(HostCallResult::Pending(op_id)) } @@ -326,37 +398,27 @@ pub(super) fn builtin_io_write( /// Flushes buffered output for an I/O handle. #[pd_host_function(name = "io::flush")] pub(super) fn builtin_io_flush(vm: &mut Vm, handle_id: i64) -> VmResult> { - let handle = io_take_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, move || { - let mut handle = handle; - let result = match &mut handle { - IoHandle::File(file) => file - .flush() - .map_err(|err| VmError::HostError(format!("io_flush failed: {err}"))) - .map(|_| CallReturn::one(Value::Bool(true))), - IoHandle::PopenWrite { child } => { - let stdin = match child.stdin.as_mut() { - Some(stdin) => stdin, - None => { - return IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result: Err(VmError::HostError( - "io_flush popen handle missing stdin".to_string(), - )), - }; - } - }; - stdin + let handle = resource_handle(handle_id)?; + let resource = io_resource_for_handle(vm, handle)?; + let op_id = schedule_io_task(vm, Some(handle), move || { + let result = resource.with_handle_mut(|handle| { + match handle { + IoHandle::File(file) => file .flush() - .map_err(|err| VmError::HostError(format!("io_flush failed: {err}"))) - .map(|_| CallReturn::one(Value::Bool(true))) + .map_err(|err| VmError::HostError(format!("io_flush failed: {err}")))?, + IoHandle::PopenWrite { child } => child + .stdin + .as_mut() + .ok_or_else(|| { + VmError::HostError("io_flush popen handle missing stdin".to_string()) + })? + .flush() + .map_err(|err| VmError::HostError(format!("io_flush failed: {err}")))?, + IoHandle::PopenRead { .. } => {} } - IoHandle::PopenRead { .. } => Ok(CallReturn::one(Value::Bool(true))), - }; - IoAsyncCompletion { - restored_handle: Some((handle_id, handle)), - result, - } + Ok(CallReturn::one(Value::Bool(true))) + }); + IoAsyncCompletion::result(result) })?; Ok(HostCallResult::Pending(op_id)) } @@ -364,10 +426,18 @@ pub(super) fn builtin_io_flush(vm: &mut Vm, handle_id: i64) -> VmResult VmResult> { - let handle = io_take_handle(vm, handle_id)?; - let op_id = schedule_io_task(vm, move || IoAsyncCompletion { - restored_handle: None, - result: close_io_handle(handle).map(|_| CallReturn::one(Value::Bool(true))), + let handle = resource_handle(handle_id)?; + let resource = io_resource_for_handle(vm, handle)?; + let op_id = schedule_io_task(vm, Some(handle), move || { + let result = resource + .take_handle() + .and_then(|handle| close_io_handle(handle, CancellationReason::ResourceClosed)) + .map(|_| CallReturn::one(Value::Bool(true))); + IoAsyncCompletion { + opened_handle: None, + closed_handle: Some(handle), + result, + } })?; Ok(HostCallResult::Pending(op_id)) } @@ -376,11 +446,10 @@ pub(super) fn builtin_io_close(vm: &mut Vm, handle_id: i64) -> VmResult VmResult> { let path = path.to_string(); - let op_id = schedule_io_task(vm, move || IoAsyncCompletion { - restored_handle: None, - result: Ok(CallReturn::one(Value::Bool( + let op_id = schedule_io_task(vm, None, move || { + IoAsyncCompletion::result(Ok(CallReturn::one(Value::Bool( std::path::Path::new(path.as_str()).exists(), - ))), + )))) })?; Ok(HostCallResult::Pending(op_id)) } @@ -396,6 +465,9 @@ fn spawn_shell_command(command: &str, mode: &str) -> VmResult { cmd }; + #[cfg(unix)] + process.process_group(0); + match mode { "r" => { process.stdout(Stdio::piped()).stdin(Stdio::null()); @@ -411,61 +483,405 @@ fn spawn_shell_command(command: &str, mode: &str) -> VmResult { .map_err(|err| VmError::HostError(format!("io_popen failed: {err}"))) } -fn io_reserve_handle_id(vm: &mut Vm) -> i64 { - let id = vm.io_state.next_handle; - vm.io_state.next_handle = vm.io_state.next_handle.saturating_add(1); - id -} - -fn io_take_handle(vm: &mut Vm, handle_id: i64) -> VmResult { +fn resource_handle(handle_id: i64) -> VmResult { if handle_id <= 0 { return Err(VmError::HostError(format!( "invalid io handle id {handle_id}; expected positive handle id" ))); } - vm.io_state - .handles - .remove(&handle_id) - .ok_or_else(|| VmError::HostError(format!("io handle {handle_id} not found"))) + ResourceHandle::from_value(&Value::Int(handle_id)).map_err(runtime_host_error) +} + +fn io_resource_for_handle(vm: &Vm, handle: ResourceHandle) -> VmResult> { + vm.host + .runtime_resources + .get::>(handle, ResourceTypeId::IO_FILE) + .cloned() + .map_err(runtime_host_error) +} + +fn insert_io_resource(vm: &mut Vm, handle: IoHandle) -> VmResult { + vm.host + .runtime_resources + .insert_with_cleanup( + ResourceTypeId::IO_FILE, + Arc::new(IoResource::new(handle)), + |resource, reason| resource.close(reason).map_err(io_cleanup_error), + ) + .map_err(runtime_host_error) } fn schedule_io_task( vm: &mut Vm, + target_resource: Option, task: impl FnOnce() -> IoAsyncCompletion + Send + 'static, ) -> VmResult { - let op_id = vm.allocate_host_op_id(); + let operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Io, + Some(&vm.run_ctx.cancellation), + None, + None, + ) + .map_err(runtime_host_error)?; + if let Some(target_resource) = target_resource { + operation.set_resource(target_resource); + } + let op_id = operation.id().raw(); + let worker_operation = operation.clone(); + let worker_token = operation.token(); let (sender, receiver) = oneshot::channel(); - std::thread::Builder::new() + let callback = match vm + .host + .runtime_resources + .insert(ResourceTypeId::CALLBACK, receiver) + { + Ok(callback) => callback, + Err(error) => { + let _ = vm + .host + .runtime_operations + .cancel(operation.id(), CancellationReason::Requested); + return Err(runtime_host_error(error)); + } + }; + operation.set_payload(callback); + + if let Err(error) = std::thread::Builder::new() .name("pd-vm-io".to_string()) .spawn(move || { - let completion = task(); + let completion = if let Some(reason) = worker_token.reason() { + IoAsyncCompletion::result(Err(VmError::HostError(format!( + "io operation cancelled: {reason:?}" + )))) + } else { + task() + }; + match &completion.result { + Ok(_) => { + let _ = worker_operation.complete(); + } + Err(error) => { + let _ = worker_operation.fail( + RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "io::operation", + error.to_string(), + ) + .with_value(op_id), + ); + } + } let _ = sender.send(completion); }) - .map_err(|err| VmError::HostError(format!("failed to spawn io task: {err}")))?; - vm.io_state.pending_ops.insert(op_id, receiver); + { + let runtime_error = RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "io::schedule", + format!("failed to spawn io task: {error}"), + ) + .with_value(op_id); + let _ = super::close_runtime_resource(vm, callback, CancellationReason::Requested); + let _ = vm + .host + .runtime_operations + .fail(operation.id(), runtime_error); + return Err(VmError::HostError(format!( + "failed to spawn io task: {error}" + ))); + } + Ok(op_id) } -fn close_io_handle(mut handle: IoHandle) -> VmResult<()> { +fn runtime_host_error(error: impl std::fmt::Display) -> VmError { + VmError::HostError(error.to_string()) +} + +fn io_cleanup_error(error: VmError) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::ResourceCleanupFailed, + "io::close", + error.to_string(), + ) +} + +fn close_io_handle(mut handle: IoHandle, reason: CancellationReason) -> VmResult<()> { match &mut handle { IoHandle::File(file) => { file.flush().ok(); } - IoHandle::PopenRead { child } => { - child - .wait() - .map_err(|err| VmError::HostError(format!("io_close popen wait failed: {err}")))?; - } + IoHandle::PopenRead { child } => wait_for_child(child, reason)?, IoHandle::PopenWrite { child } => { let _ = child.stdin.take(); - child - .wait() - .map_err(|err| VmError::HostError(format!("io_close popen wait failed: {err}")))?; + wait_for_child(child, reason)?; } } Ok(()) } +fn wait_for_child(child: &mut Child, reason: CancellationReason) -> VmResult<()> { + if reason == CancellationReason::ResourceClosed { + child + .wait() + .map_err(|err| VmError::HostError(format!("io_close popen wait failed: {err}")))?; + return Ok(()); + } + + let deadline = Instant::now() + Duration::from_millis(500); + loop { + match child.try_wait() { + Ok(Some(_)) => return Ok(()), + Ok(None) if Instant::now() >= deadline => { + if let Err(kill_error) = child.kill() { + return match child.try_wait() { + Ok(Some(_)) => Ok(()), + Ok(None) => Err(VmError::HostError(format!( + "timed out waiting for cancelled io process; direct child fallback failed: {kill_error}" + ))), + Err(wait_error) => Err(VmError::HostError(format!( + "direct child fallback failed: {kill_error}; child status check failed: {wait_error}" + ))), + }; + } + child.wait().map_err(|error| { + VmError::HostError(format!( + "io_close popen wait after direct child fallback failed: {error}" + )) + })?; + return Ok(()); + } + Ok(None) => std::thread::sleep(Duration::from_millis(5)), + Err(error) => { + return Err(VmError::HostError(format!( + "io_close popen wait failed: {error}" + ))); + } + } + } +} + +#[cfg(unix)] +fn terminate_process_tree(process_id: u32) -> VmResult<()> { + let process_id = libc::pid_t::try_from(process_id).map_err(|_| { + VmError::HostError(format!( + "io_close popen process id {process_id} exceeds the platform pid range" + )) + })?; + let group_result = signal_unix_process(-process_id); + match group_result { + Ok(()) => Ok(()), + Err(error) if error.raw_os_error() == Some(libc::ESRCH) => Ok(()), + Err(group_error) => { + let fallback_result = signal_unix_process(process_id); + let fallback_detail = match fallback_result { + Ok(()) => "direct process fallback succeeded".to_string(), + Err(error) if error.raw_os_error() == Some(libc::ESRCH) => { + "direct process had already exited".to_string() + } + Err(error) => format!("direct process fallback failed: {error}"), + }; + Err(VmError::HostError(format!( + "io_close popen process-group termination failed: {group_error}; {fallback_detail}" + ))) + } + } +} + +#[cfg(unix)] +fn signal_unix_process(process_id: libc::pid_t) -> std::io::Result<()> { + // SAFETY: process_id is either the tracked child pid or its negative process-group id. + if unsafe { libc::kill(process_id, libc::SIGKILL) } == 0 { + Ok(()) + } else { + Err(std::io::Error::last_os_error()) + } +} + +#[cfg(windows)] +fn terminate_process_tree(process_id: u32) -> VmResult<()> { + windows_process_tree::terminate(process_id) +} + +#[cfg(windows)] +mod windows_process_tree { + use std::collections::{HashMap, HashSet}; + use std::ffi::c_void; + use std::io; + use std::mem; + use std::ptr; + + use super::{VmError, VmResult}; + + type Handle = *mut c_void; + + const INVALID_HANDLE_VALUE: Handle = -1_isize as Handle; + const TH32CS_SNAPPROCESS: u32 = 0x0000_0002; + const PROCESS_TERMINATE: u32 = 0x0001; + const ERROR_NO_MORE_FILES: i32 = 18; + const ERROR_INVALID_PARAMETER: i32 = 87; + + #[repr(C)] + struct ProcessEntry32W { + size: u32, + usage_count: u32, + process_id: u32, + default_heap_id: usize, + module_id: u32, + thread_count: u32, + parent_process_id: u32, + base_priority: i32, + flags: u32, + executable: [u16; 260], + } + + #[link(name = "kernel32")] + unsafe extern "system" { + fn CreateToolhelp32Snapshot(flags: u32, process_id: u32) -> Handle; + fn Process32FirstW(snapshot: Handle, entry: *mut ProcessEntry32W) -> i32; + fn Process32NextW(snapshot: Handle, entry: *mut ProcessEntry32W) -> i32; + fn OpenProcess(access: u32, inherit_handle: i32, process_id: u32) -> Handle; + fn TerminateProcess(process: Handle, exit_code: u32) -> i32; + fn CloseHandle(handle: Handle) -> i32; + } + + pub(super) fn terminate(root_process_id: u32) -> VmResult<()> { + let descendants = match descendant_processes(root_process_id) { + Ok(descendants) => descendants, + Err(snapshot_error) => { + let fallback_detail = match terminate_process(root_process_id) { + Ok(()) => "direct process fallback succeeded".to_string(), + Err(error) => format!("direct process fallback failed: {error}"), + }; + return Err(VmError::HostError(format!( + "io_close popen Windows process-tree snapshot failed: {snapshot_error}; {fallback_detail}" + ))); + } + }; + let mut first_error = None; + for process_id in descendants.into_iter().rev() { + if let Err(error) = terminate_process(process_id) { + first_error.get_or_insert(error); + } + } + if let Err(error) = terminate_process(root_process_id) { + first_error.get_or_insert(error); + } + + match first_error { + Some(error) => Err(VmError::HostError(format!( + "io_close popen Windows process-tree termination failed: {error}" + ))), + None => Ok(()), + } + } + + fn descendant_processes(root_process_id: u32) -> VmResult> { + let entries = snapshot_processes().map_err(|error| { + VmError::HostError(format!( + "io_close popen Windows process snapshot failed: {error}" + )) + })?; + let mut children_by_parent = HashMap::>::new(); + for (process_id, parent_process_id) in entries { + children_by_parent + .entry(parent_process_id) + .or_default() + .push(process_id); + } + + let mut descendants = Vec::new(); + let mut visited = HashSet::new(); + let mut pending = vec![root_process_id]; + visited.insert(root_process_id); + while let Some(parent_process_id) = pending.pop() { + let Some(children) = children_by_parent.get(&parent_process_id) else { + continue; + }; + for &child_process_id in children { + if visited.insert(child_process_id) { + descendants.push(child_process_id); + pending.push(child_process_id); + } + } + } + Ok(descendants) + } + + fn snapshot_processes() -> io::Result> { + // SAFETY: the snapshot API receives fixed constants and initialized storage of the + // documented PROCESSENTRY32W layout. Every acquired handle is closed below. + unsafe { + let snapshot = CreateToolhelp32Snapshot(TH32CS_SNAPPROCESS, 0); + if snapshot == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + + let mut entry: ProcessEntry32W = mem::zeroed(); + entry.size = mem::size_of::() as u32; + let mut entries = Vec::new(); + if Process32FirstW(snapshot, &mut entry) == 0 { + let error = io::Error::last_os_error(); + let _ = CloseHandle(snapshot); + if error.raw_os_error() == Some(ERROR_NO_MORE_FILES) { + return Ok(entries); + } + return Err(error); + } + + loop { + entries.push((entry.process_id, entry.parent_process_id)); + entry = mem::zeroed(); + entry.size = mem::size_of::() as u32; + if Process32NextW(snapshot, &mut entry) == 0 { + let error = io::Error::last_os_error(); + let close_result = CloseHandle(snapshot); + if error.raw_os_error() != Some(ERROR_NO_MORE_FILES) { + return Err(error); + } + if close_result == 0 { + return Err(io::Error::last_os_error()); + } + return Ok(entries); + } + } + } + } + + fn terminate_process(process_id: u32) -> io::Result<()> { + // SAFETY: OpenProcess returns an owned kernel handle which is closed on every path. + unsafe { + let process = OpenProcess(PROCESS_TERMINATE, 0, process_id); + if process == ptr::null_mut() { + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(ERROR_INVALID_PARAMETER) { + return Ok(()); + } + return Err(error); + } + let terminate_result = TerminateProcess(process, 1); + let terminate_error = (terminate_result == 0).then(io::Error::last_os_error); + let close_result = CloseHandle(process); + if let Some(error) = terminate_error { + return Err(error); + } + if close_result == 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) + } + } +} + +#[cfg(not(any(unix, windows)))] +fn terminate_process_tree(process_id: u32) -> VmResult<()> { + Err(VmError::HostError(format!( + "io_close popen process-tree termination is unsupported for process {process_id}" + ))) +} + fn read_line_from_reader(reader: &mut impl Read) -> VmResult { let mut bytes = Vec::new(); let mut one = [0u8; 1]; diff --git a/src/builtins/runtime/io_wasm.rs b/src/builtins/runtime/io_wasm.rs index 3b2998e2..de2fb436 100644 --- a/src/builtins/runtime/io_wasm.rs +++ b/src/builtins/runtime/io_wasm.rs @@ -5,14 +5,6 @@ use pd_host_function::pd_host_function; use super::HostCallResult; use crate::vm::{CallReturn, HostOpId, Value, Vm, VmError, VmResult}; -pub(crate) struct IoState; - -impl Default for IoState { - fn default() -> Self { - Self - } -} - pub(super) fn poll_builtin_io_op( _vm: &mut Vm, op_id: HostOpId, @@ -23,8 +15,6 @@ pub(super) fn poll_builtin_io_op( )))) } -pub(super) fn close_all_handles(_vm: &mut Vm) {} - /// Opens a file handle for runtime I/O. #[pd_host_function(name = "io::open")] pub(super) fn builtin_io_open( diff --git a/src/builtins/runtime/json.rs b/src/builtins/runtime/json.rs index 504d1602..ab1b3ee8 100644 --- a/src/builtins/runtime/json.rs +++ b/src/builtins/runtime/json.rs @@ -10,6 +10,10 @@ use pd_host_function::pd_host_function; /// Encodes a `Value` into a JSON string. #[pd_host_function(name = "json::encode")] pub(super) fn builtin_json_encode(value: &AnyValue) -> VmResult { + encode_value_to_string(value) +} + +pub(super) fn encode_value_to_string(value: &Value) -> VmResult { let json_value = vm_to_json_value(value)?; serde_json::to_string(&json_value) .map_err(|err| VmError::HostError(format!("json_encode failed: {err}"))) diff --git a/src/builtins/runtime/mod.rs b/src/builtins/runtime/mod.rs index 67fac4fc..a79cb4fc 100644 --- a/src/builtins/runtime/mod.rs +++ b/src/builtins/runtime/mod.rs @@ -5,10 +5,31 @@ use std::task::{Context, Poll}; use crate::builtins::BuiltinFunction; use crate::vm::{CallOutcome, CallReturn, HostOpId, Value, Vm, VmResult}; +use self::cancellation::{CancellationReason, OperationId, OperationOwner, OperationState}; +use self::error::{RuntimeError, RuntimeErrorCode}; +use self::resource::ResourceHandle; +#[cfg(feature = "sqlite")] +use self::resource::ResourceTypeId; + +type RuntimeOperationPoller = fn(&mut Vm, HostOpId, &mut Context<'_>) -> Poll>; + +const RUNTIME_OPERATION_POLLERS: &[(OperationOwner, RuntimeOperationPoller)] = &[ + (OperationOwner::Io, io::poll_builtin_io_op), + (OperationOwner::Http, http::poll_pending_op), + #[cfg(feature = "sqlite")] + (OperationOwner::Sqlite, sqlite::poll_pending_op), +]; + mod aot; mod bytes; +pub(crate) mod cancellation; +pub(crate) mod context; +mod context_host; pub(crate) mod core; +pub(crate) mod error; +pub(crate) mod event; mod host; +mod http; #[cfg(not(target_arch = "wasm32"))] mod io; #[cfg(target_arch = "wasm32")] @@ -19,12 +40,16 @@ mod map_iter; mod math; pub(crate) mod print; pub(crate) mod regex; +pub(crate) mod resource; +#[cfg(feature = "sqlite")] +mod sqlite; mod typed; #[cfg(target_arch = "wasm32")] use io_wasm as io; -pub(crate) use io::IoState; +pub use http::HttpConfig; +pub(crate) use http::HttpState; pub use typed::HostCallResult; use typed::{ AnyValue, IntoBuiltinCallOutcome, IntoHostCallOutcome, NumberValue, UnknownValue, VmArray, @@ -124,8 +149,98 @@ pub(crate) fn execute_builtin_call( } } -pub(crate) fn cancel_builtin_io_op(vm: &mut Vm, op_id: HostOpId) { - io::cancel_pending_op(vm, op_id); +pub(crate) fn cancel_builtin_io_op_with_reason( + vm: &mut Vm, + op_id: HostOpId, + reason: CancellationReason, +) { + let Ok(op_id) = OperationId::from_raw(op_id) else { + return; + }; + let target_resource = vm + .host + .runtime_operations + .get(op_id) + .ok() + .filter(|operation| operation.owner() == OperationOwner::Io) + .and_then(|operation| operation.resource()); + cancel_runtime_operation(vm, op_id, reason); + if let Some(target_resource) = target_resource { + let _ = close_runtime_resource(vm, target_resource, reason); + } +} + +pub(crate) fn cancel_runtime_operation( + vm: &mut Vm, + op_id: OperationId, + reason: CancellationReason, +) { + let payload = vm + .host + .runtime_operations + .get(op_id) + .ok() + .and_then(|operation| operation.payload()); + let _ = vm.host.runtime_operations.cancel(op_id, reason); + if let Some(payload) = payload { + let _ = close_runtime_resource(vm, payload, reason); + } +} + +fn cancel_runtime_operations( + vm: &mut Vm, + operations: Vec, + reason: CancellationReason, +) { + let operations = operations + .into_iter() + .map(|operation| { + let payload = operation.payload(); + (operation, payload) + }) + .collect::>(); + for (operation, _) in &operations { + operation.token().mark_cancelled(reason); + } + for (operation, _) in &operations { + let _ = vm.host.runtime_operations.cancel(operation.id(), reason); + } + for (_, payload) in operations { + if let Some(payload) = payload { + let _ = close_runtime_resource(vm, payload, reason); + } + } +} + +pub(crate) fn close_runtime_resource( + vm: &mut Vm, + handle: ResourceHandle, + reason: CancellationReason, +) -> error::RuntimeResult { + let operations = vm.host.runtime_operations.operations_for_resource(handle); + cancel_runtime_operations(vm, operations, reason); + vm.host.runtime_resources.close(handle, reason) +} + +#[cfg(feature = "sqlite")] +pub(crate) fn close_resources_by_type( + vm: &mut Vm, + resource_type: ResourceTypeId, + reason: CancellationReason, +) { + let handles = vm.host.runtime_resources.handles_of_type(resource_type); + for handle in handles { + let _ = close_runtime_resource(vm, handle, reason); + } +} + +pub(crate) fn cancel_operations_by_owner( + vm: &mut Vm, + owner: OperationOwner, + reason: CancellationReason, +) { + let operations = vm.host.runtime_operations.operations_by_owner(owner); + cancel_runtime_operations(vm, operations, reason); } pub(crate) fn poll_builtin_io_op( @@ -133,11 +248,63 @@ pub(crate) fn poll_builtin_io_op( op_id: HostOpId, cx: &mut Context<'_>, ) -> Poll> { - io::poll_builtin_io_op(vm, op_id, cx) + let operation_id = match OperationId::from_raw(op_id) { + Ok(operation_id) => operation_id, + Err(error) => { + return Poll::Ready(Err(crate::vm::VmError::HostError(error.to_string()))); + } + }; + let operation = match vm.host.runtime_operations.get(operation_id) { + Ok(operation) => operation, + Err(error) => { + return Poll::Ready(Err(crate::vm::VmError::HostError(error.to_string()))); + } + }; + if let Err(error) = operation.token().check() { + let reason = operation + .token() + .reason() + .unwrap_or(CancellationReason::Requested); + cancel_builtin_io_op_with_reason(vm, op_id, reason); + return Poll::Ready(Err(crate::vm::VmError::HostError(error.to_string()))); + } + + let Some((_, poller)) = RUNTIME_OPERATION_POLLERS + .iter() + .find(|(owner, _)| *owner == operation.owner()) + else { + return Poll::Ready(Err(crate::vm::VmError::HostError(format!( + "runtime operation owner {:?} is unavailable in this build", + operation.owner() + )))); + }; + let result = poller(vm, op_id, cx); + + match result { + Poll::Pending => Poll::Pending, + Poll::Ready(Ok(values)) => { + let _ = vm.host.runtime_operations.complete(operation_id); + Poll::Ready(Ok(values)) + } + Poll::Ready(Err(error)) => { + if let Some(reason) = operation.token().reason() { + cancel_builtin_io_op_with_reason(vm, op_id, reason); + return Poll::Ready(Err(error)); + } + let runtime_error = RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "runtime::operation", + error.to_string(), + ) + .with_value(op_id); + let _ = vm.host.runtime_operations.fail(operation_id, runtime_error); + Poll::Ready(Err(error)) + } + } } pub(crate) fn close_all_handles(vm: &mut Vm) { - io::close_all_handles(vm); + vm.host.reset_for_reuse(); } #[cfg(test)] diff --git a/src/builtins/runtime/resource.rs b/src/builtins/runtime/resource.rs new file mode 100644 index 00000000..57b66ea7 --- /dev/null +++ b/src/builtins/runtime/resource.rs @@ -0,0 +1,566 @@ +use std::any::Any; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::vm::Value; + +use super::cancellation::CancellationReason; +use super::error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; + +pub const DEFAULT_MAX_RESOURCES: usize = 1024; + +const HANDLE_TYPE_BITS: u64 = 8; +const HANDLE_GENERATION_BITS: u64 = 17; +const HANDLE_SLOT_BITS: u64 = 18; +const HANDLE_ARENA_BITS: u64 = 63 - HANDLE_TYPE_BITS - HANDLE_GENERATION_BITS - HANDLE_SLOT_BITS; + +const HANDLE_TYPE_SHIFT: u64 = 0; +const HANDLE_GENERATION_SHIFT: u64 = HANDLE_TYPE_BITS; +const HANDLE_SLOT_SHIFT: u64 = HANDLE_GENERATION_SHIFT + HANDLE_GENERATION_BITS; +const HANDLE_ARENA_SHIFT: u64 = HANDLE_SLOT_SHIFT + HANDLE_SLOT_BITS; + +const HANDLE_TYPE_MASK: u64 = (1 << HANDLE_TYPE_BITS) - 1; +const HANDLE_GENERATION_MASK: u64 = (1 << HANDLE_GENERATION_BITS) - 1; +const HANDLE_SLOT_MASK: u64 = (1 << HANDLE_SLOT_BITS) - 1; +const HANDLE_ARENA_MASK: u64 = (1 << HANDLE_ARENA_BITS) - 1; + +/// Process-wide monotonic arena identity source. Arena identities are not +/// recycled, so a handle from a dropped VM cannot resolve in a later VM. +static NEXT_ARENA_ID: AtomicU64 = AtomicU64::new(1); + +/// Stable resource type identity carried by every opaque handle. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ResourceTypeId(u16); + +impl ResourceTypeId { + pub const IO_FILE: Self = Self(1); + #[cfg_attr(not(feature = "http-client"), allow(dead_code))] + pub const HTTP_REQUEST: Self = Self(3); + #[cfg_attr(not(feature = "sqlite"), allow(dead_code))] + pub const SQLITE_CONNECTION: Self = Self(5); + pub const CALLBACK: Self = Self(6); + + pub const fn raw(self) -> u16 { + self.0 + } +} + +/// A positive VM integer identifying one typed resource without exposing it. +/// +/// The token carries arena, slot, generation, and resource-type identity. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ResourceHandle(u64); + +impl ResourceHandle { + pub fn as_value(self) -> Value { + Value::Int(self.0 as i64) + } + + pub fn from_value(value: &Value) -> RuntimeResult { + let Value::Int(raw) = value else { + return Err(invalid_handle("resource handle must be an integer token")); + }; + if *raw <= 0 { + return Err(invalid_handle("resource handle must be a positive token")); + } + Self::from_encoded(*raw as u64) + } + + pub const fn resource_type(self) -> ResourceTypeId { + ResourceTypeId(((self.0 >> HANDLE_TYPE_SHIFT) & HANDLE_TYPE_MASK) as u16) + } + + const fn arena_id(self) -> u64 { + (self.0 >> HANDLE_ARENA_SHIFT) & HANDLE_ARENA_MASK + } + + const fn slot_identity(self) -> u64 { + (self.0 >> HANDLE_SLOT_SHIFT) & HANDLE_SLOT_MASK + } + + const fn generation(self) -> u64 { + (self.0 >> HANDLE_GENERATION_SHIFT) & HANDLE_GENERATION_MASK + } + + fn slot_index(self) -> RuntimeResult { + usize::try_from(self.slot_identity() - 1) + .map_err(|_| invalid_handle("resource handle slot is out of range")) + } + + fn from_encoded(encoded: u64) -> RuntimeResult { + let handle = Self(encoded); + if encoded == 0 + || encoded > i64::MAX as u64 + || handle.arena_id() == 0 + || handle.slot_identity() == 0 + || handle.generation() == 0 + || handle.resource_type().raw() == 0 + { + return Err(invalid_handle( + "resource handle token has an invalid encoding", + )); + } + Ok(handle) + } + + fn encode( + arena_id: u64, + slot_index: usize, + generation: u64, + resource_type: ResourceTypeId, + ) -> RuntimeResult { + let slot_identity = u64::try_from(slot_index) + .ok() + .and_then(|slot| slot.checked_add(1)) + .ok_or_else(|| invalid_handle("resource slot identity overflowed"))?; + if arena_id == 0 + || arena_id > HANDLE_ARENA_MASK + || slot_identity > HANDLE_SLOT_MASK + || generation == 0 + || generation > HANDLE_GENERATION_MASK + || resource_type.raw() == 0 + || u64::from(resource_type.raw()) > HANDLE_TYPE_MASK + { + return Err(invalid_handle( + "resource handle components are out of range", + )); + } + let encoded = (arena_id << HANDLE_ARENA_SHIFT) + | (slot_identity << HANDLE_SLOT_SHIFT) + | (generation << HANDLE_GENERATION_SHIFT) + | (u64::from(resource_type.raw()) << HANDLE_TYPE_SHIFT); + Self::from_encoded(encoded) + } +} + +type ErasedResource = Box; +type ResourceCleanup = + Box RuntimeResult<()> + Send + 'static>; + +struct ResourceSlot { + generation: u32, + resource_type: ResourceTypeId, + value: Option, + cleanup: Option, +} + +/// VM-local bounded arena for typed opaque host resources. +pub struct ResourceArena { + arena_id: u64, + max_entries: usize, + slots: Vec, + vacant_slots: Vec, + active_entries: usize, +} + +impl ResourceArena { + pub fn with_limit(max_entries: usize) -> RuntimeResult { + if max_entries == 0 || max_entries > HANDLE_SLOT_MASK as usize { + return Err(RuntimeError::new( + RuntimeErrorCode::InvalidConfiguration, + "resource::arena", + format!( + "resource arena capacity must be between 1 and {}", + HANDLE_SLOT_MASK + ), + ) + .with_limit(HANDLE_SLOT_MASK as usize)); + } + let arena_id = NEXT_ARENA_ID + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |arena_id| { + (arena_id <= HANDLE_ARENA_MASK).then_some(arena_id + 1) + }) + .map_err(|_| { + RuntimeError::new( + RuntimeErrorCode::ResourceIdExhausted, + "resource::arena", + "resource arena identity space is exhausted", + ) + })?; + Ok(Self { + arena_id, + max_entries, + slots: Vec::new(), + vacant_slots: Vec::new(), + active_entries: 0, + }) + } + + pub fn insert( + &mut self, + resource_type: ResourceTypeId, + value: T, + ) -> RuntimeResult + where + T: Any + Send + 'static, + { + self.allocate(resource_type, Box::new(value), None) + } + + pub fn insert_with_cleanup( + &mut self, + resource_type: ResourceTypeId, + value: T, + cleanup: F, + ) -> RuntimeResult + where + T: Any + Send + 'static, + F: FnOnce(T, CancellationReason) -> RuntimeResult<()> + Send + 'static, + { + let erased_cleanup: ResourceCleanup = Box::new(move |value, reason| { + let value = value.downcast::().map_err(|_| { + RuntimeError::new( + RuntimeErrorCode::ResourceTypeMismatch, + "resource::cleanup", + "resource cleanup received the wrong concrete type", + ) + })?; + cleanup(*value, reason) + }); + self.allocate(resource_type, Box::new(value), Some(erased_cleanup)) + } + + #[cfg_attr(not(feature = "sqlite"), allow(dead_code))] + pub fn count_type(&self, resource_type: ResourceTypeId) -> usize { + self.slots + .iter() + .filter(|slot| slot.resource_type == resource_type && slot.value.is_some()) + .count() + } + + #[cfg(feature = "sqlite")] + pub fn handles_of_type(&self, resource_type: ResourceTypeId) -> Vec { + self.slots + .iter() + .enumerate() + .filter(|(_, slot)| slot.resource_type == resource_type && slot.value.is_some()) + .filter_map(|(slot_index, slot)| { + ResourceHandle::encode( + self.arena_id, + slot_index, + u64::from(slot.generation), + slot.resource_type, + ) + .ok() + }) + .collect() + } + + pub fn get(&self, handle: ResourceHandle, expected_type: ResourceTypeId) -> RuntimeResult<&T> + where + T: Any + Send + 'static, + { + self.active_slot(handle, expected_type)? + .value + .as_ref() + .and_then(|value| value.downcast_ref::()) + .ok_or_else(|| type_mismatch(handle, expected_type)) + } + + pub fn get_mut( + &mut self, + handle: ResourceHandle, + expected_type: ResourceTypeId, + ) -> RuntimeResult<&mut T> + where + T: Any + Send + 'static, + { + self.active_slot_mut(handle, expected_type)? + .value + .as_mut() + .and_then(|value| value.downcast_mut::()) + .ok_or_else(|| type_mismatch(handle, expected_type)) + } + + pub fn close( + &mut self, + handle: ResourceHandle, + reason: CancellationReason, + ) -> RuntimeResult { + let slot_index = self.validate_handle_identity(handle)?; + let (value, cleanup, reusable) = { + let slot = &mut self.slots[slot_index]; + validate_slot_identity(slot, handle)?; + if slot.resource_type != handle.resource_type() { + return Err(type_mismatch(handle, slot.resource_type)); + } + let Some(value) = slot.value.take() else { + return Ok(CloseStatus::AlreadyClosed); + }; + self.active_entries -= 1; + ( + value, + slot.cleanup.take(), + u64::from(slot.generation) < HANDLE_GENERATION_MASK, + ) + }; + if reusable { + self.vacant_slots.push(slot_index); + } + let result = if let Some(cleanup) = cleanup { + cleanup(value, reason) + } else { + drop(value); + Ok(()) + }; + result.map(|()| CloseStatus::Closed).map_err(|error| { + RuntimeError::new( + RuntimeErrorCode::ResourceCleanupFailed, + "resource::close", + error.to_string(), + ) + .with_value(handle.0) + }) + } + + pub fn close_all(&mut self, reason: CancellationReason) -> RuntimeResult { + let handles = self + .slots + .iter() + .enumerate() + .filter_map(|(slot_index, slot)| { + slot.value.as_ref().map(|_| { + ResourceHandle::encode( + self.arena_id, + slot_index, + u64::from(slot.generation), + slot.resource_type, + ) + .expect("active resource slot must have an encodable handle") + }) + }) + .collect::>(); + let mut closed = 0; + let mut first_error = None; + for handle in handles { + match self.close(handle, reason) { + Ok(CloseStatus::Closed) => closed += 1, + Ok(CloseStatus::AlreadyClosed) => {} + Err(error) => { + first_error.get_or_insert(error); + } + } + } + match first_error { + Some(error) => Err(error), + None => Ok(closed), + } + } + + fn allocate( + &mut self, + resource_type: ResourceTypeId, + value: ErasedResource, + cleanup: Option, + ) -> RuntimeResult { + if resource_type.raw() == 0 || u64::from(resource_type.raw()) > HANDLE_TYPE_MASK { + return Err(RuntimeError::new( + RuntimeErrorCode::ResourceTypeMismatch, + "resource::insert", + "resource type id is outside the handle encoding range", + )); + } + if self.active_entries >= self.max_entries { + return Err(RuntimeError::new( + RuntimeErrorCode::ResourceLimitExceeded, + "resource::insert", + "resource arena capacity has been reached", + ) + .with_limit(self.max_entries)); + } + + let (slot_index, generation) = if let Some(slot_index) = self.vacant_slots.pop() { + let slot = &mut self.slots[slot_index]; + let generation = slot + .generation + .checked_add(1) + .filter(|generation| u64::from(*generation) <= HANDLE_GENERATION_MASK) + .expect("only reusable resource generations enter the vacant list"); + slot.generation = generation; + slot.resource_type = resource_type; + slot.value = Some(value); + slot.cleanup = cleanup; + (slot_index, generation) + } else { + if self.slots.len() >= self.max_entries { + return Err(RuntimeError::new( + RuntimeErrorCode::ResourceIdExhausted, + "resource::insert", + "resource slot generation space is exhausted", + )); + } + let slot_index = self.slots.len(); + let generation = 1; + self.slots.push(ResourceSlot { + generation, + resource_type, + value: Some(value), + cleanup, + }); + (slot_index, generation) + }; + self.active_entries += 1; + ResourceHandle::encode( + self.arena_id, + slot_index, + u64::from(generation), + resource_type, + ) + } + + fn validate_handle_identity(&self, handle: ResourceHandle) -> RuntimeResult { + if handle.arena_id() != self.arena_id { + return Err(wrong_arena(handle)); + } + let slot_index = handle.slot_index()?; + if slot_index >= self.slots.len() { + return Err(stale_handle(handle)); + } + Ok(slot_index) + } + + fn active_slot( + &self, + handle: ResourceHandle, + expected_type: ResourceTypeId, + ) -> RuntimeResult<&ResourceSlot> { + validate_type(handle, expected_type)?; + let slot_index = self.validate_handle_identity(handle)?; + let slot = &self.slots[slot_index]; + validate_slot(slot, handle, expected_type)?; + Ok(slot) + } + + fn active_slot_mut( + &mut self, + handle: ResourceHandle, + expected_type: ResourceTypeId, + ) -> RuntimeResult<&mut ResourceSlot> { + validate_type(handle, expected_type)?; + let slot_index = self.validate_handle_identity(handle)?; + let slot = &mut self.slots[slot_index]; + validate_slot(slot, handle, expected_type)?; + Ok(slot) + } +} + +impl Default for ResourceArena { + fn default() -> Self { + Self::with_limit(DEFAULT_MAX_RESOURCES) + .expect("default resource arena configuration should be valid") + } +} + +impl Drop for ResourceArena { + fn drop(&mut self) { + let _ = self.close_all(CancellationReason::VmReset); + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CloseStatus { + Closed, + AlreadyClosed, +} + +fn validate_type(handle: ResourceHandle, expected_type: ResourceTypeId) -> RuntimeResult<()> { + if handle.resource_type() != expected_type { + return Err(type_mismatch(handle, expected_type)); + } + Ok(()) +} + +fn validate_slot_identity(slot: &ResourceSlot, handle: ResourceHandle) -> RuntimeResult<()> { + if u64::from(slot.generation) != handle.generation() { + return Err(stale_handle(handle)); + } + Ok(()) +} + +fn validate_slot( + slot: &ResourceSlot, + handle: ResourceHandle, + expected_type: ResourceTypeId, +) -> RuntimeResult<()> { + validate_slot_identity(slot, handle)?; + if slot.resource_type != expected_type { + return Err(type_mismatch(handle, expected_type)); + } + if slot.value.is_none() { + return Err(already_closed_error(handle)); + } + Ok(()) +} + +fn invalid_handle(message: &'static str) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::InvalidResourceHandle, + "resource::handle", + message, + ) +} + +fn wrong_arena(handle: ResourceHandle) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::ResourceHandleWrongTable, + "resource::handle", + "resource handle does not belong to this VM arena", + ) + .with_value(handle.0) +} + +fn stale_handle(handle: ResourceHandle) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::ResourceStale, + "resource::handle", + "resource handle refers to a stale slot generation", + ) + .with_value(handle.0) +} + +fn already_closed_error(handle: ResourceHandle) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::ResourceAlreadyClosed, + "resource::handle", + "resource is already closed", + ) + .with_value(handle.0) +} + +fn type_mismatch(handle: ResourceHandle, expected: ResourceTypeId) -> RuntimeError { + RuntimeError::new( + RuntimeErrorCode::ResourceTypeMismatch, + "resource::handle", + format!( + "resource type {} does not match expected type {}", + handle.resource_type().raw(), + expected.raw() + ), + ) + .with_value(handle.0) +} + +#[cfg(test)] +mod tests { + use super::{CancellationReason, ResourceArena, ResourceTypeId}; + + #[test] + fn vacant_slot_reuse_increments_the_generation() { + let mut arena = ResourceArena::with_limit(1).expect("arena should be valid"); + let first = arena + .insert(ResourceTypeId::IO_FILE, 1_u8) + .expect("first resource should be inserted"); + assert_eq!( + arena + .close(first, CancellationReason::ResourceClosed) + .expect("first resource should close"), + super::CloseStatus::Closed + ); + + let replacement = arena + .insert(ResourceTypeId::IO_FILE, 2_u8) + .expect("vacant slot should be reused"); + + assert_eq!(replacement.slot_identity(), first.slot_identity()); + assert_eq!(replacement.generation(), first.generation() + 1); + } +} diff --git a/src/builtins/runtime/sqlite.rs b/src/builtins/runtime/sqlite.rs new file mode 100644 index 00000000..05b1a87a --- /dev/null +++ b/src/builtins/runtime/sqlite.rs @@ -0,0 +1,1168 @@ +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex, mpsc}; +use std::task::{Context, Poll, Waker}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use pd_host_function::pd_host_function; +use rusqlite::hooks::{AuthAction, AuthContext, Authorization}; +use rusqlite::limits::Limit; +use rusqlite::types::{Value as SqlValue, ValueRef}; +use rusqlite::{Connection, OpenFlags, TransactionBehavior, params_from_iter}; + +use super::cancellation::{ + CancellationReason, CancellationToken, OperationId, OperationOwner, OperationStatus, +}; +use super::error::{RuntimeError, RuntimeErrorCode}; +use super::resource::{ResourceHandle, ResourceTypeId}; +use super::typed::{VmArrayRef, VmMapRef}; +use super::{HostCallResult, VmMap}; +use crate::vm::{CallReturn, HostOpId, SqliteLimits, Value, Vm, VmError, VmResult}; + +const SQLITE_PROGRESS_STEPS: i32 = 1_000; +const SQLITE_CLOSE_GRACE: Duration = Duration::from_millis(100); + +/// Returns the affected-row count from a SQLite result envelope. +#[pd_host_function(name = "sqlite::rows_affected")] +pub(super) fn builtin_sqlite_rows_affected_impl(value: VmMapRef<'_>) -> VmResult { + match value.get(&Value::string("rows_affected")) { + Some(Value::Int(value)) => Ok(*value), + Some(_) => Err(VmError::TypeMismatch("SQLite rows_affected integer")), + None => Ok(0), + } +} + +/// Returns the truncation flag from a SQLite query result envelope. +#[pd_host_function(name = "sqlite::truncated")] +pub(super) fn builtin_sqlite_truncated_impl(value: VmMapRef<'_>) -> VmResult { + match value.get(&Value::string("truncated")) { + Some(Value::Bool(value)) => Ok(*value), + Some(_) => Err(VmError::TypeMismatch("SQLite truncated boolean")), + None => Ok(false), + } +} + +/// Returns the continuation cursor from a SQLite query result envelope. +#[pd_host_function(name = "sqlite::next_cursor")] +pub(super) fn builtin_sqlite_next_cursor_impl(value: VmMapRef<'_>) -> VmResult { + match value.get(&Value::string("next_cursor")) { + Some(Value::Int(value)) => Ok(*value), + Some(_) => Err(VmError::TypeMismatch("SQLite next_cursor integer")), + None => Ok(0), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum OpenMode { + Memory, + ReadOnly, + ReadWrite, + ReadWriteCreate, +} + +struct OpenOptions { + path: String, + mode: OpenMode, + root: Option, + limits: SqliteLimits, + allow_unsafe_sql: bool, +} + +struct ConnectionSlot { + connection: Mutex, + execution: Mutex<()>, + active_operation: Mutex>, + interrupt: Arc, + limits: SqliteLimits, + allow_unsafe_sql: bool, +} + +struct PendingResult { + receiver: mpsc::Receiver>, + worker: Option>, + waker: Arc>>, +} + +fn runtime_error(error: RuntimeError) -> VmError { + VmError::HostError(error.to_string()) +} + +fn operation_id(op_id: HostOpId) -> VmResult { + OperationId::from_raw(op_id).map_err(runtime_error) +} + +fn handle_value(handle: ResourceHandle) -> i64 { + match handle.as_value() { + Value::Int(value) => value, + _ => unreachable!("resource handles are integer values"), + } +} + +fn sqlite_handle(raw: i64) -> VmResult { + let handle = ResourceHandle::from_value(&Value::Int(raw)) + .map_err(|error| VmError::HostError(format!("unknown SQLite database handle: {error}")))?; + if handle.resource_type() != ResourceTypeId::SQLITE_CONNECTION { + return Err(VmError::HostError( + "unknown SQLite database handle (wrong resource type)".to_string(), + )); + } + Ok(handle) +} + +fn lookup_connection(vm: &Vm, raw: i64) -> VmResult<(ResourceHandle, Arc)> { + let handle = sqlite_handle(raw)?; + let slot = vm + .host + .runtime_resources + .get::>(handle, ResourceTypeId::SQLITE_CONNECTION) + .map_err(|error| VmError::HostError(format!("unknown SQLite database: {error}")))?; + Ok((handle, Arc::clone(slot))) +} + +fn map_value<'a>(map: &'a VmMap, key: &str) -> Option<&'a Value> { + map.get(&Value::string(key)) +} + +fn required_string(map: &VmMap, key: &str) -> VmResult { + match map_value(map, key) { + Some(Value::String(value)) if !value.is_empty() => Ok(value.as_ref().clone()), + Some(Value::String(_)) => Err(VmError::HostError(format!( + "SQLite {key} must not be empty" + ))), + Some(_) => Err(VmError::TypeMismatch("SQLite option string")), + None => Err(VmError::HostError(format!("missing SQLite {key}"))), + } +} + +fn optional_string(map: &VmMap, key: &str) -> VmResult> { + match map_value(map, key) { + Some(Value::String(value)) => Ok(Some(value.as_ref().clone())), + Some(Value::Null) | None => Ok(None), + Some(_) => Err(VmError::TypeMismatch("SQLite option string")), + } +} + +fn parse_positive_usize(value: &Value, label: &str) -> VmResult { + let Value::Int(value) = value else { + return Err(VmError::TypeMismatch("SQLite limit integer")); + }; + if *value <= 0 { + return Err(VmError::HostError(format!( + "SQLite {label} must be positive" + ))); + } + usize::try_from(*value).map_err(|_| VmError::HostError(format!("SQLite {label} is too large"))) +} + +fn parse_positive_u64(value: &Value, label: &str) -> VmResult { + let Value::Int(value) = value else { + return Err(VmError::TypeMismatch("SQLite limit integer")); + }; + if *value <= 0 { + return Err(VmError::HostError(format!( + "SQLite {label} must be positive" + ))); + } + u64::try_from(*value).map_err(|_| VmError::HostError(format!("SQLite {label} is too large"))) +} + +fn parse_limits(value: Option<&Value>, ceiling: SqliteLimits) -> VmResult { + let Some(value) = value else { + return Ok(ceiling); + }; + let Value::Map(map) = value else { + return Err(VmError::TypeMismatch("SQLite limits map")); + }; + let mut limits = ceiling; + for (key, value) in map.iter() { + let Value::String(key) = key else { + return Err(VmError::TypeMismatch("SQLite limit name")); + }; + match key.as_str() { + "max_connections" => { + limits.max_connections = + parse_positive_usize(value, key)?.min(ceiling.max_connections) + } + "max_statements" => { + limits.max_statements = + parse_positive_usize(value, key)?.min(ceiling.max_statements) + } + "max_rows" => limits.max_rows = parse_positive_usize(value, key)?.min(ceiling.max_rows), + "max_columns" => { + limits.max_columns = parse_positive_usize(value, key)?.min(ceiling.max_columns) + } + "max_result_bytes" => { + limits.max_result_bytes = + parse_positive_usize(value, key)?.min(ceiling.max_result_bytes) + } + "max_statement_bytes" => { + limits.max_statement_bytes = + parse_positive_usize(value, key)?.min(ceiling.max_statement_bytes) + } + "max_parameters" => { + limits.max_parameters = + parse_positive_usize(value, key)?.min(ceiling.max_parameters) + } + "max_parameter_bytes" => { + limits.max_parameter_bytes = + parse_positive_usize(value, key)?.min(ceiling.max_parameter_bytes) + } + "max_pending_operations" => { + limits.max_pending_operations = + parse_positive_usize(value, key)?.min(ceiling.max_pending_operations) + } + "max_transaction_ms" => { + limits.max_transaction_ms = + parse_positive_u64(value, key)?.min(ceiling.max_transaction_ms) + } + "busy_timeout_ms" => { + limits.busy_timeout_ms = + parse_positive_u64(value, key)?.min(ceiling.busy_timeout_ms) + } + _ => { + return Err(VmError::HostError(format!("unknown SQLite limit {key}"))); + } + } + } + Ok(limits) +} + +fn parse_open_options(vm: &Vm, options: &VmMap) -> VmResult { + let path = required_string(options, "path")?; + let mode = match optional_string(options, "mode")?.as_deref() { + Some("memory") => OpenMode::Memory, + Some("read_only") => OpenMode::ReadOnly, + Some("read_write") => OpenMode::ReadWrite, + Some("read_write_create") | None => OpenMode::ReadWriteCreate, + Some(mode) => { + return Err(VmError::HostError(format!( + "unknown SQLite open mode {mode}" + ))); + } + }; + let configured_root = vm + .host + .sqlite_policy + .database_root + .as_deref() + .map(PathBuf::from); + if let Some(requested_root) = optional_string(options, "root")? { + let requested_root = PathBuf::from(requested_root); + if configured_root.as_ref() != Some(&requested_root) { + return Err(VmError::HostError( + "SQLite root must match the embedding policy".to_string(), + )); + } + } + if mode != OpenMode::Memory && configured_root.is_none() { + return Err(VmError::HostError( + "SQLite database root is not configured".to_string(), + )); + } + let limits = parse_limits(map_value(options, "limits"), vm.host.sqlite_policy.limits)?; + Ok(OpenOptions { + path, + mode, + root: configured_root, + limits, + allow_unsafe_sql: vm.host.sqlite_policy.allow_unsafe_sql, + }) +} + +fn parse_query_limits(value: &VmMap, ceiling: SqliteLimits) -> VmResult { + parse_limits(Some(&Value::Map(Arc::new(value.clone()))), ceiling) +} + +fn validate_relative_path(path: &Path) -> VmResult<()> { + if path.as_os_str().is_empty() || path.is_absolute() { + return Err(VmError::HostError( + "SQLite database path must be a non-empty relative path".to_string(), + )); + } + if path.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }) { + return Err(VmError::HostError( + "SQLite database path must stay below its configured root".to_string(), + )); + } + Ok(()) +} + +fn canonical_root(root: &Path) -> VmResult { + if !root.is_absolute() { + return Err(VmError::HostError( + "SQLite database root must be absolute".to_string(), + )); + } + fs::canonicalize(root) + .map_err(|error| VmError::HostError(format!("invalid SQLite database root: {error}"))) +} + +fn resolve_database_path(options: &OpenOptions) -> VmResult> { + if options.mode == OpenMode::Memory { + if options.path != ":memory:" { + return Err(VmError::HostError( + "SQLite memory mode requires path ':memory:'".to_string(), + )); + } + return Ok(None); + } + if options.path == ":memory:" { + return Err(VmError::HostError( + "SQLite ':memory:' requires memory open mode".to_string(), + )); + } + let root = options + .root + .as_deref() + .ok_or_else(|| VmError::HostError("SQLite database root is required".to_string()))?; + let root = canonical_root(root)?; + let relative = Path::new(&options.path); + validate_relative_path(relative)?; + let candidate = root.join(relative); + let canonical = if candidate.exists() { + fs::canonicalize(&candidate) + .map_err(|error| VmError::HostError(format!("invalid SQLite database path: {error}")))? + } else { + if options.mode != OpenMode::ReadWriteCreate { + return Err(VmError::HostError(format!( + "SQLite database does not exist: {}", + candidate.display() + ))); + } + let parent = candidate + .parent() + .ok_or_else(|| VmError::HostError("SQLite database path has no parent".to_string()))?; + let canonical_parent = fs::canonicalize(parent).map_err(|error| { + VmError::HostError(format!("invalid SQLite database parent: {error}")) + })?; + let file_name = candidate.file_name().ok_or_else(|| { + VmError::HostError("SQLite database path has no file name".to_string()) + })?; + canonical_parent.join(file_name) + }; + if !canonical.starts_with(&root) { + return Err(VmError::HostError( + "SQLite database path escapes its configured root".to_string(), + )); + } + Ok(Some(canonical)) +} + +fn sqlite_limit(value: usize, label: &str) -> VmResult { + i32::try_from(value) + .map_err(|_| VmError::HostError(format!("SQLite {label} exceeds engine limits"))) +} + +fn install_connection_limits(connection: &Connection, limits: SqliteLimits) -> VmResult<()> { + let max_value_bytes = limits.max_result_bytes.max(limits.max_parameter_bytes); + connection.set_limit( + Limit::SQLITE_LIMIT_LENGTH, + sqlite_limit(max_value_bytes, "value byte limit")?, + ); + connection.set_limit( + Limit::SQLITE_LIMIT_SQL_LENGTH, + sqlite_limit(limits.max_statement_bytes, "statement byte limit")?, + ); + connection.set_limit( + Limit::SQLITE_LIMIT_COLUMN, + sqlite_limit(limits.max_columns, "column limit")?, + ); + connection.set_limit( + Limit::SQLITE_LIMIT_VARIABLE_NUMBER, + sqlite_limit(limits.max_parameters, "parameter count limit")?, + ); + Ok(()) +} + +fn install_authorizer(connection: &Connection, allow_unsafe_sql: bool) { + connection.authorizer(Some(move |context: AuthContext<'_>| { + if allow_unsafe_sql { + return Authorization::Allow; + } + match context.action { + AuthAction::Attach { .. } + | AuthAction::Detach { .. } + | AuthAction::Pragma { .. } + | AuthAction::CreateVtable { .. } + | AuthAction::DropVtable { .. } + | AuthAction::Unknown { .. } => Authorization::Deny, + AuthAction::Function { function_name } + if function_name.eq_ignore_ascii_case("load_extension") => + { + Authorization::Deny + } + _ => Authorization::Allow, + } + })); +} + +fn open_connection(options: &OpenOptions) -> VmResult { + let path = resolve_database_path(options)?; + let flags = match options.mode { + OpenMode::Memory => OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE, + OpenMode::ReadOnly => OpenFlags::SQLITE_OPEN_READ_ONLY, + OpenMode::ReadWrite => OpenFlags::SQLITE_OPEN_READ_WRITE, + OpenMode::ReadWriteCreate => { + OpenFlags::SQLITE_OPEN_READ_WRITE | OpenFlags::SQLITE_OPEN_CREATE + } + } | OpenFlags::SQLITE_OPEN_NO_MUTEX; + let connection = match path { + Some(path) => Connection::open_with_flags(path, flags), + None => Connection::open_in_memory_with_flags(flags), + } + .map_err(sqlite_error)?; + connection + .busy_timeout(Duration::from_millis(options.limits.busy_timeout_ms)) + .map_err(sqlite_error)?; + install_connection_limits(&connection, options.limits)?; + install_authorizer(&connection, options.allow_unsafe_sql); + Ok(connection) +} + +fn normalized_sql(sql: &str) -> VmResult { + let bytes = sql.as_bytes(); + let mut out = String::with_capacity(sql.len()); + let mut index = 0; + let mut quote = None; + let mut statement_ended = false; + while index < bytes.len() { + let byte = bytes[index]; + if let Some(active_quote) = quote { + if byte == active_quote { + if index + 1 < bytes.len() && bytes[index + 1] == active_quote { + index += 2; + continue; + } + quote = None; + } + index += 1; + continue; + } + if matches!(byte, b'\'' | b'"' | b'`') { + quote = Some(byte); + out.push(' '); + index += 1; + continue; + } + if byte == b'-' && index + 1 < bytes.len() && bytes[index + 1] == b'-' { + index += 2; + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + out.push(' '); + continue; + } + if byte == b'/' && index + 1 < bytes.len() && bytes[index + 1] == b'*' { + index += 2; + while index + 1 < bytes.len() && !(bytes[index] == b'*' && bytes[index + 1] == b'/') { + index += 1; + } + if index + 1 >= bytes.len() { + return Err(VmError::HostError( + "SQLite SQL contains an unterminated comment".to_string(), + )); + } + index += 2; + out.push(' '); + continue; + } + if byte == b';' { + statement_ended = true; + index += 1; + continue; + } + if statement_ended && !byte.is_ascii_whitespace() { + return Err(VmError::HostError( + "multiple SQLite statements are not allowed".to_string(), + )); + } + out.push((byte as char).to_ascii_lowercase()); + index += 1; + } + if quote.is_some() { + return Err(VmError::HostError( + "SQLite SQL contains an unterminated quote".to_string(), + )); + } + Ok(out) +} + +fn validate_sql(sql: &str, limits: SqliteLimits, allow_unsafe_sql: bool) -> VmResult<()> { + if sql.is_empty() || sql.len() > limits.max_statement_bytes || sql.as_bytes().contains(&0) { + return Err(VmError::HostError(format!( + "SQLite statement exceeds the configured {} byte limit or is invalid", + limits.max_statement_bytes + ))); + } + let normalized = normalized_sql(sql)?; + if allow_unsafe_sql { + return Ok(()); + } + let first = normalized.split_whitespace().next().unwrap_or_default(); + if matches!( + first, + "attach" + | "detach" + | "pragma" + | "vacuum" + | "begin" + | "commit" + | "rollback" + | "savepoint" + | "release" + ) { + return Err(VmError::HostError(format!( + "SQLite statement {first} is not allowed" + ))); + } + if normalized + .split(|character: char| !character.is_ascii_alphanumeric() && character != '_') + .any(|token| token == "load_extension") + { + return Err(VmError::HostError( + "SQLite extension loading is disabled".to_string(), + )); + } + Ok(()) +} + +fn sqlite_params(values: VmArrayRef<'_>, limits: SqliteLimits) -> VmResult> { + if values.len() > limits.max_parameters { + return Err(VmError::HostError( + "SQLite parameter count exceeds the configured limit".to_string(), + )); + } + let mut bytes = 0usize; + let mut params = Vec::with_capacity(values.len()); + for value in values { + let sql_value = match value { + Value::Null => SqlValue::Null, + Value::Int(value) => SqlValue::Integer(*value), + Value::Float(value) => SqlValue::Real(*value), + Value::String(value) => { + bytes = bytes.saturating_add(value.len()); + SqlValue::Text(value.as_ref().clone()) + } + Value::Bytes(value) => { + bytes = bytes.saturating_add(value.len()); + SqlValue::Blob(value.as_ref().clone()) + } + _ => { + return Err(VmError::HostError( + "SQLite parameters support only null, int, float, string, and bytes" + .to_string(), + )); + } + }; + if bytes > limits.max_parameter_bytes { + return Err(VmError::HostError(format!( + "SQLite parameters exceed the configured {} byte limit", + limits.max_parameter_bytes + ))); + } + params.push(sql_value); + } + Ok(params) +} + +fn cancellation_error(token: &CancellationToken) -> VmError { + let reason = token + .reason() + .unwrap_or(CancellationReason::Requested) + .as_str(); + VmError::HostError(format!("SQLite operation cancelled ({reason})")) +} + +fn with_connection( + slot: &ConnectionSlot, + token: &CancellationToken, + operation: impl FnOnce(&mut Connection) -> Result, +) -> VmResult { + token.check().map_err(runtime_error)?; + let mut connection = slot + .connection + .lock() + .map_err(|_| VmError::HostError("SQLite connection lock is poisoned".to_string()))?; + token.check().map_err(runtime_error)?; + let callback_token = token.clone(); + connection.progress_handler( + SQLITE_PROGRESS_STEPS, + Some(move || callback_token.is_cancelled()), + ); + let result = operation(&mut connection); + connection.progress_handler(0, None:: bool>); + if token.is_cancelled() { + return Err(cancellation_error(token)); + } + result.map_err(sqlite_error) +} + +fn sqlite_error(error: rusqlite::Error) -> VmError { + let code = error + .sqlite_error() + .map(|value| value.extended_code.to_string()) + .unwrap_or_else(|| "non_sqlite".to_string()); + let name = error + .sqlite_error_code() + .map(|value| format!("{value:?}")) + .unwrap_or_else(|| "RusqliteError".to_string()); + VmError::HostError(format!("SQLite error {name} ({code}): {error}")) +} + +fn estimate_value_bytes(value: &Value) -> usize { + match value { + Value::Null => 1, + Value::Int(_) | Value::Float(_) => 8, + Value::Bool(_) => 1, + Value::String(value) => value.len(), + Value::Bytes(value) => value.len(), + Value::Array(values) => values.iter().map(estimate_value_bytes).sum(), + Value::Map(values) => values + .iter() + .map(|(key, value)| { + estimate_value_bytes(key).saturating_add(estimate_value_bytes(value)) + }) + .sum(), + Value::Callable(_) => 8, + } +} + +fn value_from_row(row: &rusqlite::Row<'_>, index: usize) -> Result { + match row.get_ref(index)? { + ValueRef::Null => Ok(Value::Null), + ValueRef::Integer(value) => Ok(Value::Int(value)), + ValueRef::Real(value) => Ok(Value::Float(value)), + ValueRef::Text(value) => match std::str::from_utf8(value) { + Ok(value) => Ok(Value::string(value)), + Err(_) => Ok(Value::bytes(value.to_vec())), + }, + ValueRef::Blob(value) => Ok(Value::bytes(value.to_vec())), + } +} + +fn query_with_connection( + connection: &Connection, + sql: &str, + params: &[SqlValue], + limits: SqliteLimits, +) -> Result { + let mut statement = connection.prepare(sql)?; + let columns = statement + .column_names() + .into_iter() + .map(Value::string) + .collect::>(); + if columns.len() > limits.max_columns { + return Err(rusqlite::Error::InvalidColumnIndex(columns.len())); + } + let column_count = columns.len(); + let mut rows = statement.query(params_from_iter(params.iter()))?; + let mut values = Vec::new(); + let mut result_bytes = columns.iter().map(estimate_value_bytes).sum::(); + let mut truncated = false; + let mut next_cursor = None; + while let Some(row) = rows.next()? { + if values.len() >= limits.max_rows { + truncated = true; + break; + } + let mut cells = Vec::with_capacity(column_count); + let mut row_bytes = 0usize; + for index in 0..column_count { + let value = value_from_row(row, index)?; + row_bytes = row_bytes.saturating_add(estimate_value_bytes(&value)); + cells.push(value); + } + if result_bytes.saturating_add(row_bytes) > limits.max_result_bytes { + truncated = true; + break; + } + if let Some(Value::Int(cursor)) = cells.first() { + next_cursor = Some(*cursor); + } + result_bytes = result_bytes.saturating_add(row_bytes); + values.push(Value::array(cells)); + } + let mut entries = vec![ + (Value::string("columns"), Value::array(columns)), + (Value::string("rows"), Value::array(values)), + (Value::string("truncated"), Value::Bool(truncated)), + ]; + if let Some(next_cursor) = next_cursor { + entries.push((Value::string("next_cursor"), Value::Int(next_cursor))); + } + Ok(VmMap::from_entries(entries)) +} + +fn execute_with_connection( + connection: &Connection, + sql: &str, + params: &[SqlValue], +) -> Result { + let mut statement = connection.prepare(sql)?; + let rows_affected = statement.execute(params_from_iter(params.iter()))?; + drop(statement); + Ok(VmMap::from_entries(vec![ + ( + Value::string("rows_affected"), + Value::Int(i64::try_from(rows_affected).unwrap_or(i64::MAX)), + ), + ( + Value::string("last_insert_rowid"), + Value::Int(connection.last_insert_rowid()), + ), + ])) +} + +fn pending_count_for_resource(vm: &Vm, resource: ResourceHandle) -> usize { + vm.host + .runtime_operations + .operations_for_resource(resource) + .into_iter() + .filter(|operation| operation.owner() == OperationOwner::Sqlite) + .count() +} + +fn schedule_operation( + vm: &mut Vm, + resource: ResourceHandle, + slot: Arc, + operation: impl FnOnce(Arc, CancellationToken) -> VmResult + + Send + + 'static, +) -> VmResult { + if pending_count_for_resource(vm, resource) >= slot.limits.max_pending_operations { + return Err(VmError::HostError(format!( + "SQLite pending operation limit {} reached", + slot.limits.max_pending_operations + ))); + } + let deadline = + Instant::now().checked_add(Duration::from_millis(slot.limits.max_transaction_ms)); + let operation_state = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Sqlite, + Some(&vm.run_ctx.cancellation), + deadline, + None, + ) + .map_err(runtime_error)?; + let id = operation_state.id(); + let token = operation_state.token(); + let cleanup_slot = Arc::clone(&slot); + operation_state + .set_cleanup(Box::new(move |end| { + if matches!(end, super::cancellation::OperationEnd::Cancelled(_)) + && cleanup_slot + .active_operation + .lock() + .expect("SQLite active operation lock should not be poisoned") + .is_some_and(|active| active == id) + { + cleanup_slot.interrupt.interrupt(); + } + Ok(()) + })) + .map_err(runtime_error)?; + let worker_operation = operation_state.clone(); + let (sender, receiver) = mpsc::channel(); + let waker = Arc::new(Mutex::new(None::)); + let worker_waker = Arc::clone(&waker); + let worker = thread::Builder::new() + .name(format!("rustscript-sqlite-{}", id.raw())) + .spawn(move || { + let _execution = slot + .execution + .lock() + .expect("SQLite execution lock should not be poisoned"); + *slot + .active_operation + .lock() + .expect("SQLite active operation lock should not be poisoned") = Some(id); + let result = operation(Arc::clone(&slot), token); + *slot + .active_operation + .lock() + .expect("SQLite active operation lock should not be poisoned") = None; + match &result { + Ok(_) => { + let _ = worker_operation.complete(); + } + Err(error) => { + let _ = worker_operation.fail( + RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "sqlite::operation", + error.to_string(), + ) + .with_value(id.raw()), + ); + } + } + let _ = sender.send(result); + if let Ok(mut waker) = worker_waker.lock() + && let Some(waker) = waker.take() + { + waker.wake(); + } + }) + .map_err(|error| { + let _ = vm + .host + .runtime_operations + .cancel(id, CancellationReason::Requested); + VmError::HostError(format!("failed to start SQLite worker: {error}")) + })?; + let pending = PendingResult { + receiver, + worker: Some(worker), + waker, + }; + let payload = match vm.host.runtime_resources.insert_with_cleanup( + ResourceTypeId::CALLBACK, + pending, + |pending, _reason| { + wait_worker_bounded(pending); + Ok(()) + }, + ) { + Ok(payload) => payload, + Err(error) => { + let _ = vm + .host + .runtime_operations + .cancel(id, CancellationReason::ResourceClosed); + return Err(runtime_error(error)); + } + }; + operation_state.set_resource(resource); + operation_state.set_payload(payload); + Ok(id.raw()) +} + +fn wait_worker_bounded(mut pending: PendingResult) { + let deadline = Instant::now() + SQLITE_CLOSE_GRACE; + if let Some(worker) = pending.worker.take() { + while !worker.is_finished() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + if worker.is_finished() { + let _ = worker.join(); + } + } +} + +#[cfg(test)] +#[allow(dead_code)] +pub(super) fn active_operation_id(vm: &Vm, resource_id: i64) -> Option { + let handle = ResourceHandle::from_value(&Value::Int(resource_id)).ok()?; + let slot = vm + .host + .runtime_resources + .get::>(handle, ResourceTypeId::SQLITE_CONNECTION) + .ok()?; + let active = *slot + .active_operation + .lock() + .expect("SQLite active operation lock should not be poisoned"); + active.map(OperationId::raw) +} + +fn cancel_operation(vm: &mut Vm, id: OperationId, reason: CancellationReason) { + let Ok(operation) = vm.host.runtime_operations.get(id) else { + return; + }; + if operation.owner() != OperationOwner::Sqlite { + return; + } + super::cancel_runtime_operation(vm, id, reason); +} + +pub(super) fn poll_pending_op( + vm: &mut Vm, + op_id: HostOpId, + cx: &mut Context<'_>, +) -> Poll> { + let id = match operation_id(op_id) { + Ok(id) => id, + Err(error) => return Poll::Ready(Err(error)), + }; + let operation = match vm.host.runtime_operations.get(id) { + Ok(operation) if operation.owner() == OperationOwner::Sqlite => operation, + Ok(_) => { + return Poll::Ready(Err(VmError::HostError(format!( + "host operation {op_id} is not owned by SQLite" + )))); + } + Err(error) => return Poll::Ready(Err(runtime_error(error))), + }; + let Some(payload) = operation.payload() else { + return Poll::Ready(Err(VmError::HostError(format!( + "SQLite operation {op_id} has no completion payload" + )))); + }; + if operation.token().is_cancelled() { + let reason = operation + .token() + .reason() + .unwrap_or(CancellationReason::Requested); + let error = cancellation_error(&operation.token()); + cancel_operation(vm, id, reason); + return Poll::Ready(Err(error)); + } + let (received, worker) = { + let pending = match vm + .host + .runtime_resources + .get_mut::(payload, ResourceTypeId::CALLBACK) + { + Ok(pending) => pending, + Err(error) => return Poll::Ready(Err(runtime_error(error))), + }; + if let Ok(mut waker) = pending.waker.lock() { + *waker = Some(cx.waker().clone()); + } + let received = pending.receiver.try_recv(); + let worker = if matches!(received, Err(mpsc::TryRecvError::Empty)) { + None + } else { + pending.worker.take() + }; + (received, worker) + }; + if let Some(worker) = worker { + let _ = worker.join(); + } + match received { + Err(mpsc::TryRecvError::Empty) => { + if operation.token().is_cancelled() { + let reason = operation + .token() + .reason() + .unwrap_or(CancellationReason::Requested); + let error = cancellation_error(&operation.token()); + cancel_operation(vm, id, reason); + Poll::Ready(Err(error)) + } else { + Poll::Pending + } + } + Err(mpsc::TryRecvError::Disconnected) => { + let _ = super::close_runtime_resource(vm, payload, CancellationReason::ResourceClosed); + Poll::Ready(Err(VmError::HostError( + "SQLite worker ended without a result".to_string(), + ))) + } + Ok(result) => { + let _ = super::close_runtime_resource(vm, payload, CancellationReason::ResourceClosed); + match result { + Ok(value) => { + if let OperationStatus::Cancelled(_) = operation.status() { + Poll::Ready(Err(cancellation_error(&operation.token()))) + } else { + Poll::Ready(Ok(value)) + } + } + Err(error) => { + if operation.token().is_cancelled() { + Poll::Ready(Err(cancellation_error(&operation.token()))) + } else { + Poll::Ready(Err(error)) + } + } + } + } + } +} + +/// Opens a SQLite database under the embedding-owned path and limit policy. +#[pd_host_function(name = "sqlite::open")] +pub(super) fn builtin_sqlite_open_impl(vm: &mut Vm, options: VmMapRef<'_>) -> VmResult { + let options = parse_open_options(vm, options)?; + let open_count = vm + .host + .runtime_resources + .count_type(ResourceTypeId::SQLITE_CONNECTION); + if open_count >= options.limits.max_connections { + return Err(VmError::HostError(format!( + "SQLite connection limit {} reached", + options.limits.max_connections + ))); + } + let connection = open_connection(&options)?; + let interrupt = Arc::new(connection.get_interrupt_handle()); + let slot = Arc::new(ConnectionSlot { + connection: Mutex::new(connection), + execution: Mutex::new(()), + active_operation: Mutex::new(None), + interrupt: Arc::clone(&interrupt), + limits: options.limits, + allow_unsafe_sql: options.allow_unsafe_sql, + }); + let cleanup_interrupt = Arc::clone(&interrupt); + let handle = vm + .host + .runtime_resources + .insert_with_cleanup( + ResourceTypeId::SQLITE_CONNECTION, + slot, + move |_slot, _reason| { + cleanup_interrupt.interrupt(); + Ok(()) + }, + ) + .map_err(runtime_error)?; + Ok(handle_value(handle)) +} + +/// Executes one parameterized SQLite statement asynchronously. +#[pd_host_function(name = "sqlite::execute")] +pub(super) fn builtin_sqlite_execute_impl( + vm: &mut Vm, + db_id: i64, + sql: &str, + params: VmArrayRef<'_>, +) -> VmResult> { + let (resource, slot) = lookup_connection(vm, db_id)?; + validate_sql(sql, slot.limits, slot.allow_unsafe_sql)?; + let sql = sql.to_string(); + let params = sqlite_params(params, slot.limits)?; + let op_id = schedule_operation(vm, resource, slot, move |slot, token| { + with_connection(&slot, &token, |connection| { + execute_with_connection(connection, &sql, ¶ms) + }) + .map(|value| CallReturn::one(Value::Map(Arc::new(value)))) + })?; + Ok(HostCallResult::Pending(op_id)) +} + +/// Runs one parameterized SQLite query with row and result-byte bounds. +#[pd_host_function(name = "sqlite::query")] +pub(super) fn builtin_sqlite_query_impl( + vm: &mut Vm, + db_id: i64, + sql: &str, + params: VmArrayRef<'_>, + limits: VmMapRef<'_>, +) -> VmResult> { + let (resource, slot) = lookup_connection(vm, db_id)?; + let query_limits = parse_query_limits(limits, slot.limits)?; + validate_sql(sql, query_limits, slot.allow_unsafe_sql)?; + let sql = sql.to_string(); + let params = sqlite_params(params, slot.limits)?; + let op_id = schedule_operation(vm, resource, slot, move |slot, token| { + with_connection(&slot, &token, |connection| { + query_with_connection(connection, &sql, ¶ms, query_limits) + }) + .map(|value| CallReturn::one(Value::Map(Arc::new(value)))) + })?; + Ok(HostCallResult::Pending(op_id)) +} + +struct TransactionStatement { + sql: String, + params: Vec, + query: bool, + limits: SqliteLimits, +} + +fn parse_transaction_statements( + statements: VmArrayRef<'_>, + limits: SqliteLimits, + allow_unsafe_sql: bool, +) -> VmResult> { + if statements.is_empty() { + return Err(VmError::HostError( + "SQLite transaction requires at least one statement".to_string(), + )); + } + if statements.len() > limits.max_statements { + return Err(VmError::HostError(format!( + "SQLite transaction exceeds the configured {} statement limit", + limits.max_statements + ))); + } + statements + .iter() + .map(|statement| { + let Value::Map(statement) = statement else { + return Err(VmError::TypeMismatch("SQLite transaction statement map")); + }; + let sql = required_string(statement, "sql")?; + validate_sql(&sql, limits, allow_unsafe_sql)?; + let params = match map_value(statement, "params") { + Some(Value::Array(params)) => sqlite_params(params, limits)?, + Some(_) => return Err(VmError::TypeMismatch("SQLite parameter array")), + None => Vec::new(), + }; + let query = match map_value(statement, "query") { + Some(Value::Bool(query)) => *query, + Some(_) => return Err(VmError::TypeMismatch("SQLite query flag")), + None => false, + }; + let statement_limits = match map_value(statement, "limits") { + Some(Value::Map(statement_limits)) => parse_query_limits(statement_limits, limits)?, + Some(_) => return Err(VmError::TypeMismatch("SQLite limits map")), + None => limits, + }; + Ok(TransactionStatement { + sql, + params, + query, + limits: statement_limits, + }) + }) + .collect() +} + +/// Executes ordered statements atomically and returns ordered result envelopes. +#[pd_host_function(name = "sqlite::transaction")] +pub(super) fn builtin_sqlite_transaction_impl( + vm: &mut Vm, + db_id: i64, + statements: VmArrayRef<'_>, +) -> VmResult>> { + let (resource, slot) = lookup_connection(vm, db_id)?; + let statements = parse_transaction_statements(statements, slot.limits, slot.allow_unsafe_sql)?; + let op_id = schedule_operation(vm, resource, slot, move |slot, token| { + with_connection(&slot, &token, |connection| { + let transaction = + connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let mut results = Vec::with_capacity(statements.len()); + for statement in statements { + let value = if statement.query { + query_with_connection( + &transaction, + &statement.sql, + &statement.params, + statement.limits, + )? + } else { + execute_with_connection(&transaction, &statement.sql, &statement.params)? + }; + results.push(Value::Map(Arc::new(value))); + } + transaction.commit()?; + Ok(results) + }) + .map(|values| CallReturn::one(Value::array(values))) + })?; + Ok(HostCallResult::Pending(op_id)) +} + +/// Closes a SQLite resource and cancels operations using it. +#[pd_host_function(name = "sqlite::close")] +pub(super) fn builtin_sqlite_close_impl(vm: &mut Vm, db_id: i64) -> VmResult<()> { + let handle = sqlite_handle(db_id)?; + super::close_runtime_resource(vm, handle, CancellationReason::ResourceClosed) + .map_err(|error| VmError::HostError(format!("unknown SQLite database: {error}")))?; + Ok(()) +} diff --git a/src/builtins/runtime/typed.rs b/src/builtins/runtime/typed.rs index 55612e0e..cf5d5f03 100644 --- a/src/builtins/runtime/typed.rs +++ b/src/builtins/runtime/typed.rs @@ -459,7 +459,7 @@ where { fn into_builtin_call_outcome(self) -> BuiltinCallOutcome { match self { - Self::Return(value) => value.into_builtin_call_outcome(), + Self::Return(value) => BuiltinCallOutcome::Return(return_one(value)), Self::Pending(op_id) => BuiltinCallOutcome::Pending(op_id), } } diff --git a/src/bytecode.rs b/src/bytecode.rs index 23d12a71..95551dff 100644 --- a/src/bytecode.rs +++ b/src/bytecode.rs @@ -80,6 +80,7 @@ pub struct ExportedCallable { #[derive(Debug)] pub struct CallableEnvironment { + #[cfg_attr(not(feature = "runtime"), allow(dead_code))] pub(crate) cells: std::sync::Mutex>, } diff --git a/src/cli.rs b/src/cli.rs index a02f99c1..45684d7f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -332,16 +332,51 @@ fn run_vm_loop( fn render_source_path_error(source_path: &Path, err: &SourcePathError) -> String { match err { - SourcePathError::Source(vm::SourceError::Parse(parse)) => { - let source = std::fs::read_to_string(source_path).unwrap_or_default(); + SourcePathError::SourceWithMap { .. } => { + vm::render_source_path_error(source_path, err, true) + } + SourcePathError::Source(error) => render_source_error_at_path(source_path, None, error), + SourcePathError::InvalidImportSyntax { + path, + line, + message, + } => { + let source = std::fs::read_to_string(path).unwrap_or_default(); let mut source_map = SourceMap::new(); - let source_id = source_map.add_source(source_path.display().to_string(), source); + let source_id = source_map.add_source(path.display().to_string(), source); + let parse = vm::ParseError::at_line(*line, message.clone()) + .with_line_span_from_source(&source_map, source_id); + render_source_error(&source_map, &parse, true) + } + _ => err.to_string(), + } +} + +fn render_source_error_at_path( + source_path: &Path, + source_override: Option<&str>, + error: &vm::SourceError, +) -> String { + match error { + vm::SourceError::Parse(parse) => { + let render_path = parse + .message + .split_once(": ") + .map(|(path, _)| Path::new(path)) + .filter(|path| path.exists()) + .unwrap_or(source_path); + let source = source_override + .filter(|_| render_path == source_path) + .map(str::to_owned) + .unwrap_or_else(|| std::fs::read_to_string(render_path).unwrap_or_default()); + let mut source_map = SourceMap::new(); + let source_id = source_map.add_source(render_path.display().to_string(), source); let parse = parse .clone() .with_line_span_from_source(&source_map, source_id); render_source_error(&source_map, &parse, true) } - SourcePathError::Source(vm::SourceError::Compile(compile)) => { + vm::SourceError::Compile(compile) => { let render_path = compile .source_name() .map(Path::new) @@ -352,19 +387,6 @@ fn render_source_path_error(source_path: &Path, err: &SourcePathError) -> String source_map.add_source(render_path.display().to_string(), source); vm::render_compile_error(&source_map, compile, true) } - SourcePathError::InvalidImportSyntax { - path, - line, - message, - } => { - let source = std::fs::read_to_string(path).unwrap_or_default(); - let mut source_map = SourceMap::new(); - let source_id = source_map.add_source(path.display().to_string(), source); - let parse = vm::ParseError::at_line(*line, message.clone()) - .with_line_span_from_source(&source_map, source_id); - render_source_error(&source_map, &parse, true) - } - _ => err.to_string(), } } @@ -2372,4 +2394,117 @@ mod tests { fn repl_input_incomplete_for_trailing_operator() { assert!(!super::is_repl_input_complete("let a = 1 +")); } + + fn cli_diagnostic_root(prefix: &str) -> std::path::PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("cli diagnostic root should be created"); + root.canonicalize().unwrap_or(root) + } + + #[test] + fn cli_nested_module_parse_error_renders_nested_source_frame() { + let root = cli_diagnostic_root("pd-vm-cli-nested-parse-diag"); + let main_path = root.join("main.rss"); + let nested_path = root.join("nested.rss"); + std::fs::write(&main_path, "use self::nested as nested;\nnested::run();\n") + .expect("main fixture should write"); + std::fs::write(&nested_path, "pub fn run( {\n").expect("nested fixture should write"); + + let error = match vm::compile_source_file_with_options( + &main_path, + vm::CompileSourceFileOptions::default(), + ) { + Ok(_) => panic!("nested parse error fixture should fail"), + Err(error) => error, + }; + let rendered = super::render_source_path_error(&main_path, &error); + + // The rendered frame must belong to the nested source: its path, its + // line text, and an underline, not the root file. + assert!( + rendered.contains(&nested_path.display().to_string()), + "rendered diagnostic should name the nested path: {rendered}" + ); + assert!( + rendered.contains("pub fn run( {"), + "rendered diagnostic should show the nested source line: {rendered}" + ); + assert!( + rendered.contains('^'), + "rendered diagnostic should underline the nested source: {rendered}" + ); + assert!( + !rendered.contains("use self::nested as nested;"), + "rendered diagnostic should not show the root source frame: {rendered}" + ); + + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn cli_nested_strict_type_error_renders_nested_source_frame() { + let root = cli_diagnostic_root("pd-vm-cli-nested-strict-diag"); + let main_path = root.join("main.rss"); + let nested_path = root.join("nested.rss"); + std::fs::write(&main_path, "use self::nested as nested;\nnested::run();\n") + .expect("main fixture should write"); + std::fs::write(&nested_path, "pub fn run() -> unknown { 1 }\n") + .expect("nested fixture should write"); + + let error = match vm::compile_source_file_with_options( + &main_path, + vm::CompileSourceFileOptions::default(), + ) { + Ok(_) => panic!("strict nested fixture should fail"), + Err(error) => error, + }; + let rendered = super::render_source_path_error(&main_path, &error); + + assert!( + rendered.contains(&nested_path.display().to_string()), + "rendered diagnostic should name the nested path: {rendered}" + ); + assert!( + rendered.contains("pub fn run() -> unknown { 1 }"), + "rendered diagnostic should show the nested source line: {rendered}" + ); + assert!( + rendered.contains('^'), + "rendered diagnostic should underline the nested source: {rendered}" + ); + + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn render_source_error_uses_source_override_for_virtual_paths() { + let virtual_path = std::path::Path::new("__pd_vm_inmemory__/main.rss"); + let error = vm::SourceError::Parse(vm::ParseError::at_line(2, "boom")); + let rendered = super::render_source_error_at_path( + virtual_path, + Some("line one\nline two target\nline three"), + &error, + ); + + assert!( + rendered.contains("__pd_vm_inmemory__/main.rss"), + "rendered diagnostic should name the virtual path: {rendered}" + ); + assert!( + rendered.contains("line two target"), + "rendered diagnostic should show the override source line: {rendered}" + ); + assert!( + rendered.contains('^'), + "rendered diagnostic should underline the override source: {rendered}" + ); + } } diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 5b9b43f3..3b3fe564 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -668,6 +668,14 @@ impl Compiler { let slot = self.ensure_function_value_slot(*index, type_args)?; self.emit_copy_ldloc(slot)?; } + // Resolved module targets are lowered into plain flat-index calls + // by `linker::merge_units`; reaching codegen means the merge + // missed a site. + Expr::ModuleFunctionRef(..) + | Expr::ModuleCall(..) + | Expr::UnresolvedFunctionRef { .. } => { + return Err(CompileError::UnresolvedModuleCall); + } Expr::Call(index, _, args) => { self.compile_function_call(*index, args)?; } diff --git a/src/compiler/diagnostics.rs b/src/compiler/diagnostics.rs index f7a27517..832b0bb1 100644 --- a/src/compiler/diagnostics.rs +++ b/src/compiler/diagnostics.rs @@ -1,5 +1,5 @@ use super::source_map::{SourceMap, Span}; -use super::{CompileError, ParseError}; +use super::{CompileError, ParseError, SourceError, SourcePathError}; pub fn render_source_error(source_map: &SourceMap, err: &ParseError, _styled: bool) -> String { let code_prefix = err @@ -19,12 +19,21 @@ pub fn render_source_error(source_map: &SourceMap, err: &ParseError, _styled: bo pub fn render_compile_error(source_map: &SourceMap, err: &CompileError, _styled: bool) -> String { let message = err.diagnostic_message(); - let source_id = err - .source_name() - .and_then(|name| source_map.source_id_by_name(name)) - .unwrap_or(0); + let source_name = err.source_name(); + + // Resolve the owning source id: by name when the error names its source, + // or as the single-file fallback used by inline/REPL compiles (a map with + // exactly one file at id 0). A named error is never rendered against + // another file: when its source is missing from the map it renders as a + // plain path/line message instead of misattributing the span. + let source_id = match source_name { + Some(name) => source_map.source_id_by_name(name), + None if source_map.file(0).is_some() && source_map.file(1).is_none() => Some(0), + None => None, + }; if let Some(line) = err.line() + && let Some(source_id) = source_id && let Some(span) = source_map.line_span(source_id, line) && let Some(rendered) = render_span_snippet(source_map, span, &message) { @@ -32,7 +41,7 @@ pub fn render_compile_error(source_map: &SourceMap, err: &CompileError, _styled: } if let Some(line) = err.line() { - if let Some(source_name) = err.source_name() { + if let Some(source_name) = source_name { return format!("compile error: {source_name}:{line}: {message}"); } return format!("compile error: line {line}: {message}"); @@ -41,6 +50,64 @@ pub fn render_compile_error(source_map: &SourceMap, err: &CompileError, _styled: format!("compile error: {message}") } +/// Render a source error (parse or compile) against the compilation-wide +/// source map carried by a [`SourcePathError`] when present, falling back to +/// a map-less render otherwise. Parse errors whose span references a source +/// id outside the map keep their path-prefixed message. +pub fn render_source_path_error( + source_path: &std::path::Path, + err: &SourcePathError, + _styled: bool, +) -> String { + match err { + SourcePathError::SourceWithMap { error, sources } => match error { + SourceError::Parse(parse) => render_source_error(sources, parse, _styled), + SourceError::Compile(compile) => render_compile_error(sources, compile, _styled), + }, + SourcePathError::Source(error) => match error { + SourceError::Parse(parse) => { + let render_path = parse + .message + .split_once(": ") + .map(|(path, _)| std::path::Path::new(path)) + .filter(|path| path.exists()) + .unwrap_or(source_path); + let source = std::fs::read_to_string(render_path).unwrap_or_default(); + let mut source_map = SourceMap::new(); + let source_id = source_map.add_source(render_path.display().to_string(), source); + let parse = parse + .clone() + .with_line_span_from_source(&source_map, source_id); + render_source_error(&source_map, &parse, _styled) + } + SourceError::Compile(compile) => { + let render_path = compile + .source_name() + .map(std::path::Path::new) + .filter(|path| path.exists()) + .unwrap_or(source_path); + let source = std::fs::read_to_string(render_path).unwrap_or_default(); + let mut source_map = SourceMap::new(); + source_map.add_source(render_path.display().to_string(), source); + render_compile_error(&source_map, compile, _styled) + } + }, + SourcePathError::InvalidImportSyntax { + path, + line, + message, + } => { + let source = std::fs::read_to_string(path).unwrap_or_default(); + let mut source_map = SourceMap::new(); + let source_id = source_map.add_source(path.display().to_string(), source); + let parse = ParseError::at_line(*line, message.clone()) + .with_line_span_from_source(&source_map, source_id); + render_source_error(&source_map, &parse, _styled) + } + _ => err.to_string(), + } +} + fn render_span_snippet(source_map: &SourceMap, span: Span, message: &str) -> Option { let file = source_map.file(span.source_id)?; let (line, col) = file.line_col_for_offset(span.lo)?; diff --git a/src/compiler/frontends/mod.rs b/src/compiler/frontends/mod.rs index 45a0380b..af0ed9c8 100644 --- a/src/compiler/frontends/mod.rs +++ b/src/compiler/frontends/mod.rs @@ -20,11 +20,62 @@ pub(super) fn parse_source( source: &str, flavor: SourceFlavor, options: &CompileSourceFileOptions, +) -> Result { + parse_source_with_source_id(source, flavor, options, 0) +} + +/// Parse `source` and attribute every produced span to `original_source_id`. +/// +/// The id belongs to the compilation-wide [`SourceMap`] built by the source +/// loader, whose ids are the semantic module graph's +/// [`SourceId`](crate::compiler::modules::SourceId) space. Spans produced by +/// this parse (including the error span on failure) therefore stay owned by +/// the module's source after unit merge. The default id `0` preserves the +/// legacy single-source behavior for entry points that build their own map. +pub(super) fn parse_source_with_source_id( + source: &str, + flavor: SourceFlavor, + options: &CompileSourceFileOptions, + original_source_id: u32, +) -> Result { + parse_source_with_source_id_and_externs(source, flavor, options, original_source_id, false) +} + +/// Parse one module's source for the source loader (module mode). +/// +/// Module-mode parses enable the parser's implicit-extern fallback so that +/// calls to imported module functions and module namespace members parse +/// before the loader resolves them by [`SymbolId`](crate::compiler::modules::SymbolId). +/// The produced IR carries the implicit-extern names on +/// [`FrontendIr::implicit_extern_names`] so the loader keeps those synthetic +/// declarations out of module declaration/export tables. +pub(super) fn parse_module_source_with_source_id( + source: &str, + flavor: SourceFlavor, + options: &CompileSourceFileOptions, + original_source_id: u32, +) -> Result { + parse_source_with_source_id_and_externs(source, flavor, options, original_source_id, true) +} + +fn parse_source_with_source_id_and_externs( + source: &str, + flavor: SourceFlavor, + options: &CompileSourceFileOptions, + original_source_id: u32, + allow_implicit_externs: bool, ) -> Result { match flavor { SourceFlavor::RustScript => { let lowered = rustscript::lower(source)?; - parse_lowered_with_mapping(source, lowered, false, false, true) + parse_lowered_with_mapping( + source, + lowered, + allow_implicit_externs, + false, + true, + original_source_id, + ) } SourceFlavor::JavaScript | SourceFlavor::Lua => { let Some(plugin) = options.source_plugin_for_flavor(flavor) else { @@ -60,6 +111,7 @@ pub fn parse_source_with_dialect( options.allow_implicit_externs, options.allow_implicit_semicolons, options.enforce_mutable_bindings, + options.import_scan_mode, dialect, ) } @@ -86,6 +138,7 @@ fn parse_with_parser( allow_implicit_externs: bool, allow_implicit_semicolons: bool, enforce_mutable_bindings: bool, + import_scan_mode: bool, dialect: &'static dyn ParserDialect, ) -> Result { let mut parser = Parser::new( @@ -94,6 +147,7 @@ fn parse_with_parser( allow_implicit_externs, allow_implicit_semicolons, enforce_mutable_bindings, + import_scan_mode, dialect, )?; let stmts = parser.parse_program()?; @@ -107,6 +161,8 @@ fn parse_with_parser( function_impls: parser.function_impls(), stmt_sources: Vec::new(), function_sources: HashMap::new(), + use_declarations: parser.use_declarations(), + implicit_extern_names: parser.implicit_extern_names(), }) } @@ -142,6 +198,8 @@ fn parse_repl_with_parser( function_impls: parser.function_impls(), stmt_sources: Vec::new(), function_sources: HashMap::new(), + use_declarations: parser.use_declarations(), + implicit_extern_names: parser.implicit_extern_names(), }, bindings, }) @@ -153,9 +211,10 @@ fn parse_lowered_with_mapping( allow_implicit_externs: bool, allow_implicit_semicolons: bool, enforce_mutable_bindings: bool, + original_source_id: u32, ) -> Result { let mut source_map = SourceMap::new(); - let original_source_id = source_map.add_source("", original_source.to_string()); + source_map.add_source_at(original_source_id, "", original_source.to_string()); let lowered_source_id = source_map.add_source("", lowered.text.clone()); match parse_with_parser( @@ -164,6 +223,7 @@ fn parse_lowered_with_mapping( allow_implicit_externs, allow_implicit_semicolons, enforce_mutable_bindings, + false, rustscript::parser_dialect(), ) { Ok(mut ir) => { diff --git a/src/compiler/ir.rs b/src/compiler/ir.rs index e17494a5..a0f8388b 100644 --- a/src/compiler/ir.rs +++ b/src/compiler/ir.rs @@ -4,6 +4,7 @@ use crate::ValueType; use crate::builtins::default_host_callable; use super::ParseError; +use super::modules::SymbolId; pub type LocalSlot = u16; @@ -186,6 +187,25 @@ pub enum Expr { String(String), Bytes(Vec), FunctionRef(u16, Vec), + /// A function value whose target was resolved to a compiler-owned module + /// symbol before unit merge (milestone 4). + /// + /// Produced by the source loader's resolution pass for imported function + /// values and lowered by `linker::merge_units` into a plain + /// [`Expr::FunctionRef`] against the merged flat function table. + ModuleFunctionRef(SymbolId, Vec), + /// A function value reference whose target is not yet resolved (module + /// mode only). + /// + /// Produced by the parser in module mode when a function value refers to + /// a name the parser cannot resolve locally (an imported function binding + /// whose export table only the source loader knows). The loader's + /// resolution pass maps it to [`Expr::ModuleFunctionRef`] before unit + /// merge, so downstream passes never observe it. + UnresolvedFunctionRef { + name: String, + type_args: Vec, + }, OptionalGet { container: Box, key: Box, @@ -198,6 +218,17 @@ pub enum Expr { fallback: Box, }, Call(u16, Vec, Vec), + /// A call whose target was resolved to a compiler-owned module symbol + /// before unit merge (milestone 4). + /// + /// The source loader's resolution pass rewrites calls to imported + /// functions into this form, carrying the [`SymbolId`] of the source + /// module's declaration; `linker::merge_units` lowers it back into a + /// plain [`Expr::Call`] against the merged flat function table. Unlike + /// [`Expr::Call`]'s flat index, the symbol identity never depends on + /// unit-local index assignment or on the source name, so same-named + /// declarations in independent modules resolve to distinct targets. + ModuleCall(SymbolId, Vec, Vec), LocalCall(LocalSlot, Vec, Vec), Closure(ClosureExpr), ClosureCall(ClosureExpr, Vec), @@ -342,6 +373,10 @@ pub struct FunctionDecl { pub type_params: Vec, pub exported: bool, pub return_type: ValueType, + /// Semantic symbol owned by the declaring module, assigned by the source + /// loader after parse (milestone 3). `None` for IR that has not been + /// attached to a module yet (parser output, REPL snippets). + pub symbol: Option, } #[derive(Clone, Debug)] @@ -364,6 +399,18 @@ pub struct FrontendIr { pub function_impls: HashMap, pub stmt_sources: Vec>, pub function_sources: HashMap, + /// Structured `use` directives parsed from RustScript source, with spans + /// and clauses. Consumed by the source loader for import discovery. + pub use_declarations: Vec, + /// Names created by the parser's implicit-extern fallback (module mode). + /// + /// Module-mode parses tolerate calls whose target only the source loader + /// can resolve (imported module functions, module namespace members). + /// These synthetic declarations must never receive a module symbol or a + /// flat entry; the loader resolves their call sites or rejects them. + /// Plain (non-module) parses leave this empty because implicit externs are + /// disabled there. + pub implicit_extern_names: Vec, } pub struct LocalIrBuilder { @@ -475,6 +522,7 @@ impl LocalIrBuilder { type_params: Vec::new(), exported: false, return_type: ValueType::Unknown, + symbol: None, }); self.function_meta.insert(name.to_string(), (index, arity)); Ok(()) @@ -521,6 +569,8 @@ impl LocalIrBuilder { function_impls: HashMap::new(), stmt_sources: Vec::new(), function_sources: HashMap::new(), + use_declarations: Vec::new(), + implicit_extern_names: Vec::new(), } } diff --git a/src/compiler/lifetime/availability.rs b/src/compiler/lifetime/availability.rs index e9f977bf..10c5562c 100644 --- a/src/compiler/lifetime/availability.rs +++ b/src/compiler/lifetime/availability.rs @@ -665,7 +665,9 @@ impl AvailabilityAnalyzer { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => Ok(state.clone()), + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => Ok(state.clone()), Expr::Var(index) => { self.require_available(*index, state, line)?; self.require_local_not_moved(*index, state, line)?; @@ -720,6 +722,9 @@ impl AvailabilityAnalyzer { let then_state = self.analyze_expr(fallback, &value_state, line)?; Ok(self.merge_states(then_state, value_state)) } + // Resolved module calls (pre-merge only) analyze their arguments; + // interprocedural effects apply to the post-merge flat call. + Expr::ModuleCall(_, _, args) => self.analyze_args(args, state, line), Expr::Call(index, _, args) => { if !self.enable_local_move_semantics { if let Some(root_slot) = self.extract_collection_mutation_root(*index, args) { diff --git a/src/compiler/lifetime/availability/captures.rs b/src/compiler/lifetime/availability/captures.rs index f234fc17..82cb1774 100644 --- a/src/compiler/lifetime/availability/captures.rs +++ b/src/compiler/lifetime/availability/captures.rs @@ -329,7 +329,9 @@ impl AvailabilityAnalyzer { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) => { if *index == captured_slot { *seen = true; @@ -358,7 +360,7 @@ impl AvailabilityAnalyzer { self.capture_mode_for_expr(value, captured_slot, context, mode, seen); self.capture_mode_for_expr(fallback, captured_slot, context, mode, seen); } - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => { for arg in args { self.capture_mode_for_expr(arg, captured_slot, context, mode, seen); } diff --git a/src/compiler/lifetime/availability/consumption.rs b/src/compiler/lifetime/availability/consumption.rs index f410291b..c37e1226 100644 --- a/src/compiler/lifetime/availability/consumption.rs +++ b/src/compiler/lifetime/availability/consumption.rs @@ -171,7 +171,9 @@ pub(super) fn expr_uses_slot(expr: &Expr, slot: LocalSlot) -> bool { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => false, + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => false, Expr::Var(index) | Expr::MoveVar(index) => *index == slot, Expr::MoveField { root, .. } | Expr::MoveIndex { root, .. } => *root == slot, Expr::OptionalGet { @@ -190,7 +192,7 @@ pub(super) fn expr_uses_slot(expr: &Expr, slot: LocalSlot) -> bool { value_slot, fallback, } => *value_slot == slot || expr_uses_slot(value, slot) || expr_uses_slot(fallback, slot), - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => { args.iter().any(|arg| expr_uses_slot(arg, slot)) } Expr::Closure(closure) => { @@ -376,6 +378,8 @@ pub(super) fn collect_consumed_positions_from_expr( | Expr::Bytes(_) | Expr::String(_) | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } | Expr::Var(_) => {} Expr::MoveVar(slot) => { if let Some(position) = function_impl @@ -459,6 +463,18 @@ pub(super) fn collect_consumed_positions_from_expr( } } } + // Resolved module calls (pre-merge only) have no per-unit consumed + // position table; their arguments are still scanned. + Expr::ModuleCall(_, _, args) => { + for arg in args { + collect_consumed_positions_from_expr( + arg, + function_impl, + known_consumed_positions, + out, + ); + } + } Expr::LocalCall(_, _, args) => { for arg in args { collect_consumed_positions_from_expr( diff --git a/src/compiler/lifetime/liveness.rs b/src/compiler/lifetime/liveness.rs index 175437bc..42f5fed8 100644 --- a/src/compiler/lifetime/liveness.rs +++ b/src/compiler/lifetime/liveness.rs @@ -437,7 +437,9 @@ impl LivenessRewriter { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) | Expr::MoveVar(index) => self.mark_live(live, *index), Expr::MoveField { root, .. } | Expr::MoveIndex { root, .. } => { self.mark_live(live, *root) @@ -472,6 +474,14 @@ impl LivenessRewriter { self.union_inplace(live, &footprint); } } + // Resolved module calls (pre-merge only) contribute their + // arguments' uses; the callee lives in another unit and its + // footprint is folded in by the post-merge call lowering. + Expr::ModuleCall(_, _, args) => { + for arg in args { + self.add_expr_uses(arg, live); + } + } Expr::LocalCall(index, _, args) => { self.mark_live(live, *index); for arg in args { @@ -736,7 +746,9 @@ impl LivenessRewriter { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) | Expr::MoveVar(index) | Expr::LocalCall(index, _, _) => { self.mark_live(footprint, *index); } @@ -770,6 +782,14 @@ impl LivenessRewriter { self.collect_expr_footprint(arg, footprint, stack); } } + // Resolved module calls (pre-merge only) contribute their + // arguments' footprint; the callee lives in another unit and is + // folded in by the post-merge call lowering. + Expr::ModuleCall(_, _, args) => { + for arg in args { + self.collect_expr_footprint(arg, footprint, stack); + } + } Expr::Closure(closure) => { for slot in &closure.param_slots { self.mark_live(footprint, *slot); @@ -898,6 +918,8 @@ fn expr_contains_local_call(expr: &Expr) -> bool { | Expr::Bytes(_) | Expr::String(_) | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } | Expr::Var(_) | Expr::MoveVar(_) | Expr::MoveField { .. } @@ -908,7 +930,9 @@ fn expr_contains_local_call(expr: &Expr) -> bool { Expr::OptionUnwrapOr { value, fallback, .. } => expr_contains_local_call(value) || expr_contains_local_call(fallback), - Expr::Call(_, _, args) => args.iter().any(expr_contains_local_call), + Expr::Call(_, _, args) | Expr::ModuleCall(_, _, args) => { + args.iter().any(expr_contains_local_call) + } Expr::Closure(closure) => expr_contains_local_call(&closure.body), Expr::ClosureCall(closure, args) => { args.iter().any(expr_contains_local_call) || expr_contains_local_call(&closure.body) @@ -1133,7 +1157,9 @@ impl LocalSlotAllocator { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) | Expr::MoveVar(index) => { self.add_slot_live_edges(*index, &live_during); } @@ -1170,6 +1196,13 @@ impl LocalSlotAllocator { self.add_cross_live_with_set(&live_during, &footprint); } } + // Resolved module calls (pre-merge only) constrain their + // arguments; the callee's footprint is folded in post-merge. + Expr::ModuleCall(_, _, args) => { + for arg in args { + self.collect_expr_constraints(arg, &live_during)?; + } + } Expr::LocalCall(index, _, args) => { self.add_slot_live_edges(*index, &live_during); for arg in args { @@ -1357,7 +1390,9 @@ impl LocalSlotAllocator { | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) | Expr::MoveVar(index) | Expr::LocalCall(index, _, _) => { self.mark_set_slot(set, *index) } @@ -1397,6 +1432,11 @@ impl LocalSlotAllocator { self.collect_expr_footprint(arg, set, stack); } } + Expr::ModuleCall(_, _, args) => { + for arg in args { + self.collect_expr_footprint(arg, set, stack); + } + } Expr::Closure(closure) => { for (source_slot, captured_slot) in &closure.capture_copies { self.mark_set_slot(set, *source_slot); @@ -1745,6 +1785,8 @@ fn collect_persistent_closure_sources_from_expr(expr: &Expr, slots: &mut BTreeSe | Expr::String(_) | Expr::Bytes(_) | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } | Expr::Var(_) | Expr::MoveVar(_) | Expr::MoveField { .. } @@ -1759,7 +1801,7 @@ fn collect_persistent_closure_sources_from_expr(expr: &Expr, slots: &mut BTreeSe collect_persistent_closure_sources_from_expr(value, slots); collect_persistent_closure_sources_from_expr(fallback, slots); } - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => { for arg in args { collect_persistent_closure_sources_from_expr(arg, slots); } @@ -1897,8 +1939,10 @@ fn remap_expr_slots(expr: &mut Expr, mapping: &[LocalSlot]) -> Result<(), ParseE | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) => {} - Expr::FunctionRef(..) => {} - Expr::Call(_, _, args) => { + Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} + Expr::Call(_, _, args) | Expr::ModuleCall(_, _, args) => { for arg in args { remap_expr_slots(arg, mapping)?; } diff --git a/src/compiler/linker.rs b/src/compiler/linker.rs index 677840b6..cdef2c29 100644 --- a/src/compiler/linker.rs +++ b/src/compiler/linker.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use crate::builtins::BuiltinFunction; @@ -6,18 +6,42 @@ use crate::builtins::BuiltinFunction; use super::{ ParseError, SourceError, SourcePathError, ir::{Expr, FrontendIr, FunctionDecl, FunctionImpl, LocalSlot, Stmt, StructDecl}, + modules::{ModuleId, SymbolId}, }; pub(super) struct ParsedUnit { pub(super) parsed: FrontendIr, - pub(super) scope_prefix: Option, + /// Deterministic scope identity for the unit's local bindings at the flat + /// bytecode boundary. `None` for the root unit (which keeps bare names); + /// otherwise a mangled full-module identity computed by the loader, never + /// a bare file stem (milestone 4). + pub(super) scope_identity: Option, pub(super) source_name: String, + /// Semantic module identity assigned by the module graph during discovery. + /// Consumed by milestone 4+ symbol resolution; carried on the unit so the + /// link between parsed IR and graph node survives the merge pipeline. + /// (The loader resolves call sites with it before merge; the flat merge + /// itself keys on [`SymbolId`].) + #[allow(dead_code)] + pub(super) module: ModuleId, + /// Graph `SourceId` of the unit's module (milestone 5). Every span the + /// unit's IR carries references this id in the compilation-wide + /// [`SourceMap`](crate::compiler::source_map::SourceMap), so merged + /// diagnostics always render from the owning source. + #[allow(dead_code)] + pub(super) source_id: u32, } -pub(super) fn sanitize_scope_prefix(path: &Path) -> String { - path.file_stem() - .and_then(|value| value.to_str()) - .unwrap_or("module") +/// Deterministic flat-boundary scope identity for a non-root module's local +/// bindings. +/// +/// Encodes the full canonical module identity (never a bare file stem) plus +/// the compiler-owned [`ModuleId`], so same-stem modules in different +/// directories and same-named locals across independent modules never collide +/// at the flat boundary. +pub(super) fn module_scope_prefix(identity: &Path, module: ModuleId) -> String { + let sanitized: String = identity + .to_string_lossy() .chars() .map(|ch| { if ch.is_ascii_alphanumeric() || ch == '_' { @@ -26,7 +50,16 @@ pub(super) fn sanitize_scope_prefix(path: &Path) -> String { '_' } }) - .collect() + .collect(); + format!("{sanitized}__m{}", module.0) +} + +/// Deterministic flat name for a module function whose source name is already +/// claimed by another flat entry. The mangling encodes the compiler-owned +/// module identity, so it is stable across compilations and never depends on +/// discovery-order-dependent string synthesis. +fn deterministic_flat_name(name: &str, symbol: SymbolId) -> String { + format!("{}__m{}", name, symbol.module.0) } pub(super) fn merge_units(units: Vec) -> Result { @@ -38,22 +71,39 @@ pub(super) fn merge_units(units: Vec) -> Result::new(); let mut merged_function_sources = HashMap::::new(); - let mut function_index_by_name = HashMap::::new(); + + // Milestone 4 flat identity maps. + // + // Module functions (declarations with implementations) are merged by + // compiler-owned `SymbolId`, so same-named declarations in independent + // modules each get their own flat entry. Host imports (declarations + // without implementations) keep name-keyed deduplication: their names are + // the runtime binding surface (`program.imports`, `Vm::bind_function`), + // so the legacy merge semantics apply verbatim. + let mut flat_index_by_symbol = HashMap::::new(); + let mut host_index_by_name = HashMap::::new(); + // Every flat name claimed so far. Module functions that collide are + // deterministically mangled with their module identity; host imports are + // deduplicated by name before ever reaching this set. + let mut claimed_flat_names = HashSet::::new(); + let mut local_base = 0usize; for unit in units { let source_name = unit.source_name.clone(); - let function_map = remap_functions( - &unit.parsed.functions, + let function_map = register_unit_functions( + &unit, &mut merged_functions, - &mut function_index_by_name, + &mut flat_index_by_symbol, + &mut host_index_by_name, + &mut claimed_flat_names, )?; let unit_local_base = local_base; let unit_local_count = unit.parsed.locals; let mut remapped_stmts = unit.parsed.stmts; for stmt in &mut remapped_stmts { - remap_stmt_indices(stmt, unit_local_base, &function_map)?; + remap_stmt_indices(stmt, unit_local_base, &function_map, &flat_index_by_symbol)?; } merged_stmt_sources.extend(std::iter::repeat_n( Some(source_name.clone()), @@ -63,8 +113,8 @@ pub(super) fn merge_units(units: Vec) -> Result) -> Result) -> Result Result { + u16::try_from(merged_functions.len()).map_err(|_| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: "too many functions across imported modules".to_string(), + })) + }) +} + +/// Register one unit's declarations in the flat function table and return the +/// unit-index → flat-index map. +/// +/// Synthetic prelude declarations (symbol-less, mirroring imported bindings) +/// never become flat entries: the loader already resolved their call sites to +/// [`Expr::ModuleCall`] with the target's [`SymbolId`]. +fn register_unit_functions( + unit: &ParsedUnit, merged_functions: &mut Vec, - function_index_by_name: &mut HashMap, + flat_index_by_symbol: &mut HashMap, + host_index_by_name: &mut HashMap, + claimed_flat_names: &mut HashSet, ) -> Result, SourcePathError> { let mut map = HashMap::new(); - for func in unit_functions { - let merged_index = if let Some(existing_index) = function_index_by_name.get(&func.name) { - let existing = &mut merged_functions[*existing_index as usize]; - if existing.arity != func.arity { - return Err(SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: format!( - "function '{}' declared with conflicting arity {} vs {}", - func.name, existing.arity, func.arity - ), - }))); - } - if existing.return_type != func.return_type { - match (existing.return_type, func.return_type) { - (crate::ValueType::Unknown, known) => existing.return_type = known, - (known, crate::ValueType::Unknown) => existing.return_type = known, - (lhs, rhs) => { - return Err(SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: format!( - "function '{}' declared with conflicting return type {} vs {}", - func.name, - value_type_name(lhs), - value_type_name(rhs) - ), - }))); - } - } - } - if existing.return_schema != func.return_schema { - match (&existing.return_schema, &func.return_schema) { - (None, Some(schema)) => existing.return_schema = Some(schema.clone()), - (Some(_), None) => {} - (Some(lhs), Some(rhs)) if lhs == rhs => {} - _ => { - return Err(SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: format!( - "function '{}' declared with conflicting return schemas across imported modules", - func.name - ), - }))); - } - } - } - if existing.type_params != func.type_params { - if existing.type_params.is_empty() { - existing.type_params = func.type_params.clone(); - } else if !func.type_params.is_empty() { - return Err(SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: format!( - "function '{}' declared with conflicting type parameters across imported modules", - func.name - ), - }))); - } - } - if existing.arg_schemas != func.arg_schemas { - if existing.arg_schemas.iter().all(Option::is_none) { - existing.arg_schemas = func.arg_schemas.clone(); - } else if !func.arg_schemas.iter().all(Option::is_none) { - return Err(SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: format!( - "function '{}' declared with conflicting parameter schemas across imported modules", - func.name - ), - }))); - } - } - if function_args_are_placeholders(&existing.args) - && !function_args_are_placeholders(&func.args) - { - existing.args = func.args.clone(); + for func in &unit.parsed.functions { + let Some(symbol) = func.symbol else { + // Synthetic prelude/stub declaration; resolved by the loader. + continue; + }; + if let Some(&existing) = flat_index_by_symbol.get(&symbol) { + map.insert(func.index, existing); + continue; + } + let has_impl = unit.parsed.function_impls.contains_key(&func.index); + let flat = if !has_impl { + // Host import: name-keyed deduplication preserves the legacy + // merge semantics and the runtime name-binding surface. + if let Some(&existing) = host_index_by_name.get(&func.name) { + merge_host_import_metadata(&mut merged_functions[existing as usize], func)?; + flat_index_by_symbol.insert(symbol, existing); + map.insert(func.index, existing); + continue; } - existing.exported = existing.exported || func.exported; - *existing_index - } else { - let next_index = u16::try_from(merged_functions.len()).map_err(|_| { - SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: "too many functions across imported modules".to_string(), - })) - })?; + let flat = next_flat_index(merged_functions)?; merged_functions.push(FunctionDecl { name: func.name.clone(), arity: func.arity, - index: next_index, + index: flat, + args: func.args.clone(), + arg_schemas: func.arg_schemas.clone(), + return_schema: func.return_schema.clone(), + type_params: func.type_params.clone(), + exported: func.exported, + return_type: func.return_type, + symbol: Some(symbol), + }); + host_index_by_name.insert(func.name.clone(), flat); + claimed_flat_names.insert(func.name.clone()); + flat + } else { + // Module function: one flat entry per symbol; the source name is + // kept unless another flat entry already claimed it, in which + // case it is deterministically mangled with the module identity. + let flat = next_flat_index(merged_functions)?; + let flat_name = if claimed_flat_names.insert(func.name.clone()) { + func.name.clone() + } else { + deterministic_flat_name(&func.name, symbol) + }; + merged_functions.push(FunctionDecl { + name: flat_name, + arity: func.arity, + index: flat, args: func.args.clone(), arg_schemas: func.arg_schemas.clone(), return_schema: func.return_schema.clone(), type_params: func.type_params.clone(), exported: func.exported, return_type: func.return_type, + symbol: Some(symbol), }); - function_index_by_name.insert(func.name.clone(), next_index); - next_index + flat }; - map.insert(func.index, merged_index); + flat_index_by_symbol.insert(symbol, flat); + map.insert(func.index, flat); } Ok(map) } +/// Replicate the legacy name-merge metadata rules for host imports that are +/// declared by more than one unit: arity conflicts are errors, `Unknown` +/// return types are refined, and schemas/type parameters merge. +fn merge_host_import_metadata( + existing: &mut FunctionDecl, + func: &FunctionDecl, +) -> Result<(), SourcePathError> { + if existing.arity != func.arity { + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: format!( + "function '{}' declared with conflicting arity {} vs {}", + func.name, existing.arity, func.arity + ), + }))); + } + if existing.return_type != func.return_type { + match (existing.return_type, func.return_type) { + (crate::ValueType::Unknown, known) => existing.return_type = known, + (known, crate::ValueType::Unknown) => existing.return_type = known, + (lhs, rhs) => { + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: format!( + "function '{}' declared with conflicting return type {} vs {}", + func.name, + value_type_name(lhs), + value_type_name(rhs) + ), + }))); + } + } + } + if existing.return_schema != func.return_schema { + match (&existing.return_schema, &func.return_schema) { + (None, Some(schema)) => existing.return_schema = Some(schema.clone()), + (Some(_), None) => {} + (Some(lhs), Some(rhs)) if lhs == rhs => {} + _ => { + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: format!( + "function '{}' declared with conflicting return schemas across imported modules", + func.name + ), + }))); + } + } + } + if existing.type_params != func.type_params { + if existing.type_params.is_empty() { + existing.type_params = func.type_params.clone(); + } else if !func.type_params.is_empty() { + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: format!( + "function '{}' declared with conflicting type parameters across imported modules", + func.name + ), + }))); + } + } + if existing.arg_schemas != func.arg_schemas { + if existing.arg_schemas.iter().all(Option::is_none) { + existing.arg_schemas = func.arg_schemas.clone(); + } else if !func.arg_schemas.iter().all(Option::is_none) { + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: format!( + "function '{}' declared with conflicting parameter schemas across imported modules", + func.name + ), + }))); + } + } + if function_args_are_placeholders(&existing.args) && !function_args_are_placeholders(&func.args) + { + existing.args = func.args.clone(); + } + existing.exported = existing.exported || func.exported; + Ok(()) +} + fn function_args_are_placeholders(args: &[String]) -> bool { args.iter() .enumerate() @@ -325,37 +446,51 @@ fn remap_stmt_indices( stmt: &mut Stmt, local_base: usize, function_map: &HashMap, + flat_index_by_symbol: &HashMap, ) -> Result<(), SourcePathError> { match stmt { Stmt::Noop { .. } => {} Stmt::Let { index, expr, .. } => { *index = remap_local_index(*index, local_base)?; - remap_expr_indices(expr, local_base, function_map)?; + remap_expr_indices(expr, local_base, function_map, flat_index_by_symbol)?; } Stmt::Assign { index, expr, .. } => { *index = remap_local_index(*index, local_base)?; - remap_expr_indices(expr, local_base, function_map)?; + remap_expr_indices(expr, local_base, function_map, flat_index_by_symbol)?; } Stmt::ClosureLet { closure, .. } => { for (source_index, captured_slot) in &mut closure.capture_copies { *source_index = remap_local_index(*source_index, local_base)?; *captured_slot = remap_local_index(*captured_slot, local_base)?; } - remap_expr_indices(&mut closure.body, local_base, function_map)?; + remap_expr_indices( + &mut closure.body, + local_base, + function_map, + flat_index_by_symbol, + )?; } - Stmt::FuncDecl { index, .. } => { - *index = function_map.get(index).copied().ok_or_else(|| { - SourcePathError::Source(SourceError::Parse(ParseError { - span: None, - code: None, - line: 1, - message: "function index remap failed while merging imported modules" - .to_string(), - })) - })?; + Stmt::FuncDecl { + index, has_impl, .. + } => { + // Implementation-less declarations (import prelude stubs, extern + // prototypes) never enter the flat table and codegen ignores + // their index; only declarations with implementations are + // remapped to their symbol-owned flat entry. + if *has_impl { + *index = function_map.get(index).copied().ok_or_else(|| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: "function index remap failed while merging imported modules" + .to_string(), + })) + })?; + } } Stmt::Expr { expr, .. } => { - remap_expr_indices(expr, local_base, function_map)?; + remap_expr_indices(expr, local_base, function_map, flat_index_by_symbol)?; } Stmt::IfElse { condition, @@ -363,12 +498,12 @@ fn remap_stmt_indices( else_branch, .. } => { - remap_expr_indices(condition, local_base, function_map)?; + remap_expr_indices(condition, local_base, function_map, flat_index_by_symbol)?; for stmt in then_branch { - remap_stmt_indices(stmt, local_base, function_map)?; + remap_stmt_indices(stmt, local_base, function_map, flat_index_by_symbol)?; } for stmt in else_branch { - remap_stmt_indices(stmt, local_base, function_map)?; + remap_stmt_indices(stmt, local_base, function_map, flat_index_by_symbol)?; } } Stmt::For { @@ -378,19 +513,19 @@ fn remap_stmt_indices( body, .. } => { - remap_stmt_indices(init, local_base, function_map)?; - remap_expr_indices(condition, local_base, function_map)?; - remap_stmt_indices(post, local_base, function_map)?; + remap_stmt_indices(init, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(condition, local_base, function_map, flat_index_by_symbol)?; + remap_stmt_indices(post, local_base, function_map, flat_index_by_symbol)?; for stmt in body { - remap_stmt_indices(stmt, local_base, function_map)?; + remap_stmt_indices(stmt, local_base, function_map, flat_index_by_symbol)?; } } Stmt::While { condition, body, .. } => { - remap_expr_indices(condition, local_base, function_map)?; + remap_expr_indices(condition, local_base, function_map, flat_index_by_symbol)?; for stmt in body { - remap_stmt_indices(stmt, local_base, function_map)?; + remap_stmt_indices(stmt, local_base, function_map, flat_index_by_symbol)?; } } Stmt::Break { .. } | Stmt::Continue { .. } => {} @@ -405,6 +540,7 @@ fn remap_expr_indices( expr: &mut Expr, local_base: usize, function_map: &HashMap, + flat_index_by_symbol: &HashMap, ) -> Result<(), SourcePathError> { match expr { Expr::Null @@ -426,6 +562,29 @@ fn remap_expr_indices( }))); } } + Expr::ModuleFunctionRef(symbol, _) => { + let flat = flat_index_by_symbol.get(symbol).copied().ok_or_else(|| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: + "resolved module function value target is missing from the merged function table" + .to_string(), + })) + })?; + *expr = Expr::FunctionRef(flat, std::mem::take(&mut expr_type_args(expr))); + } + Expr::UnresolvedFunctionRef { .. } => { + // The loader resolves every function-value reference before + // merge; reaching the merge means resolution missed a site. + return Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: "unresolved function value reference reached the module merge".to_string(), + }))); + } Expr::Call(index, _, args) => { if let Some(remapped_index) = function_map.get(index).copied() { *index = remapped_index; @@ -439,8 +598,24 @@ fn remap_expr_indices( }))); } for arg in args { - remap_expr_indices(arg, local_base, function_map)?; + remap_expr_indices(arg, local_base, function_map, flat_index_by_symbol)?; + } + } + Expr::ModuleCall(symbol, type_args, args) => { + for arg in args.iter_mut() { + remap_expr_indices(arg, local_base, function_map, flat_index_by_symbol)?; } + let flat = flat_index_by_symbol.get(symbol).copied().ok_or_else(|| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: None, + code: None, + line: 1, + message: + "resolved module call target is missing from the merged function table" + .to_string(), + })) + })?; + *expr = Expr::Call(flat, std::mem::take(type_args), std::mem::take(args)); } Expr::OptionalGet { container, @@ -450,8 +625,8 @@ fn remap_expr_indices( } => { *container_slot = remap_local_index(*container_slot, local_base)?; *key_slot = remap_local_index(*key_slot, local_base)?; - remap_expr_indices(container, local_base, function_map)?; - remap_expr_indices(key, local_base, function_map)?; + remap_expr_indices(container, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(key, local_base, function_map, flat_index_by_symbol)?; } Expr::OptionUnwrapOr { value, @@ -459,13 +634,13 @@ fn remap_expr_indices( fallback, } => { *value_slot = remap_local_index(*value_slot, local_base)?; - remap_expr_indices(value, local_base, function_map)?; - remap_expr_indices(fallback, local_base, function_map)?; + remap_expr_indices(value, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(fallback, local_base, function_map, flat_index_by_symbol)?; } Expr::LocalCall(index, _, args) => { *index = remap_local_index(*index, local_base)?; for arg in args { - remap_expr_indices(arg, local_base, function_map)?; + remap_expr_indices(arg, local_base, function_map, flat_index_by_symbol)?; } } Expr::Closure(closure) => { @@ -476,7 +651,12 @@ fn remap_expr_indices( *source_index = remap_local_index(*source_index, local_base)?; *captured_slot = remap_local_index(*captured_slot, local_base)?; } - remap_expr_indices(&mut closure.body, local_base, function_map)?; + remap_expr_indices( + &mut closure.body, + local_base, + function_map, + flat_index_by_symbol, + )?; } Expr::ClosureCall(closure, args) => { for param_slot in &mut closure.param_slots { @@ -486,9 +666,14 @@ fn remap_expr_indices( *source_index = remap_local_index(*source_index, local_base)?; *captured_slot = remap_local_index(*captured_slot, local_base)?; } - remap_expr_indices(&mut closure.body, local_base, function_map)?; + remap_expr_indices( + &mut closure.body, + local_base, + function_map, + flat_index_by_symbol, + )?; for arg in args { - remap_expr_indices(arg, local_base, function_map)?; + remap_expr_indices(arg, local_base, function_map, flat_index_by_symbol)?; } } Expr::Add(lhs, rhs) @@ -501,15 +686,15 @@ fn remap_expr_indices( | Expr::Eq(lhs, rhs) | Expr::Lt(lhs, rhs) | Expr::Gt(lhs, rhs) => { - remap_expr_indices(lhs, local_base, function_map)?; - remap_expr_indices(rhs, local_base, function_map)?; + remap_expr_indices(lhs, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(rhs, local_base, function_map, flat_index_by_symbol)?; } Expr::Neg(inner) | Expr::Not(inner) | Expr::ToOwned(inner) | Expr::Borrow(inner) | Expr::BorrowMut(inner) => { - remap_expr_indices(inner, local_base, function_map)?; + remap_expr_indices(inner, local_base, function_map, flat_index_by_symbol)?; } Expr::Var(index) | Expr::MoveVar(index) => { *index = remap_local_index(*index, local_base)?; @@ -522,9 +707,9 @@ fn remap_expr_indices( then_expr, else_expr, } => { - remap_expr_indices(condition, local_base, function_map)?; - remap_expr_indices(then_expr, local_base, function_map)?; - remap_expr_indices(else_expr, local_base, function_map)?; + remap_expr_indices(condition, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(then_expr, local_base, function_map, flat_index_by_symbol)?; + remap_expr_indices(else_expr, local_base, function_map, flat_index_by_symbol)?; } Expr::Match { value_slot, @@ -535,21 +720,32 @@ fn remap_expr_indices( } => { *value_slot = remap_local_index(*value_slot, local_base)?; *result_slot = remap_local_index(*result_slot, local_base)?; - remap_expr_indices(value, local_base, function_map)?; + remap_expr_indices(value, local_base, function_map, flat_index_by_symbol)?; for (pattern, arm_expr) in arms { if let crate::compiler::ir::MatchPattern::SomeBinding(binding_slot) = pattern { *binding_slot = remap_local_index(*binding_slot, local_base)?; } - remap_expr_indices(arm_expr, local_base, function_map)?; + remap_expr_indices(arm_expr, local_base, function_map, flat_index_by_symbol)?; } - remap_expr_indices(default, local_base, function_map)?; + remap_expr_indices(default, local_base, function_map, flat_index_by_symbol)?; } Expr::Block { stmts, expr } => { for stmt in stmts { - remap_stmt_indices(stmt, local_base, function_map)?; + remap_stmt_indices(stmt, local_base, function_map, flat_index_by_symbol)?; } - remap_expr_indices(expr, local_base, function_map)?; + remap_expr_indices(expr, local_base, function_map, flat_index_by_symbol)?; } } Ok(()) } + +/// Borrow the type arguments of a resolved function-value node. +/// +/// Only used while converting a [`Expr::ModuleFunctionRef`] into a plain +/// [`Expr::FunctionRef`] in place. +fn expr_type_args(expr: &mut Expr) -> Vec { + match expr { + Expr::ModuleFunctionRef(_, type_args) => std::mem::take(type_args), + _ => Vec::new(), + } +} diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index 3b52d54b..ff77d89f 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -14,6 +14,7 @@ mod frontends; pub mod ir; mod lifetime; mod linker; +mod modules; mod parser; mod pipeline; mod source_loader; @@ -31,6 +32,10 @@ pub use self::ir::{ AssignmentKind, ClosureExpr, Expr, FrontendIr, FunctionDecl, FunctionImpl, FunctionParam, LocalIrBuilder, LocalSlot, MatchPattern, MatchTypePattern, Stmt, StructDecl, TypeSchema, }; +pub use self::modules::{ + DeclSymbol, ExportEntry, ImportTargetKind, ImportedBinding, ModuleGraph, ModuleId, ModuleNode, + ResolvedImport, SymbolId, UseDecl, UsePathSegment, +}; pub use self::parser::ParserDialect; pub use self::pipeline::{ InferredLocalTypeHint, UnknownInferredLocal, collect_inferred_local_type_hints, @@ -90,6 +95,10 @@ pub enum CompileError { source_name: Option, detail: String, }, + /// Internal error: a symbol-resolved module call or function value + /// survived unit merge and reached codegen, where flat function indices + /// are the only valid call targets. + UnresolvedModuleCall, } impl CompileError { @@ -160,6 +169,9 @@ impl CompileError { CompileError::InvalidFieldAccess { detail, .. } => detail.clone(), CompileError::FunctionParameterTypeConflict { detail, .. } => detail.clone(), CompileError::StrictTypingRequired { detail, .. } => detail.clone(), + CompileError::UnresolvedModuleCall => { + "internal compiler error: unresolved module call reached codegen".to_string() + } } } } @@ -272,6 +284,25 @@ pub enum SourcePathError { message: String, }, Source(SourceError), + /// A source error plus the compilation-wide [`SourceMap`] that resolves + /// every span it carries (milestone 5). Produced by the module-loading + /// compile entry points; spans reference the semantic module graph's + /// `SourceId` space, so rendering against this map always reads from the + /// owning source. `Display` delegates to the inner error. + SourceWithMap { + error: SourceError, + sources: SourceMap, + }, +} + +impl SourcePathError { + /// The compilation-wide source map carried with this error, if any. + pub fn sources(&self) -> Option<&SourceMap> { + match self { + SourcePathError::SourceWithMap { sources, .. } => Some(sources), + _ => None, + } + } } impl fmt::Display for SourcePathError { @@ -309,6 +340,7 @@ impl fmt::Display for SourcePathError { message ), SourcePathError::Source(err) => write!(f, "{err}"), + SourcePathError::SourceWithMap { error, .. } => write!(f, "{error}"), } } } @@ -366,6 +398,12 @@ pub struct SharedParserOptions { pub allow_implicit_externs: bool, pub allow_implicit_semicolons: bool, pub enforce_mutable_bindings: bool, + /// Import-scan mode: used by the source loader's discovery parse. The + /// parser tolerates calls to not-yet-declared imported functions + /// (`allow_implicit_externs`) and records host aliases for multi-segment + /// file-module paths so namespace calls parse during the scan; the + /// resulting IR is discarded after `use` declarations are extracted. + pub import_scan_mode: bool, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] diff --git a/src/compiler/modules.rs b/src/compiler/modules.rs new file mode 100644 index 00000000..efe8ea97 --- /dev/null +++ b/src/compiler/modules.rs @@ -0,0 +1,1046 @@ +//! Compiler-owned module identities and the semantic module graph. +//! +//! Milestones 1-6 of the semantic module system: every source that takes +//! part in a compilation is assigned a deterministic [`ModuleId`] and +//! [`SourceId`], `use` directives are parsed into structured [`UseDecl`] +//! nodes with spans and clauses, the source loader records resolved import +//! edges in a [`ModuleGraph`], and every declaration receives a +//! [`SymbolId`] owned by its module alongside an explicit public export +//! table and a separate imported-binding table. Identities never depend on +//! a file stem alone: two modules with the same basename in different +//! directories are distinct nodes, and re-visiting the same module identity +//! reuses the same node. +//! +//! Since milestone 6 the semantic graph is the *sole* file-module path: the +//! textual import rewriting, the synthetic imported-function prelude, and +//! the prelude line-map remapping are removed. Call sites resolve to +//! [`SymbolId`]s in the source loader and the linker merges units by symbol +//! identity, applying deterministic flat-boundary mangling only at the final +//! bytecode boundary. +//! +//! [`SourceId`] here is the module graph's own identity space, distinct from +//! `source_map::SourceId` (which is assigned per-unit by ad hoc `SourceMap` +//! instances). Milestone 5 reconciles the two spaces: every module's raw +//! text is registered in the compilation-wide `SourceMap` at its graph +//! `SourceId`, so spans survive unit merge with their owning source. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use super::SourcePathError; +use super::frontends::{is_ident_continue, is_ident_start}; +use super::source_loader::ImportClause; +use super::source_map::Span; + +/// Deterministic identity of one module within a single compilation. +/// +/// Assigned in discovery order: the root unit is always `ModuleId(0)`, and +/// every discovered module gets the next unused id the first time its +/// canonical identity is registered. Re-importing the same module yields the +/// same id; two modules that merely share a file stem are distinct. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ModuleId(pub u32); + +/// Deterministic identity of one parsed source text within a compilation. +/// +/// Distinct from `source_map::SourceId`: the module graph hands out its own +/// monotonic ids so that graph edges can reference sources without depending +/// on per-unit `SourceMap` construction order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SourceId(pub u32); + +/// Deterministic identity of one declaration within a compilation. +/// +/// Composed of the owning [`ModuleId`] and a module-local index, so two +/// same-named declarations in independent modules never collide. Milestone 3 +/// assigns symbol ids to declarations; the type is defined here so the whole +/// identity surface lands in one place. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SymbolId { + pub module: ModuleId, + pub index: u32, +} + +/// One segment of a structured `use` path. +/// +/// `self`/`super` are only classified as qualifiers while they lead the path, +/// mirroring the legacy line-based resolver: a `self` appearing mid-path is a +/// literal file segment (e.g. `use a::self::b;` resolves `a/self/b.rss`). +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UsePathSegment { + Self_, + Super, + Ident(String), +} + +/// A structured `use` directive parsed from RustScript source. +/// +/// Carries the full path (including `self`/`super` qualifiers), the import +/// clause, the exact source span of the directive, and the directive line. +/// The source loader consumes these nodes for discovery instead of treating +/// line-prefix stripping as the authoritative import parser. +#[derive(Clone, Debug)] +pub struct UseDecl { + pub path: Vec, + pub clause: ImportClause, + pub span: Span, + pub line: usize, +} + +/// Classification of one resolved import edge. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ImportTargetKind { + /// A RustScript file module loaded from disk or an override. + FileModule, + /// A virtual host namespace resolved on the dedicated host path. + HostNamespace, + /// A builtin namespace such as `io`, `json`, or `re`. + BuiltinNamespace, +} + +/// A resolved import edge inside a [`ModuleGraph`]. +/// +/// `target` is `Some(ModuleId)` once the destination module node is known +/// (`FileModule` edges), and `None` for host/builtin namespaces that stay on +/// their dedicated resolution paths. +#[derive(Clone, Debug)] +pub struct ResolvedImport { + pub kind: ImportTargetKind, + /// Normalized module specifier (e.g. `./nested.rss`). + pub spec: String, + pub clause: ImportClause, + pub span: Span, + pub line: usize, + pub target: Option, +} + +/// One declaration owned by a module, with its deterministic [`SymbolId`]. +/// +/// Milestone 3: every declaration in a module receives a symbol whose +/// `module` is the owning [`ModuleId`] and whose `index` is the declaration's +/// position in the module's declaration table. Two same-named declarations in +/// independent modules therefore never share a symbol. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeclSymbol { + pub symbol: SymbolId, + pub name: String, + /// `true` when the declaration is marked `pub` and appears in the module's + /// public export table. + pub public: bool, +} + +/// One entry of a module's public export table. +/// +/// The table is populated exclusively from local public declarations: +/// imported bindings never appear here, so re-exporting another module's +/// functions requires an explicit mechanism and there is no implicit +/// transitive re-export through the graph. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ExportEntry { + pub name: String, + pub symbol: SymbolId, +} + +/// A binding introduced into a module by a resolved import edge. +/// +/// Imported bindings are tracked separately from local declarations: they are +/// never part of [`ModuleNode::declarations`] and never enter +/// [`ModuleNode::exports`]. `local_name` is the name the importing module +/// binds (`as` alias for named imports, or the source name otherwise); +/// `source_name` is the declaration's name in the source module. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ImportedBinding { + pub local_name: String, + pub source_module: ModuleId, + pub source_symbol: SymbolId, + pub source_name: String, +} + +/// One node of the [`ModuleGraph`]: a module, its resolved imports, and its +/// milestone-3 declaration/export/imported-binding tables. +/// +/// Milestone 1-2 fills `imports` during source-loader discovery; milestone 3 +/// fills `declarations`, `exports`, and `imported_bindings` once each module's +/// unit is parsed. +#[derive(Clone, Debug)] +pub struct ModuleNode { + pub module: ModuleId, + pub source: SourceId, + /// Canonical disk identity (or normalized virtual identity) of the module. + pub identity: PathBuf, + /// Display name used in diagnostics. + pub source_name: String, + pub imports: Vec, + /// Local declarations in source order, each with its owned symbol. + pub declarations: Vec, + /// Public export table: local `pub` declarations only. + pub exports: Vec, + /// Bindings introduced by import edges, separate from local declarations. + pub imported_bindings: Vec, +} + +/// The module graph for one compilation. +/// +/// Nodes are registered in deterministic discovery order; the first node is +/// always the root unit. `by_identity` guarantees that the same canonical +/// module identity maps to exactly one node, so modules with identical file +/// stems in different directories stay distinct while lexically equivalent +/// paths collapse. +#[derive(Default)] +pub struct ModuleGraph { + nodes: Vec, + by_identity: HashMap, + next_source: u32, +} + +impl ModuleGraph { + pub fn new() -> Self { + Self::default() + } + + /// Register a module node for `identity`, or return the existing node id. + /// + /// The first call for the root unit yields `ModuleId(0)` / `SourceId(0)`; + /// subsequent first-time registrations receive the next unused ids in + /// call order. + pub fn add_node( + &mut self, + identity: PathBuf, + source_name: String, + imports: Vec, + ) -> ModuleId { + if let Some(existing) = self.by_identity.get(&identity) { + return *existing; + } + let module = ModuleId(u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)); + let source = SourceId(self.next_source); + self.next_source = self.next_source.saturating_add(1); + self.by_identity.insert(identity.clone(), module); + self.nodes.push(ModuleNode { + module, + source, + identity, + source_name, + imports, + declarations: Vec::new(), + exports: Vec::new(), + imported_bindings: Vec::new(), + }); + module + } + + pub fn node(&self, module: ModuleId) -> Option<&ModuleNode> { + self.nodes.get(module.0 as usize) + } + + pub fn nodes(&self) -> &[ModuleNode] { + &self.nodes + } + + pub fn len(&self) -> usize { + self.nodes.len() + } + + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + pub fn module_id_for_identity(&self, identity: &Path) -> Option { + self.by_identity.get(identity).copied() + } + + /// Append one resolved import edge to a module node. + pub fn add_import(&mut self, module: ModuleId, import: ResolvedImport) { + if let Some(node) = self.nodes.get_mut(module.0 as usize) { + node.imports.push(import); + } + } + + /// Register one local declaration of `module` and return its symbol. + /// + /// Symbols are deterministic: the first declaration of a module is + /// `SymbolId { module, index: 0 }`, the second `index: 1`, and so on in + /// registration order. Public declarations are appended to the module's + /// export table at the same time; imported bindings never enter it. + /// + /// Errors when the module already declares the same name (the parser + /// normally reports this first with a source line) or when the name is + /// already bound by an import. + pub fn add_declaration( + &mut self, + module: ModuleId, + name: &str, + public: bool, + ) -> Result { + let node = self.node_mut(module)?; + if node.declarations.iter().any(|decl| decl.name == name) { + return Err(format!( + "duplicate local declaration '{name}' in module '{}'", + node.source_name + )); + } + if node + .imported_bindings + .iter() + .any(|binding| binding.local_name == name) + { + return Err(format!( + "local declaration '{name}' conflicts with an imported binding in module '{}'", + node.source_name + )); + } + let index = u32::try_from(node.declarations.len()) + .map_err(|_| format!("too many declarations in module '{}'", node.source_name))?; + let symbol = SymbolId { module, index }; + node.declarations.push(DeclSymbol { + symbol, + name: name.to_string(), + public, + }); + if public { + node.exports.push(ExportEntry { + name: name.to_string(), + symbol, + }); + } + Ok(symbol) + } + + /// Record one binding introduced by an import edge of `module`. + /// + /// Imported bindings stay out of `declarations` and `exports`: nothing in + /// the graph re-exports them implicitly. Binding the same local name from + /// several modules is recorded as several bindings (the legacy pipeline + /// merges such imports by name; milestone 4 resolves them by symbol), but + /// a binding that clashes with a local declaration is rejected. + pub fn add_imported_binding( + &mut self, + module: ModuleId, + binding: ImportedBinding, + ) -> Result<(), String> { + let node = self.node_mut(module)?; + if node + .declarations + .iter() + .any(|decl| decl.name == binding.local_name) + { + return Err(format!( + "imported binding '{}' conflicts with a local declaration in module '{}'", + binding.local_name, node.source_name + )); + } + node.imported_bindings.push(binding); + Ok(()) + } + + pub fn declaration(&self, module: ModuleId, name: &str) -> Option<&DeclSymbol> { + self.node(module)? + .declarations + .iter() + .find(|decl| decl.name == name) + } + + pub fn declaration_symbol(&self, module: ModuleId, name: &str) -> Option { + self.declaration(module, name).map(|decl| decl.symbol) + } + + pub fn export(&self, module: ModuleId, name: &str) -> Option<&ExportEntry> { + self.node(module)? + .exports + .iter() + .find(|entry| entry.name == name) + } + + pub fn symbol_for_export(&self, module: ModuleId, name: &str) -> Option { + self.export(module, name).map(|entry| entry.symbol) + } + + pub fn imported_binding(&self, module: ModuleId, name: &str) -> Option<&ImportedBinding> { + self.node(module)? + .imported_bindings + .iter() + .find(|binding| binding.local_name == name) + } + + fn node_mut(&mut self, module: ModuleId) -> Result<&mut ModuleNode, String> { + self.nodes + .get_mut(module.0 as usize) + .ok_or_else(|| format!("unknown module id {} in module graph", module.0)) + } +} + +fn is_valid_ident_segment(input: &str) -> bool { + let mut chars = input.chars(); + let Some(first) = chars.next() else { + return false; + }; + is_ident_start(first) && chars.all(is_ident_continue) +} + +/// Convert a structured `use` path into a module specifier. +/// +/// Replicates the legacy resolver's semantics from structured segments: +/// leading `self`/`super` qualifiers become `./`/`../` prefixes, a leading +/// `crate` is rejected, remaining segments join into a relative path, and the +/// result is normalized to a `.rss` specifier. +pub(super) fn use_path_to_spec( + path: &Path, + line: usize, + segments: &[UsePathSegment], +) -> Result { + let invalid = |message: &str| SourcePathError::InvalidImportSyntax { + path: path.to_path_buf(), + line, + message: message.to_string(), + }; + if segments.is_empty() { + return Err(invalid("expected module path after 'use'")); + } + + let mut path_prefix = PathBuf::new(); + let mut cursor = 0usize; + let mut explicit_self = false; + while cursor < segments.len() { + match &segments[cursor] { + UsePathSegment::Self_ => { + explicit_self = true; + cursor += 1; + } + UsePathSegment::Super => { + path_prefix.push(".."); + cursor += 1; + } + UsePathSegment::Ident(name) if name == "crate" => { + return Err(invalid( + "crate:: paths are not supported; use relative module paths", + )); + } + UsePathSegment::Ident(_) => break, + } + } + if cursor >= segments.len() { + return Err(invalid("expected module name after path qualifiers")); + } + + for segment in &segments[cursor..] { + match segment { + UsePathSegment::Ident(name) => { + if !is_valid_ident_segment(name) { + return Err(invalid(&format!( + "invalid module path segment '{name}' in use directive" + ))); + } + path_prefix.push(name); + } + // Mid-path qualifier words are literal file segments, mirroring + // the legacy line-based resolver. + UsePathSegment::Self_ => path_prefix.push("self"), + UsePathSegment::Super => path_prefix.push("super"), + } + } + + let mut spec = path_prefix.to_string_lossy().replace('\\', "/"); + if spec.is_empty() { + return Err(invalid("expected module path after 'use'")); + } + if explicit_self && !spec.starts_with("../") { + spec = format!("./{spec}"); + } + if !spec.ends_with(".rss") { + spec.push_str(".rss"); + } + Ok(spec) +} + +#[cfg(test)] +mod tests { + use super::{ + ImportTargetKind, ImportedBinding, ModuleGraph, ModuleId, ResolvedImport, SourceId, + SymbolId, UsePathSegment, use_path_to_spec, + }; + use crate::compiler::source_loader::ImportClause; + use crate::compiler::source_map::Span; + use std::path::PathBuf; + + fn ident(name: &str) -> UsePathSegment { + UsePathSegment::Ident(name.to_string()) + } + + #[test] + fn use_path_to_spec_plain_module_path() { + let path = PathBuf::from("/root/main.rss"); + let spec = use_path_to_spec(&path, 1, &[ident("helpers")]).expect("spec should resolve"); + assert_eq!(spec, "helpers.rss"); + let spec = + use_path_to_spec(&path, 1, &[ident("a"), ident("b")]).expect("spec should resolve"); + assert_eq!(spec, "a/b.rss"); + } + + #[test] + fn use_path_to_spec_self_and_super_qualifiers() { + let path = PathBuf::from("/root/pkg/main.rss"); + let spec = use_path_to_spec(&path, 1, &[UsePathSegment::Self_, ident("nested")]) + .expect("spec should resolve"); + assert_eq!(spec, "./nested.rss"); + let spec = use_path_to_spec(&path, 1, &[UsePathSegment::Super, ident("shared")]) + .expect("spec should resolve"); + assert_eq!(spec, "../shared.rss"); + let spec = use_path_to_spec( + &path, + 1, + &[ + UsePathSegment::Self_, + UsePathSegment::Super, + ident("nested"), + ], + ) + .expect("spec should resolve"); + assert_eq!(spec, "../nested.rss"); + let spec = use_path_to_spec( + &path, + 1, + &[UsePathSegment::Self_, UsePathSegment::Self_, ident("x")], + ) + .expect("spec should resolve"); + assert_eq!(spec, "./x.rss"); + } + + #[test] + fn use_path_to_spec_rejects_leading_crate() { + let path = PathBuf::from("/root/main.rss"); + let err = use_path_to_spec(&path, 4, &[ident("crate"), ident("x")]) + .expect_err("crate:: should be rejected"); + let message = err.to_string(); + assert!( + message.contains("crate:: paths are not supported"), + "unexpected error: {message}" + ); + assert!( + message.contains("line 4"), + "line should be preserved: {message}" + ); + } + + #[test] + fn use_path_to_spec_requires_module_name_after_qualifiers() { + let path = PathBuf::from("/root/main.rss"); + let err = use_path_to_spec(&path, 2, &[UsePathSegment::Self_]) + .expect_err("bare self:: should be rejected"); + assert!( + err.to_string() + .contains("expected module name after path qualifiers"), + "unexpected error: {err}" + ); + } + + #[test] + fn use_path_to_spec_mid_path_qualifier_words_are_literal_segments() { + let path = PathBuf::from("/root/main.rss"); + let spec = use_path_to_spec(&path, 1, &[ident("a"), UsePathSegment::Self_, ident("b")]) + .expect("mid-path self should be a literal segment"); + assert_eq!(spec, "a/self/b.rss"); + } + + #[test] + fn module_graph_ids_are_deterministic_and_deduplicated() { + let mut graph = ModuleGraph::new(); + let root = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + assert_eq!(root, ModuleId(0)); + let again = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + assert_eq!( + again, root, + "re-registering the same identity reuses the node" + ); + + let nested = graph.add_node( + PathBuf::from("/root/nested.rss"), + "/root/nested.rss".to_string(), + Vec::new(), + ); + assert_eq!(nested, ModuleId(1)); + assert_eq!(graph.node(nested).expect("node exists").source, SourceId(1)); + } + + #[test] + fn module_graph_same_stem_modules_are_distinct() { + let mut graph = ModuleGraph::new(); + let first = graph.add_node( + PathBuf::from("/root/a/common.rss"), + "/root/a/common.rss".to_string(), + Vec::new(), + ); + let second = graph.add_node( + PathBuf::from("/root/b/common.rss"), + "/root/b/common.rss".to_string(), + Vec::new(), + ); + assert_ne!( + first, second, + "modules that share a stem but differ by directory must be distinct" + ); + assert_eq!(graph.len(), 2); + assert_eq!( + graph.module_id_for_identity(PathBuf::from("/root/b/common.rss").as_path()), + Some(second) + ); + } + + #[test] + fn module_graph_records_import_edges_and_targets() { + let mut graph = ModuleGraph::new(); + let root = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + let nested = graph.add_node( + PathBuf::from("/root/nested.rss"), + "/root/nested.rss".to_string(), + Vec::new(), + ); + graph.add_import( + root, + ResolvedImport { + kind: ImportTargetKind::FileModule, + spec: "./nested.rss".to_string(), + clause: ImportClause::Namespace("nested".to_string()), + span: Span::new(0, 0, 0), + line: 1, + target: Some(nested), + }, + ); + graph.add_import( + root, + ResolvedImport { + kind: ImportTargetKind::BuiltinNamespace, + spec: "json.rss".to_string(), + clause: ImportClause::AllPublic, + span: Span::new(0, 0, 0), + line: 2, + target: None, + }, + ); + let node = graph.node(root).expect("root node exists"); + assert_eq!(node.imports.len(), 2); + assert_eq!(node.imports[0].target, Some(nested)); + assert_eq!(node.imports[1].kind, ImportTargetKind::BuiltinNamespace); + assert_eq!(node.imports[1].target, None); + } + + #[test] + fn symbol_ids_are_module_scoped() { + let a = SymbolId { + module: ModuleId(0), + index: 3, + }; + let b = SymbolId { + module: ModuleId(1), + index: 3, + }; + assert_ne!(a, b, "same index in different modules must differ"); + assert_eq!( + SymbolId { + module: ModuleId(0), + index: 3 + }, + a + ); + } + + #[test] + fn declaration_symbols_are_deterministic_and_module_owned() { + let mut graph = ModuleGraph::new(); + let first = graph.add_node( + PathBuf::from("/root/a.rss"), + "/root/a.rss".to_string(), + Vec::new(), + ); + let second = graph.add_node( + PathBuf::from("/root/b.rss"), + "/root/b.rss".to_string(), + Vec::new(), + ); + let first_symbol = graph + .add_declaration(first, "run", true) + .expect("declaration registers"); + let second_symbol = graph + .add_declaration(second, "run", true) + .expect("declaration registers"); + assert_eq!( + first_symbol, + SymbolId { + module: first, + index: 0 + }, + "first declaration of a module owns symbol index 0" + ); + assert_eq!( + second_symbol, + SymbolId { + module: second, + index: 0 + }, + "same index in a different module is a different symbol" + ); + assert_ne!(first_symbol, second_symbol); + + let next = graph + .add_declaration(first, "helper", false) + .expect("declaration registers"); + assert_eq!( + next, + SymbolId { + module: first, + index: 1 + }, + "symbols are assigned in declaration order" + ); + assert_eq!(graph.declaration_symbol(first, "run"), Some(first_symbol)); + assert_eq!(graph.declaration_symbol(second, "run"), Some(second_symbol)); + } + + #[test] + fn export_table_contains_only_public_declarations() { + let mut graph = ModuleGraph::new(); + let module = graph.add_node( + PathBuf::from("/root/lib.rss"), + "/root/lib.rss".to_string(), + Vec::new(), + ); + let public = graph + .add_declaration(module, "visible", true) + .expect("public declaration registers"); + graph + .add_declaration(module, "hidden", false) + .expect("private declaration registers"); + + let node = graph.node(module).expect("node exists"); + assert_eq!( + node.exports.len(), + 1, + "only the pub declaration is exported" + ); + assert_eq!(node.exports[0].name, "visible"); + assert_eq!(node.exports[0].symbol, public); + assert_eq!(graph.symbol_for_export(module, "visible"), Some(public)); + assert_eq!( + graph.symbol_for_export(module, "hidden"), + None, + "private declarations never enter the export table" + ); + assert!( + !graph + .declaration(module, "hidden") + .expect("declaration exists") + .public + ); + assert_eq!(node.declarations.len(), 2); + } + + #[test] + fn duplicate_local_declaration_is_rejected() { + let mut graph = ModuleGraph::new(); + let module = graph.add_node( + PathBuf::from("/root/lib.rss"), + "/root/lib.rss".to_string(), + Vec::new(), + ); + graph + .add_declaration(module, "run", true) + .expect("first declaration registers"); + let err = graph + .add_declaration(module, "run", false) + .expect_err("second same-named declaration must be rejected"); + assert!( + err.contains("duplicate local declaration 'run'"), + "unexpected error: {err}" + ); + // The same name in another module is not a duplicate. + let other = graph.add_node( + PathBuf::from("/root/other.rss"), + "/root/other.rss".to_string(), + Vec::new(), + ); + graph + .add_declaration(other, "run", true) + .expect("independent module may reuse the name"); + } + + #[test] + fn imported_bindings_stay_separate_from_local_declarations() { + let mut graph = ModuleGraph::new(); + let importer = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + let source = graph.add_node( + PathBuf::from("/root/util.rss"), + "/root/util.rss".to_string(), + Vec::new(), + ); + let exported = graph + .add_declaration(source, "helper", true) + .expect("source export registers"); + graph + .add_imported_binding( + importer, + ImportedBinding { + local_name: "helper".to_string(), + source_module: source, + source_symbol: exported, + source_name: "helper".to_string(), + }, + ) + .expect("imported binding registers"); + + let node = graph.node(importer).expect("importer node exists"); + assert_eq!( + node.declarations.len(), + 0, + "imported symbols are not local declarations" + ); + assert_eq!( + node.exports.len(), + 0, + "imported symbols never enter the export table" + ); + assert_eq!(node.imported_bindings.len(), 1); + let binding = graph + .imported_binding(importer, "helper") + .expect("binding exists"); + assert_eq!(binding.source_module, source); + assert_eq!(binding.source_symbol, exported); + assert_eq!(binding.source_name, "helper"); + } + + #[test] + fn same_named_declarations_across_modules_coexist() { + let mut graph = ModuleGraph::new(); + let first = graph.add_node( + PathBuf::from("/root/a/util.rss"), + "/root/a/util.rss".to_string(), + Vec::new(), + ); + let second = graph.add_node( + PathBuf::from("/root/b/util.rss"), + "/root/b/util.rss".to_string(), + Vec::new(), + ); + let first_helper = graph + .add_declaration(first, "helper", false) + .expect("private helper registers"); + let second_helper = graph + .add_declaration(second, "helper", false) + .expect("private helper registers"); + assert_ne!( + first_helper, second_helper, + "same-named helpers in independent modules have distinct symbols" + ); + assert_eq!( + graph.declaration_symbol(first, "helper"), + Some(first_helper) + ); + assert_eq!( + graph.declaration_symbol(second, "helper"), + Some(second_helper) + ); + + // Same-named *public* functions in independent modules coexist too, + // each in its own export table. + graph + .add_declaration(first, "run", true) + .expect("public run registers in first module"); + graph + .add_declaration(second, "run", true) + .expect("public run registers in second module"); + let first_run = graph + .symbol_for_export(first, "run") + .expect("first export exists"); + let second_run = graph + .symbol_for_export(second, "run") + .expect("second export exists"); + assert_ne!(first_run, second_run); + } + + #[test] + fn no_implicit_transitive_reexport_through_the_graph() { + let mut graph = ModuleGraph::new(); + let root = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + let middle = graph.add_node( + PathBuf::from("/root/middle.rss"), + "/root/middle.rss".to_string(), + Vec::new(), + ); + let leaf = graph.add_node( + PathBuf::from("/root/leaf.rss"), + "/root/leaf.rss".to_string(), + Vec::new(), + ); + let shared = graph + .add_declaration(leaf, "shared", true) + .expect("leaf export registers"); + let middle_own = graph + .add_declaration(middle, "middle_own", true) + .expect("middle export registers"); + + // middle imports leaf's export; root imports middle's export. + for (importer, source, source_symbol, source_name) in [ + (middle, leaf, shared, "shared"), + (root, middle, middle_own, "middle_own"), + ] { + graph + .add_imported_binding( + importer, + ImportedBinding { + local_name: source_name.to_string(), + source_module: source, + source_symbol, + source_name: source_name.to_string(), + }, + ) + .expect("imported binding registers"); + } + + assert_eq!( + graph.symbol_for_export(middle, "shared"), + None, + "middle's export table must not re-export leaf's function" + ); + assert_eq!( + graph.symbol_for_export(root, "shared"), + None, + "root's export table must not see leaf's function through middle" + ); + assert_eq!( + graph.symbol_for_export(middle, "middle_own"), + Some(middle_own), + "middle's own public declaration stays in its export table" + ); + assert_eq!( + graph.symbol_for_export(root, "middle_own"), + None, + "root's export table is empty: direct imports become bindings, not exports" + ); + assert!(graph.imported_binding(root, "middle_own").is_some()); + assert!(graph.imported_binding(middle, "shared").is_some()); + assert!( + graph + .node(root) + .expect("root node exists") + .exports + .is_empty() + ); + } + + #[test] + fn imported_binding_clashing_with_local_declaration_is_rejected() { + let mut graph = ModuleGraph::new(); + let importer = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + let source = graph.add_node( + PathBuf::from("/root/util.rss"), + "/root/util.rss".to_string(), + Vec::new(), + ); + let exported = graph + .add_declaration(source, "helper", true) + .expect("source export registers"); + graph + .add_declaration(importer, "helper", true) + .expect("local declaration registers"); + + let err = graph + .add_imported_binding( + importer, + ImportedBinding { + local_name: "helper".to_string(), + source_module: source, + source_symbol: exported, + source_name: "helper".to_string(), + }, + ) + .expect_err("binding a name already declared locally must be rejected"); + assert!( + err.contains("conflicts with a local declaration"), + "unexpected error: {err}" + ); + } + + #[test] + fn duplicate_imported_bindings_are_recorded_like_the_legacy_merge() { + // Two modules exporting the same name, both imported into one module: + // the legacy pipeline merges such imports by name, so the graph keeps + // every binding instead of rejecting the second one. + let mut graph = ModuleGraph::new(); + let importer = graph.add_node( + PathBuf::from("/root/main.rss"), + "/root/main.rss".to_string(), + Vec::new(), + ); + let first = graph.add_node( + PathBuf::from("/root/a/util.rss"), + "/root/a/util.rss".to_string(), + Vec::new(), + ); + let second = graph.add_node( + PathBuf::from("/root/b/util.rss"), + "/root/b/util.rss".to_string(), + Vec::new(), + ); + let first_helper = graph + .add_declaration(first, "helper", true) + .expect("first export registers"); + let second_helper = graph + .add_declaration(second, "helper", true) + .expect("second export registers"); + graph + .add_imported_binding( + importer, + ImportedBinding { + local_name: "helper".to_string(), + source_module: first, + source_symbol: first_helper, + source_name: "helper".to_string(), + }, + ) + .expect("first binding registers"); + graph + .add_imported_binding( + importer, + ImportedBinding { + local_name: "helper".to_string(), + source_module: second, + source_symbol: second_helper, + source_name: "helper".to_string(), + }, + ) + .expect("second binding with the same local name registers"); + + let node = graph.node(importer).expect("importer node exists"); + assert_eq!(node.imported_bindings.len(), 2); + assert_eq!(node.imported_bindings[0].source_module, first); + assert_eq!(node.imported_bindings[1].source_module, second); + assert_eq!( + node.declarations.len(), + 0, + "bindings are never declarations" + ); + assert!(node.exports.is_empty(), "bindings are never exports"); + } +} diff --git a/src/compiler/parser/expressions.rs b/src/compiler/parser/expressions.rs index 451b1d57..e9c6217e 100644 --- a/src/compiler/parser/expressions.rs +++ b/src/compiler/parser/expressions.rs @@ -398,7 +398,7 @@ impl Parser { .get(1..) .map(|tail| tail.to_vec()) .unwrap_or_default(); - if let Some((builtin_namespace, builtin_member)) = + let expr = if let Some((builtin_namespace, builtin_member)) = self.resolve_builtins_call_path(&name, &member, &subpath) { let builtin_namespace = builtin_namespace.to_string(); @@ -406,23 +406,37 @@ impl Parser { if let Some(builtin) = resolve_builtin_namespace_call(&builtin_namespace, &builtin_member) { - let expr = - self.build_builtin_call_expr_with_type_args(builtin, args, type_args)?; - return Ok(expr); + self.build_builtin_call_expr_with_type_args(builtin, args, type_args)? + } else { + return Err(ParseError { + span: None, + code: None, + line: self.current_line(), + message: format!( + "unknown builtin function '{}::{}'", + builtin_namespace, builtin_member + ), + }); } + } else if let Some(host_name) = + self.resolve_host_namespace_call_target(&name, &member, &subpath) + { + self.build_host_call_expr_with_type_args(&host_name, args, type_args)? + } else if self.allow_implicit_externs + && self.module_namespace_alias(&name).is_some() + { + // File-module namespace call (`alias::member(...)`): the + // parser cannot resolve the member against the target + // module's exports — only the source loader can. Emit an + // implicit extern carrying the qualified name; the loader + // resolves the call to a `ModuleCall` or rejects it. + // Type-argument validation is deferred to the loader, + // which knows the exported type parameters. + let qualified = format!("{}::{}", name, path_segments.join("::")); + let decl = self.resolve_function_for_call(&qualified, args.len())?; + Expr::Call(decl.index, type_args, args) + } else { return Err(ParseError { - span: None, - code: None, - line: self.current_line(), - message: format!( - "unknown builtin function '{}::{}'", - builtin_namespace, builtin_member - ), - }); - } - let host_name = self - .resolve_host_namespace_call_target(&name, &member, &subpath) - .ok_or_else(|| ParseError { span: None, code: None, line: self.current_line(), @@ -432,8 +446,11 @@ impl Parser { path_segments.join("::"), builtin_namespace_hint() ), - })?; - let expr = self.build_host_call_expr_with_type_args(&host_name, args, type_args)?; + }); + }; + // Namespace calls participate in postfix access like any + // other call (`iter::range(n)[0]`, `json::decode::(s).x`). + let expr = self.parse_postfix_access(expr)?; return Ok(expr); } @@ -488,7 +505,11 @@ impl Parser { } } else { let decl = self.resolve_function_for_call(&name, args.len())?; - self.validate_named_call_type_args(&decl, &type_args)?; + // Implicit externs mirror imported calls the loader + // validates against the exported signature. + if !self.is_implicit_extern(&name) { + self.validate_named_call_type_args(&decl, &type_args)?; + } Expr::Call(decl.index, type_args, args) } } else if let Some(expr) = self.try_build_language_builtin_call(&name, &args)? { @@ -507,7 +528,14 @@ impl Parser { self.build_host_call_expr_with_type_args(&host_name, args, type_args)? } else { let decl = self.resolve_function_for_call(&name, args.len())?; - self.validate_named_call_type_args(&decl, &type_args)?; + // Import-scan and module-mode parses tolerate type + // arguments on calls whose target only the source + // loader can type; the loader validates them against + // the exported signature. Plain compile parses + // validate locally declared functions normally. + if !self.import_scan_mode && !self.is_implicit_extern(&name) { + self.validate_named_call_type_args(&decl, &type_args)?; + } Expr::Call(decl.index, type_args, args) } } else { @@ -541,6 +569,11 @@ impl Parser { }); } Expr::FunctionRef(index, Vec::new()) + } else if self.allow_implicit_externs { + // Module mode: the name may be an imported function + // binding the loader resolves to a module symbol + // (`Expr::ModuleFunctionRef`) before unit merge. + Expr::UnresolvedFunctionRef { name, type_args } } else { return Err(ParseError { span: None, @@ -1610,7 +1643,12 @@ impl Parser { args: Vec, type_args: Vec, ) -> Result { - self.validate_host_call_type_args(host_name, &type_args)?; + // Import-scan parses discard their IR; the compile parse validates + // host type arguments unless the namespace may name a file module + // (deferred to the source loader, which knows the exports). + if !self.import_scan_mode && self.host_type_args_validated_at_parse(host_name) { + self.validate_host_call_type_args(host_name, &type_args)?; + } let arity = u8::try_from(args.len()).map_err(|_| ParseError { span: None, code: None, @@ -1621,6 +1659,20 @@ impl Parser { Ok(Expr::Call(decl.index, type_args, args)) } + /// Whether host type arguments are validated at parse time. + /// + /// Single-segment import forms (`use module;`, `use module::{wrap}`) may + /// name a file module; the parser cannot know, so calls through such + /// namespaces defer type-argument validation to the source loader, which + /// validates against the module's exported type parameters. Builtin + /// namespaces keep their parse-time validation. + fn host_type_args_validated_at_parse(&self, host_name: &str) -> bool { + match host_name.split_once("::") { + Some((namespace, _)) => is_builtin_namespace(namespace), + None => true, + } + } + pub(super) fn contextualize_function_value( &self, expr: &mut Expr, @@ -2392,7 +2444,7 @@ fn builtin_generic_type_arg_arity(builtin: BuiltinFunction) -> GenericCallableTy } } -fn host_generic_type_arg_arity(host_name: &str) -> Option { +pub(crate) fn host_generic_type_arg_arity(host_name: &str) -> Option { match host_name { "json::decode" => Some(1), _ => None, diff --git a/src/compiler/parser/mod.rs b/src/compiler/parser/mod.rs index f989111e..98f6b40d 100644 --- a/src/compiler/parser/mod.rs +++ b/src/compiler/parser/mod.rs @@ -15,8 +15,10 @@ use crate::builtins::{ BuiltinFunction, builtin_namespace_hint, default_host_callable, is_builtin_namespace, resolve_builtin_namespace_call, }; +use crate::compiler::modules::{UseDecl, UsePathSegment}; use crate::compiler::source_map::{SourceId, Span}; +pub(crate) use self::expressions::host_generic_type_arg_arity; use self::lexer::{Lexer, ParserFormatArg, Token, TokenKind, is_ident_continue, is_ident_start}; use self::symbols::is_virtual_host_namespace_spec; use super::{ @@ -138,6 +140,19 @@ pub(super) struct Parser { host_namespace_aliases: HashMap, direct_host_call_aliases: HashMap, direct_host_wildcard_imports: HashSet, + /// Names created through the implicit-extern fallback (module mode). + /// The source loader uses this marker to keep synthetic externs out of + /// module declaration/export tables and to resolve (or reject) their + /// call sites. + implicit_extern_names: HashSet, + /// Namespace aliases introduced by file-module `use` directives. + /// + /// Unlike [`Parser::host_namespace_aliases`] these are recorded in every + /// parse mode; a namespace call that is neither builtin nor host resolves + /// through this map into a loader-resolved module call placeholder. + module_namespace_aliases: HashMap, + use_declarations: Vec, + import_scan_mode: bool, mutable_locals: Vec, borrowed_map_iter_locals: Vec, local_schemas: HashMap, @@ -155,6 +170,7 @@ impl Parser { allow_implicit_externs: bool, allow_implicit_semicolons: bool, enforce_mutable_bindings: bool, + import_scan_mode: bool, dialect: &'static dyn ParserDialect, ) -> Result { let mut lexer = Lexer::new(source, source_id, dialect); @@ -193,6 +209,10 @@ impl Parser { host_namespace_aliases: HashMap::new(), direct_host_call_aliases: HashMap::new(), direct_host_wildcard_imports: HashSet::new(), + implicit_extern_names: HashSet::new(), + module_namespace_aliases: HashMap::new(), + use_declarations: Vec::new(), + import_scan_mode, mutable_locals: Vec::new(), borrowed_map_iter_locals: Vec::new(), local_schemas: HashMap::new(), @@ -214,6 +234,7 @@ impl Parser { allow_implicit_externs, allow_implicit_semicolons, enforce_mutable_bindings, + false, dialect, )?; for binding in predeclared_locals { @@ -222,6 +243,10 @@ impl Parser { Ok(parser) } + pub(super) fn use_declarations(&self) -> Vec { + self.use_declarations.clone() + } + pub(super) fn parse_program(&mut self) -> Result, ParseError> { self.predeclare_functions()?; let mut stmts = Vec::new(); @@ -336,6 +361,7 @@ impl Parser { type_params, exported, return_type: ValueType::Unknown, + symbol: None, }; self.functions.insert(name, decl.clone()); self.function_list.push(decl); @@ -388,6 +414,28 @@ impl Parser { self.unknown_type_spans.clone() } + pub(super) fn implicit_extern_names(&self) -> Vec { + let mut names = self + .implicit_extern_names + .iter() + .cloned() + .collect::>(); + names.sort(); + names + } + + pub(super) fn is_implicit_extern(&self, name: &str) -> bool { + self.implicit_extern_names.contains(name) + } + + /// Look up a file-module namespace alias recorded from a structured + /// `use` directive (both parse modes). + pub(super) fn module_namespace_alias(&self, namespace: &str) -> Option<&str> { + self.module_namespace_aliases + .get(namespace) + .map(String::as_str) + } + fn validate_schema_reference_sites(&self) -> Result<(), ParseError> { for (name, arg_count, line, span) in &self.schema_reference_sites { let Some(decl) = self.struct_schemas.get(name) else { diff --git a/src/compiler/parser/statements.rs b/src/compiler/parser/statements.rs index 53811c91..92e24cb3 100644 --- a/src/compiler/parser/statements.rs +++ b/src/compiler/parser/statements.rs @@ -1,5 +1,17 @@ use super::*; +use crate::compiler::source_loader::{ImportClause, NamedImport}; + +/// Classify a `use` path segment: leading `self`/`super` words become +/// qualifiers; every other segment is a plain identifier. +fn classify_use_segment(segment: &str) -> UsePathSegment { + match segment { + "self" => UsePathSegment::Self_, + "super" => UsePathSegment::Super, + _ => UsePathSegment::Ident(segment.to_string()), + } +} + impl Parser { pub(super) fn parse_stmt(&mut self) -> Result { if self.match_kind(&TokenKind::Pub) { @@ -94,21 +106,42 @@ impl Parser { pub(super) fn parse_use_stmt(&mut self) -> Result { let line = self.last_line(); - let namespace = self.expect_ident("expected namespace after 'use'")?; - if self.match_kind(&TokenKind::Semicolon) { - self.host_namespace_aliases - .insert(namespace.clone(), namespace); - return Ok(Stmt::Noop { line }); - } + let directive_start = self + .tokens + .get(self.pos.saturating_sub(1)) + .map(|token| token.span.lo) + .unwrap_or(0); - if self.match_kind(&TokenKind::As) { - let alias = self.expect_ident("expected namespace alias after 'as'")?; - self.expect(&TokenKind::Semicolon, "expected ';' after use alias")?; - self.host_namespace_aliases.insert(alias, namespace); - return Ok(Stmt::Noop { line }); - } + let namespace = self.expect_ident("expected namespace after 'use'")?; if !self.match_path_separator() { + // Single-segment host-namespace forms: `use io;`, `use io as x;`. + if self.match_kind(&TokenKind::Semicolon) { + self.host_namespace_aliases + .insert(namespace.clone(), namespace.clone()); + self.record_use_decl( + vec![classify_use_segment(&namespace)], + ImportClause::AllPublic, + line, + directive_start, + ); + return Ok(Stmt::Noop { line }); + } + + if self.match_kind(&TokenKind::As) { + let alias = self.expect_ident("expected namespace alias after 'as'")?; + self.expect(&TokenKind::Semicolon, "expected ';' after use alias")?; + self.host_namespace_aliases + .insert(alias.clone(), namespace.clone()); + self.record_use_decl( + vec![classify_use_segment(&namespace)], + ImportClause::Namespace(alias), + line, + directive_start, + ); + return Ok(Stmt::Noop { line }); + } + return Err(ParseError { span: None, code: None, @@ -130,45 +163,210 @@ impl Parser { }); } + if namespace == "crate" { + return Err(ParseError { + span: None, + code: None, + line: self.current_line(), + message: "crate:: paths are not supported; use relative module paths".to_string(), + }); + } + if self.match_kind(&TokenKind::Star) { - self.direct_host_wildcard_imports.insert(namespace); + self.direct_host_wildcard_imports.insert(namespace.clone()); self.expect( &TokenKind::Semicolon, "expected ';' after host wildcard import", )?; + self.record_use_decl( + vec![classify_use_segment(&namespace)], + ImportClause::AllPublic, + line, + directive_start, + ); return Ok(Stmt::Noop { line }); } - self.expect(&TokenKind::LBrace, "expected '{' after host import path")?; - if self.match_kind(&TokenKind::Star) { - self.direct_host_wildcard_imports.insert(namespace); - self.expect(&TokenKind::RBrace, "expected '}' after '*'")?; + if self.match_kind(&TokenKind::LBrace) { + let named = self.parse_use_named_list(&namespace, true)?; + self.expect(&TokenKind::RBrace, "expected '}' after use list")?; self.expect(&TokenKind::Semicolon, "expected ';' after use list")?; + self.record_use_decl( + vec![classify_use_segment(&namespace)], + ImportClause::Named(named), + line, + directive_start, + ); return Ok(Stmt::Noop { line }); } + // Multi-segment file-module path: `use a::b;`, `use a::b::*;`, + // `use a::b::{x};`, `use a::b as alias;`, with optional leading + // `self`/`super` qualifiers. These directives are consumed as + // structured nodes; the source loader resolves them against the + // module graph, so no host aliases are recorded here. + let mut path = vec![classify_use_segment(&namespace)]; + let mut qualifier_run = matches!(path[0], UsePathSegment::Self_ | UsePathSegment::Super); loop { - let imported = self.expect_ident("expected host function name in use list")?; - let local = if self.match_kind(&TokenKind::As) { - self.expect_ident("expected local alias after 'as'")? - } else { - imported.clone() - }; - let target = format!("{namespace}::{imported}"); - if let Some(existing) = self.direct_host_call_aliases.get(&local) - && existing != &target - { + if self.match_kind(&TokenKind::Star) { + self.expect(&TokenKind::Semicolon, "expected ';' after use wildcard")?; + self.record_use_decl(path, ImportClause::AllPublic, line, directive_start); + return Ok(Stmt::Noop { line }); + } + if self.match_kind(&TokenKind::LBrace) { + let named = self.parse_use_named_list(&namespace, false)?; + self.expect(&TokenKind::RBrace, "expected '}' after use list")?; + self.expect(&TokenKind::Semicolon, "expected ';' after use list")?; + self.record_use_decl(path, ImportClause::Named(named), line, directive_start); + return Ok(Stmt::Noop { line }); + } + let segment = self.expect_ident("expected module path segment after '::'")?; + if qualifier_run && segment == "crate" { return Err(ParseError { span: None, code: None, line: self.current_line(), - message: format!( - "host import alias '{local}' already maps to '{existing}', cannot remap to '{target}'" - ), + message: "crate:: paths are not supported; use relative module paths" + .to_string(), }); } - self.direct_host_call_aliases.insert(local, target); + let classified = if qualifier_run { + classify_use_segment(&segment) + } else { + UsePathSegment::Ident(segment) + }; + qualifier_run = matches!(classified, UsePathSegment::Self_ | UsePathSegment::Super); + path.push(classified); + if !self.match_path_separator() { + break; + } + } + + if self.match_kind(&TokenKind::As) { + let alias = self.expect_ident("expected namespace alias after 'as'")?; + self.expect(&TokenKind::Semicolon, "expected ';' after use alias")?; + self.record_use_decl(path, ImportClause::Namespace(alias), line, directive_start); + } else { + self.expect(&TokenKind::Semicolon, "expected ';' after use directive")?; + self.record_use_decl(path, ImportClause::AllPublic, line, directive_start); + } + Ok(Stmt::Noop { line }) + } + + /// Record a structured `use` declaration with its directive span. + /// + /// In import-scan mode (source-loader discovery) the parser also records + /// host aliases for file-module paths so that namespace calls parse + /// before the loader's resolution pass runs; the compile parse runs with + /// scan mode off and resolves module namespaces through the structured + /// declarations instead. The alias is the clause alias for namespace + /// imports and the default namespace (last path segment) for all-public + /// and named imports. + fn record_use_decl( + &mut self, + path: Vec, + clause: ImportClause, + line: u32, + directive_start: usize, + ) { + let source_id = self.current_span().source_id; + let end = self + .tokens + .get(self.pos.saturating_sub(1)) + .map(|token| token.span.hi) + .unwrap_or(directive_start); + if self.import_scan_mode { + let alias = match &clause { + ImportClause::Namespace(alias) => Some(alias.clone()), + ImportClause::AllPublic | ImportClause::Named(_) => match path.last() { + Some(UsePathSegment::Ident(name)) => Some(name.clone()), + _ => None, + }, + ImportClause::Prefix(_) => None, + }; + if let Some(alias) = alias { + let joined = path + .iter() + .map(|segment| match segment { + UsePathSegment::Self_ => "self".to_string(), + UsePathSegment::Super => "super".to_string(), + UsePathSegment::Ident(name) => name.clone(), + }) + .collect::>() + .join("::"); + self.host_namespace_aliases.insert(alias, joined); + } + } + // File-module namespace aliases are recorded in every parse mode so + // the compile parse can recognize `alias::member(...)` calls and emit + // a loader-resolved placeholder. The alias mirrors the loader's + // clause-based namespace table: the `as` alias for namespace + // imports, the last path segment for all-public imports, and no + // namespace for named imports (which bind direct names only). + let module_alias = match &clause { + ImportClause::Namespace(alias) => Some(alias.clone()), + ImportClause::AllPublic => match path.last() { + Some(UsePathSegment::Ident(name)) => Some(name.clone()), + _ => None, + }, + ImportClause::Named(_) | ImportClause::Prefix(_) => None, + }; + if let Some(alias) = module_alias { + let joined = path + .iter() + .map(|segment| match segment { + UsePathSegment::Self_ => "self".to_string(), + UsePathSegment::Super => "super".to_string(), + UsePathSegment::Ident(name) => name.clone(), + }) + .collect::>() + .join("::"); + self.module_namespace_aliases.insert(alias, joined); + } + self.use_declarations.push(UseDecl { + path, + clause, + span: Span::new(source_id, directive_start, end), + line: line as usize, + }); + } + /// Parse the named list of a `use` directive. + /// + /// Single-segment forms keep the legacy host behavior: every binding is + /// recorded as a direct host call alias (`use io::{read};` maps `read` to + /// `io::read`). Multi-segment file-module forms only collect the bindings; + /// the module graph's resolution pass owns their resolution. + fn parse_use_named_list( + &mut self, + namespace: &str, + record_host_aliases: bool, + ) -> Result, ParseError> { + let mut named = Vec::::new(); + loop { + let imported = self.expect_ident("expected host function name in use list")?; + let local = if self.match_kind(&TokenKind::As) { + self.expect_ident("expected local alias after 'as'")? + } else { + imported.clone() + }; + if record_host_aliases { + let target = format!("{namespace}::{imported}"); + if let Some(existing) = self.direct_host_call_aliases.get(&local) + && existing != &target + { + return Err(ParseError { + span: None, + code: None, + line: self.current_line(), + message: format!( + "host import alias '{local}' already maps to '{existing}', cannot remap to '{target}'" + ), + }); + } + self.direct_host_call_aliases.insert(local.clone(), target); + } + named.push(NamedImport { imported, local }); if self.match_kind(&TokenKind::Comma) { if self.check(&TokenKind::RBrace) { break; @@ -177,9 +375,7 @@ impl Parser { } break; } - self.expect(&TokenKind::RBrace, "expected '}' after use list")?; - self.expect(&TokenKind::Semicolon, "expected ';' after use list")?; - Ok(Stmt::Noop { line }) + Ok(named) } pub(super) fn parse_js_import_stmt(&mut self) -> Result { @@ -494,6 +690,7 @@ impl Parser { type_params: type_params.clone(), exported, return_type, + symbol: None, }; self.functions.insert(name.clone(), decl.clone()); let current_line = self.current_line(); diff --git a/src/compiler/parser/symbols.rs b/src/compiler/parser/symbols.rs index 35adcd84..8290e75f 100644 --- a/src/compiler/parser/symbols.rs +++ b/src/compiler/parser/symbols.rs @@ -280,6 +280,7 @@ impl Parser { type_params: Vec::new(), exported: true, return_type: ValueType::Unknown, + symbol: None, }; self.functions.insert(name.to_string(), decl.clone()); self.function_list.push(decl.clone()); @@ -310,6 +311,10 @@ impl Parser { message: format!("name '{name}' already used by a local binding"), }); } + // The module loader resolves (or rejects) every implicit extern's + // call sites; the marker keeps synthetic externs out of module + // declaration/export tables. + self.implicit_extern_names.insert(name.to_string()); let index = self.next_function; self.next_function = self.next_function.checked_add(1).ok_or(ParseError { span: None, @@ -328,6 +333,7 @@ impl Parser { type_params: Vec::new(), exported: true, return_type: ValueType::Unknown, + symbol: None, }; self.functions.insert(name.to_string(), decl.clone()); self.function_list.push(decl.clone()); @@ -376,6 +382,7 @@ impl Parser { type_params: Vec::new(), exported: false, return_type: known_host_return_type(name), + symbol: None, }; self.functions.insert(name.to_string(), decl.clone()); self.function_list.push(decl.clone()); diff --git a/src/compiler/pipeline.rs b/src/compiler/pipeline.rs index 7f98363d..dda97e63 100644 --- a/src/compiler/pipeline.rs +++ b/src/compiler/pipeline.rs @@ -7,7 +7,8 @@ use super::ReplLocalState; use super::codegen::Compiler; use super::frontends; use super::ir::{Expr, FrontendIr, FunctionDecl, FunctionImpl, LocalSlot, Stmt, TypeSchema}; -use super::linker::merge_units; +use super::linker::{ParsedUnit, merge_units}; +use super::modules::ModuleGraph; use super::source_loader::load_units_for_source_file; use super::source_map::SourceMap; use super::{ @@ -173,7 +174,9 @@ fn record_expr_local_debug_ranges( | Expr::Bool(_) | Expr::Bytes(_) | Expr::String(_) - | Expr::FunctionRef(..) => {} + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => {} Expr::Var(index) | Expr::MoveVar(index) => { note_local_use(ranges, *index, line); } @@ -200,7 +203,7 @@ fn record_expr_local_debug_ranges( record_expr_local_debug_ranges(value, line, ranges); record_expr_local_debug_ranges(fallback, line, ranges); } - Expr::Call(_, _, args) => { + Expr::Call(_, _, args) | Expr::ModuleCall(_, _, args) => { for arg in args { record_expr_local_debug_ranges(arg, line, ranges); } @@ -579,7 +582,9 @@ fn schema_is_fully_known(schema: &TypeSchema) -> bool { | TypeSchema::GenericParam(_) => true, TypeSchema::Optional(inner) => schema_is_fully_known(inner), TypeSchema::Named(_, type_args) => type_args.iter().all(schema_is_fully_known), - TypeSchema::Array(item) | TypeSchema::Map(item) => schema_is_fully_known(item), + TypeSchema::Array(item) | TypeSchema::Map(item) => { + matches!(item.as_ref(), TypeSchema::Unknown) || schema_is_fully_known(item) + } TypeSchema::ArrayTuple(items) => items.iter().all(schema_is_fully_known), TypeSchema::ArrayTupleRest { prefix, rest } => { prefix.iter().all(schema_is_fully_known) && schema_is_fully_known(rest) @@ -867,8 +872,9 @@ fn lint_unknown_inferred_local_types_at_path_with_options_impl( ) -> Result, SourcePathError> { let mut source_map = SourceMap::new(); let source_id = source_map.add_source(path.display().to_string(), source.to_string()); - let (_root_parse_source, units) = load_units_for_source_file(path, flavor, source, options)?; - let parsed = units + let loaded = load_units_for_source_file(path, flavor, source, options)?; + let parsed = loaded + .units .into_iter() .last() .map(|unit| unit.parsed) @@ -886,8 +892,9 @@ fn collect_inferred_local_type_hints_at_path_with_options_impl( flavor: SourceFlavor, options: &CompileSourceFileOptions, ) -> Result, SourcePathError> { - let (_root_parse_source, units) = load_units_for_source_file(path, flavor, source, options)?; - let parsed = units + let loaded = load_units_for_source_file(path, flavor, source, options)?; + let parsed = loaded + .units .into_iter() .last() .map(|unit| unit.parsed) @@ -1319,6 +1326,45 @@ fn compile_source_with_flavor_impl( } } +fn compile_loaded_units( + source: String, + units: Vec, + flavor: SourceFlavor, + // Carried from the loader for Milestone 2+ (structured imports, symbol + // resolution); codegen output is unchanged until then. + _module_graph: ModuleGraph, + // Compilation-wide source map keyed by the module graph's `SourceId` + // space (milestone 5). Every span produced during load/merge references + // this map, so errors are returned with it and render from the owning + // source. + sources: SourceMap, +) -> Result { + let diagnostic_path = units + .iter() + .find(|unit| !unit.parsed.unknown_type_spans.is_empty()) + .map(|unit| PathBuf::from(&unit.source_name)); + let merged = merge_units(units)?; + compile_parsed_output( + source, + merged, + CompileBehavior::DEFAULT, + TypingMode::for_flavor(flavor), + matches!(flavor, SourceFlavor::RustScript), + ) + .map_err(|error| match (error, diagnostic_path) { + (SourceError::Parse(mut parse), Some(path)) + if parse.code.as_deref() == Some("E_STRICT_UNKNOWN_TYPE") => + { + parse.message = format!("{}: {}", path.display(), parse.message); + SourcePathError::SourceWithMap { + error: SourceError::Parse(parse), + sources, + } + } + (error, _) => SourcePathError::SourceWithMap { error, sources }, + }) +} + fn compile_source_with_flavor_and_options_impl( source: &str, flavor: SourceFlavor, @@ -1330,16 +1376,14 @@ fn compile_source_with_flavor_and_options_impl( } let path = virtual_inmemory_entry_path(flavor); - let (_root_parse_source, units) = load_units_for_source_file(&path, flavor, source, options)?; - let merged = merge_units(units)?; - compile_parsed_output( + let loaded = load_units_for_source_file(&path, flavor, source, options)?; + compile_loaded_units( source.to_string(), - merged, - CompileBehavior::DEFAULT, - TypingMode::for_flavor(flavor), - matches!(flavor, SourceFlavor::RustScript), + loaded.units, + flavor, + loaded.module_graph, + loaded.sources, ) - .map_err(SourcePathError::Source) } fn compile_source_at_path_with_flavor_and_options_impl( @@ -1348,16 +1392,14 @@ fn compile_source_at_path_with_flavor_and_options_impl( flavor: SourceFlavor, options: &CompileSourceFileOptions, ) -> Result { - let (_root_parse_source, units) = load_units_for_source_file(path, flavor, source, options)?; - let merged = merge_units(units)?; - compile_parsed_output( + let loaded = load_units_for_source_file(path, flavor, source, options)?; + compile_loaded_units( source.to_string(), - merged, - CompileBehavior::DEFAULT, - TypingMode::for_flavor(flavor), - matches!(flavor, SourceFlavor::RustScript), + loaded.units, + flavor, + loaded.module_graph, + loaded.sources, ) - .map_err(SourcePathError::Source) } fn virtual_inmemory_entry_path(flavor: SourceFlavor) -> PathBuf { @@ -1387,17 +1429,14 @@ fn compile_source_file_impl( ) -> Result { let flavor = SourceFlavor::from_path_with_options(path, options)?; let source_raw = std::fs::read_to_string(path)?; - let (_root_parse_source, units) = - load_units_for_source_file(path, flavor, &source_raw, options)?; - let merged = merge_units(units)?; - compile_parsed_output( + let loaded = load_units_for_source_file(path, flavor, &source_raw, options)?; + compile_loaded_units( source_raw, - merged, - CompileBehavior::DEFAULT, - TypingMode::for_flavor(flavor), - matches!(flavor, SourceFlavor::RustScript), + loaded.units, + flavor, + loaded.module_graph, + loaded.sources, ) - .map_err(SourcePathError::Source) } fn run_with_compiler_stack(f: F) -> T diff --git a/src/compiler/source_loader.rs b/src/compiler/source_loader.rs index 5478bae2..1228550b 100644 --- a/src/compiler/source_loader.rs +++ b/src/compiler/source_loader.rs @@ -1,7 +1,49 @@ +//! Semantic file-module loading. +//! +//! This module is the sole file-module path of the compiler (milestone 6): +//! there is no textual import rewriting, no synthetic imported-function +//! prelude, and no prelude line-map remapping anymore. Every module source is +//! parsed verbatim with the real frontend parser; `use` directives become +//! structured [`UseDecl`](crate::compiler::modules::UseDecl) nodes, the +//! [`ModuleGraph`] assigns deterministic [`ModuleId`]s/[`SourceId`]s and +//! records import edges, exports, and imported bindings, and calls to +//! imported functions are resolved to [`SymbolId`]s before unit merge. +//! +//! ## Load pipeline +//! +//! 1. `collect_module_units` discovers the import graph from the root, +//! registering every module (disk or in-memory override) in the +//! [`ModuleGraph`] and its raw text in the compilation-wide +//! [`SourceMap`](crate::compiler::source_map::SourceMap) at its graph +//! `SourceId`. +//! 2. Each module is parsed in module mode (implicit-extern fallback on): +//! calls the parser cannot resolve locally — imported module functions, +//! module namespace members, imported function values — parse into +//! synthetic externs (tracked on `FrontendIr::implicit_extern_names`) or +//! `Expr::UnresolvedFunctionRef`, and are resolved afterwards. +//! 3. `record_module_symbols` assigns every real declaration its owned +//! [`SymbolId`], fills the module's public export table and imported +//! binding table, then resolves every call site to an `Expr::ModuleCall` +//! or `Expr::ModuleFunctionRef` carrying the target symbol, validating +//! arity and type arguments against the exported signature. +//! 4. `linker::merge_units` merges units by symbol identity and applies the +//! deterministic flat-boundary mangling only there. +//! +//! Host namespace imports (`use io;`, `use myhost;`) never enter the textual +//! machinery: the parser keeps their dedicated host resolution path and the +//! loader records them as host/builtin import edges. Single-segment imports +//! that may name a file module (`use module;`) parse as host-form calls and +//! are fixed up by the loader when the spec resolves to a file module. +//! +//! Every span produced here references the owning module's graph `SourceId` +//! in the compilation-wide map, so diagnostics always render from the +//! owning source. + use std::path::Path; use crate::compiler::source_map::SourceMap; +use super::modules::ModuleGraph; use super::{ CompileSourceFileOptions, SourceError, SourceFlavor, SourcePathError, frontends, linker::ParsedUnit, @@ -9,68 +51,547 @@ use super::{ mod graph; mod imports; -mod line_map; mod model; -mod rewrite; -use graph::{build_rustscript_import_prelude, collect_module_units}; -use imports::{parse_module_imports, strip_import_directives}; -use line_map::remap_frontend_ir_line_numbers; +use graph::{collect_module_units, record_module_symbols}; +use imports::{module_identity, parse_module_imports, strip_import_directives}; use model::ModuleCollectState; pub use model::{FrontendImportSyntax, ImportClause, ModuleImport, NamedImport}; -use rewrite::rewrite_imported_call_sites; +pub(super) struct LoadedSourceUnits { + pub(super) units: Vec, + /// Semantic module graph built during discovery (milestones 1-3). + pub(super) module_graph: ModuleGraph, + /// Compilation-wide source map keyed by the module graph's `SourceId` + /// space (milestone 5). Every module's raw text is registered here at + /// its graph source id, so spans carried by the loaded units and by any + /// load-time diagnostic resolve to the owning source. + pub(super) sources: SourceMap, +} pub(super) fn load_units_for_source_file( path: &Path, flavor: SourceFlavor, source_raw: &str, options: &CompileSourceFileOptions, -) -> Result<(String, Vec), SourcePathError> { - let root_imports = parse_module_imports(source_raw, flavor, path, options)?; - let source = strip_import_directives(source_raw, flavor, options)?; +) -> Result { + // The root participates in the same identity scheme as every module: + // canonical disk identity when the file exists, normalized virtual + // identity otherwise. This keeps `seen`/`visiting`/exports/overrides + // keyed uniformly across the whole import graph. + let path = module_identity(path.to_path_buf()); + let path = path.as_path(); let mut collect_state = ModuleCollectState::default(); + // Pre-register the root text at its graph source id. The root node is + // always registered first (SourceId(0)); registering the text here lets + // the root's own scan/parse diagnostics attach spans against the + // compilation-wide map before collection runs. + collect_state + .sources + .add_source_at(0, path.display().to_string(), source_raw.to_string()); collect_state.visiting.push(path.to_path_buf()); - collect_module_units(path, source_raw, flavor, options, &mut collect_state)?; - let rewritten_root = rewrite_imported_call_sites( - &source, + let root_imports = parse_module_imports(source_raw, flavor, path, options).map_err(|err| { + // The root's own scan/parse diagnostics attach their span against + // the pre-registered root source and carry the compilation-wide map, + // so they render from the root's text. + match err { + SourcePathError::Source(SourceError::Parse(mut parse)) => { + parse.span = None; + parse = parse.with_line_span_from_source(&collect_state.sources, 0); + SourcePathError::SourceWithMap { + error: SourceError::Parse(parse), + sources: collect_state.sources.clone(), + } + } + other => other, + } + })?; + + collect_module_units(path, source_raw, flavor, options, &mut collect_state).map_err(|err| { + // Load-time source diagnostics (nested scan/parse errors, symbol + // resolution, imported-call resolution) already carry spans keyed to + // the compilation-wide map; attach the map so they render from the + // owning source. + match err { + SourcePathError::Source(error) => SourcePathError::SourceWithMap { + error, + sources: collect_state.sources.clone(), + }, + other => other, + } + })?; + let root_module = collect_state + .module_graph + .module_id_for_identity(path) + .expect("root module should be registered in the module graph"); + let root_source_id = collect_state + .module_graph + .node(root_module) + .map(|node| node.source.0) + .unwrap_or(0); + let root_parse_source = strip_import_directives(source_raw, flavor, options)?; + + let mut root_parsed = frontends::parse_module_source_with_source_id( + &root_parse_source, flavor, - path, - &root_imports, - &collect_state.module_exports, options, - )?; - let mut prelude = build_rustscript_import_prelude( + root_source_id, + ) + .map_err(|mut err| { + // Module sources are parsed verbatim (no synthetic prelude, no + // textual rewrite), so parse lines already refer to the owning + // source; rebuild the span against the compilation-wide map so the + // diagnostic renders from the root's text. + err.span = None; + let parse = err.with_line_span_from_source(&collect_state.sources, root_source_id); + SourcePathError::SourceWithMap { + error: SourceError::Parse(parse), + sources: collect_state.sources.clone(), + } + })?; + record_module_symbols( + &mut collect_state, + root_module, path, &root_imports, - &collect_state.module_exports, + &mut root_parsed, options, - )?; - let root_prelude_lines = prelude.lines().count(); - prelude.push_str(&rewritten_root.source); - let root_parse_source = prelude; - - let mut root_source_map = SourceMap::new(); - let root_source_id = root_source_map.add_source(path.display().to_string(), source_raw); - let mut root_parsed = frontends::parse_source(&root_parse_source, flavor, options) - .map_err(|mut err| { - if root_prelude_lines > 0 { - err.line = err.line.saturating_sub(root_prelude_lines).max(1); - // Reattach span against original source text for diagnostics. - err.span = None; - } - SourceError::Parse(err.with_line_span_from_source(&root_source_map, root_source_id)) - }) - .map_err(SourcePathError::Source)?; - if root_prelude_lines > 0 { - remap_frontend_ir_line_numbers(&mut root_parsed, root_prelude_lines); - } + ) + .map_err(|err| match err { + // Root resolution diagnostics (unknown/ambiguous imported calls, + // visibility failures) already carry spans keyed to the + // compilation-wide map; attach the map so they render from the + // owning source. + SourcePathError::Source(error) => SourcePathError::SourceWithMap { + error, + sources: collect_state.sources.clone(), + }, + other => other, + })?; collect_state.units.push(ParsedUnit { parsed: root_parsed, - scope_prefix: None, + scope_identity: None, source_name: path.display().to_string(), + module: root_module, + source_id: root_source_id, }); - Ok((root_parse_source, collect_state.units)) + Ok(LoadedSourceUnits { + units: collect_state.units, + module_graph: collect_state.module_graph, + sources: collect_state.sources, + }) +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use super::super::modules::ModuleId; + use super::*; + + fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + // Module identities are canonical for existing files; keep expected + // paths canonical too so assertions match under symlinked temp dirs. + root.canonicalize().unwrap_or(root) + } + + fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source) + .unwrap_or_else(|err| panic!("{description} should write: {err}")); + } + + fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); + } + + /// Two modules named `util.rss` in different directories plus a root that + /// imports both. Returns `(main, a/util, b/util)` paths. + fn write_same_stem_fixture(root: &Path) -> (PathBuf, PathBuf, PathBuf) { + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + + let a_module = a_dir.join("util.rss"); + let b_module = b_dir.join("util.rss"); + write_source(&a_module, "pub fn helper() { 1; }\n", "a/util source"); + write_source(&b_module, "pub fn helper() { 2; }\n", "b/util source"); + + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nfn run() { au::helper(); bu::helper(); }\n", + "main source", + ); + (main_path, a_module, b_module) + } + + #[test] + fn loader_graph_records_same_stem_modules_in_different_directories() { + let root = temp_module_root("semantic_m1_same_stem"); + let (main_path, a_module, b_module) = write_same_stem_fixture(&root); + let main_source = std::fs::read_to_string(&main_path).expect("main source readable"); + + let loaded = load_units_for_source_file( + &main_path, + SourceFlavor::RustScript, + &main_source, + &CompileSourceFileOptions::default(), + ) + .expect("load should succeed"); + let graph = loaded.module_graph; + assert_eq!(graph.len(), 3, "root plus two same-stem modules"); + + let main_id = graph + .module_id_for_identity(&main_path) + .expect("main module should be registered"); + let a_id = graph + .module_id_for_identity(&a_module) + .expect("a/util should be registered"); + let b_id = graph + .module_id_for_identity(&b_module) + .expect("b/util should be registered"); + assert_ne!( + a_id, b_id, + "same-stem modules in different dirs must differ" + ); + assert_eq!(main_id, ModuleId(0), "root module is always module 0"); + + // Import edges from the root to both modules, in source order. + let main_node = graph.node(main_id).expect("main node should exist"); + assert_eq!(main_node.imports.len(), 2); + let targets: Vec<_> = main_node + .imports + .iter() + .map(|import| import.target) + .collect(); + assert!(targets.contains(&Some(a_id))); + assert!(targets.contains(&Some(b_id))); + assert!(main_node.imports.iter().all(|import| import.line >= 1)); + assert_eq!(main_node.imports[0].spec, "a/util.rss"); + assert_eq!(main_node.imports[1].spec, "b/util.rss"); + + remove_module_root(&root); + } + + #[test] + fn loader_graph_is_deterministic_across_loads() { + let root = temp_module_root("semantic_m1_deterministic"); + let (main_path, _, _) = write_same_stem_fixture(&root); + + let load = || { + let source = std::fs::read_to_string(&main_path).expect("main source readable"); + load_units_for_source_file( + &main_path, + SourceFlavor::RustScript, + &source, + &CompileSourceFileOptions::default(), + ) + .expect("load should succeed") + .module_graph + }; + let first = load(); + let second = load(); + assert_eq!(first.len(), second.len()); + let sequence = |graph: &ModuleGraph| { + graph + .nodes() + .iter() + .map(|node| (node.module, node.identity.clone())) + .collect::>() + }; + assert_eq!(sequence(&first), sequence(&second)); + + remove_module_root(&root); + } + + #[test] + fn loader_graph_uses_virtual_identity_for_in_memory_modules() { + let path = PathBuf::from("__pd_vm_inmemory__/main.rss"); + let source = "use a::util;\nfn run() { helper(); }\n"; + let options = CompileSourceFileOptions::new() + .with_module_override_source("a/util.rss", "pub fn helper() { 1; }\n"); + + let loaded = load_units_for_source_file(&path, SourceFlavor::RustScript, source, &options) + .expect("virtual load should succeed"); + let graph = loaded.module_graph; + assert_eq!(graph.len(), 2, "virtual root plus overridden module"); + + let main_id = graph + .module_id_for_identity(&path) + .expect("virtual main should be registered"); + let a_id = graph + .module_id_for_identity(PathBuf::from("__pd_vm_inmemory__/a/util.rss").as_path()) + .expect("virtual override module should be registered"); + assert_ne!(main_id, a_id); + + let main_node = graph.node(main_id).expect("main node should exist"); + assert_eq!(main_node.imports.len(), 1); + assert_eq!(main_node.imports[0].target, Some(a_id)); + assert_eq!( + main_node.imports[0].kind, + super::super::modules::ImportTargetKind::FileModule + ); + } + + /// Fixture: `main` imports `a/util` (pub alpha + private helper) and + /// `b/util` (pub beta + private helper). Both helpers are private and + /// same-named; `a/util` also imports a third module `leaf` (pub shared) + /// so the transitive re-export rule is exercised through the real loader. + fn write_symbol_fixture(root: &Path) -> (PathBuf, PathBuf, PathBuf, PathBuf) { + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + + let leaf_module = a_dir.join("leaf.rss"); + write_source(&leaf_module, "pub fn shared() { 100; }\n", "leaf source"); + + let a_module = a_dir.join("util.rss"); + write_source( + &a_module, + "use self::leaf;\npub fn alpha() { helper(); }\nfn helper() { 11; }\n", + "a/util source", + ); + let b_module = b_dir.join("util.rss"); + write_source( + &b_module, + "pub fn beta() { helper(); }\nfn helper() { 22; }\n", + "b/util source", + ); + + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nfn run() { au::alpha(); bu::beta(); }\n", + "main source", + ); + (main_path, a_module, b_module, leaf_module) + } + + fn load_fixture(main_path: &Path) -> ModuleGraph { + let main_source = std::fs::read_to_string(main_path).expect("main source readable"); + load_units_for_source_file( + main_path, + SourceFlavor::RustScript, + &main_source, + &CompileSourceFileOptions::default(), + ) + .expect("load should succeed") + .module_graph + } + + #[test] + fn loader_records_public_exports_and_private_declarations() { + let root = temp_module_root("semantic_m3_exports"); + let (main_path, a_module, b_module, _) = write_symbol_fixture(&root); + let graph = load_fixture(&main_path); + + let a_id = graph + .module_id_for_identity(&a_module) + .expect("a/util should be registered"); + let a_node = graph.node(a_id).expect("a/util node exists"); + let a_names = |node: &super::super::modules::ModuleNode| { + node.declarations + .iter() + .map(|decl| decl.name.clone()) + .collect::>() + }; + assert_eq!(a_names(a_node), vec!["alpha", "helper"]); + assert_eq!( + a_node + .exports + .iter() + .map(|entry| entry.name.as_str()) + .collect::>(), + vec!["alpha"], + "only the pub declaration is exported" + ); + assert!( + !graph + .declaration(a_id, "helper") + .expect("private helper exists") + .public + ); + assert_eq!( + graph.symbol_for_export(a_id, "helper"), + None, + "private helpers never enter the export table" + ); + + let b_id = graph + .module_id_for_identity(&b_module) + .expect("b/util should be registered"); + assert_eq!( + graph + .symbol_for_export(b_id, "beta") + .expect("beta is exported") + .module, + b_id, + "each module's exports are owned by that module" + ); + + remove_module_root(&root); + } + + #[test] + fn loader_keeps_imported_bindings_separate_and_blocks_transitive_reexport() { + let root = temp_module_root("semantic_m3_bindings"); + let (main_path, a_module, _, leaf_module) = write_symbol_fixture(&root); + let graph = load_fixture(&main_path); + + let a_id = graph + .module_id_for_identity(&a_module) + .expect("a/util should be registered"); + let a_node = graph.node(a_id).expect("a/util node exists"); + assert_eq!( + a_node.imported_bindings.len(), + 1, + "a/util imports exactly leaf::shared" + ); + let binding = &a_node.imported_bindings[0]; + assert_eq!(binding.local_name, "shared"); + assert_eq!(binding.source_name, "shared"); + assert_eq!( + binding.source_module, + graph + .module_id_for_identity(&leaf_module) + .expect("leaf should be registered") + ); + assert_eq!( + graph + .symbol_for_export(a_id, "shared") + .map(|symbol| symbol.module), + None, + "a/util must not re-export leaf's function" + ); + assert!( + a_node.declarations.iter().all(|decl| decl.name != "shared"), + "the imported function is not a local declaration of a/util" + ); + + let main_id = graph + .module_id_for_identity(&main_path) + .expect("main should be registered"); + let main_node = graph.node(main_id).expect("main node exists"); + assert_eq!( + main_node.imported_bindings.len(), + 2, + "main imports alpha and beta" + ); + assert!( + main_node + .imported_bindings + .iter() + .all(|binding| binding.local_name == "alpha" || binding.local_name == "beta") + ); + assert!( + main_node.imported_bindings.iter().all(|binding| { + graph + .symbol_for_export(main_id, &binding.local_name) + .is_none() + }), + "main's export table stays empty: no implicit re-export of anything imported" + ); + + remove_module_root(&root); + } + + #[test] + fn loader_assigns_distinct_symbols_to_same_named_private_helpers() { + let root = temp_module_root("semantic_m3_same_named"); + let (main_path, a_module, b_module, _) = write_symbol_fixture(&root); + let graph = load_fixture(&main_path); + + let a_id = graph + .module_id_for_identity(&a_module) + .expect("a/util should be registered"); + let b_id = graph + .module_id_for_identity(&b_module) + .expect("b/util should be registered"); + let a_helper = graph + .declaration_symbol(a_id, "helper") + .expect("a helper exists"); + let b_helper = graph + .declaration_symbol(b_id, "helper") + .expect("b helper exists"); + assert_ne!( + a_helper, b_helper, + "same-named private helpers in independent modules own distinct symbols" + ); + assert_eq!(a_helper.module, a_id); + assert_eq!(b_helper.module, b_id); + + let main_id = graph + .module_id_for_identity(&main_path) + .expect("main should be registered"); + let main_alpha = graph + .declaration_symbol(main_id, "run") + .expect("run exists"); + assert_eq!( + main_alpha, + super::super::modules::SymbolId { + module: main_id, + index: 0 + }, + "root declarations start at symbol index 0" + ); + + remove_module_root(&root); + } + + #[test] + fn loader_symbols_are_deterministic_across_loads() { + let root = temp_module_root("semantic_m3_symbol_determinism"); + let (main_path, a_module, _, _) = write_symbol_fixture(&root); + let first = load_fixture(&main_path); + let second = load_fixture(&main_path); + + let symbol_sequence = |graph: &ModuleGraph| { + graph + .nodes() + .iter() + .map(|node| { + ( + node.module, + node.declarations + .iter() + .map(|decl| (decl.name.clone(), decl.symbol)) + .collect::>(), + node.exports + .iter() + .map(|entry| (entry.name.clone(), entry.symbol)) + .collect::>(), + node.imported_bindings + .iter() + .map(|binding| { + ( + binding.local_name.clone(), + binding.source_symbol, + binding.source_module, + ) + }) + .collect::>(), + ) + }) + .collect::>() + }; + assert_eq!(symbol_sequence(&first), symbol_sequence(&second)); + assert_eq!(first.len(), second.len()); + let _ = a_module; + + remove_module_root(&root); + } } diff --git a/src/compiler/source_loader/graph.rs b/src/compiler/source_loader/graph.rs index 82e95c30..a997a1ec 100644 --- a/src/compiler/source_loader/graph.rs +++ b/src/compiler/source_loader/graph.rs @@ -1,16 +1,18 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use crate::compiler::source_map::SourceMap; use super::super::{ - CompileSourceFileOptions, SourceError, SourceFlavor, SourcePathError, frontends, - linker::{ParsedUnit, sanitize_scope_prefix}, + CompileSourceFileOptions, ParseError, SourceError, SourceFlavor, SourcePathError, frontends, + ir::{Expr, FrontendIr, FunctionDecl, Stmt, TypeSchema}, + linker::{ParsedUnit, module_scope_prefix}, + modules::{ImportTargetKind, ImportedBinding, ModuleGraph, ModuleId, ResolvedImport, SymbolId}, }; use super::imports::{ is_builtin_host_namespace_spec, is_module_specifier, is_virtual_host_namespace_spec, - parse_module_imports, resolve_module_path, should_treat_missing_module_as_host_namespace, - strip_import_directives, + parse_module_imports, resolve_module_path, scan_module_imports, + should_treat_missing_module_as_host_namespace, }; use super::model::{ExportedFunctionSignature, ImportClause, ModuleCollectState, ModuleImport}; @@ -21,13 +23,79 @@ pub(super) fn collect_module_units( options: &CompileSourceFileOptions, state: &mut ModuleCollectState, ) -> Result<(), SourcePathError> { - let imports = parse_module_imports(source, flavor, path, options)?; - for import in imports { - let spec = import.spec; + // Register this module in the semantic graph. Registration is + // identity-keyed and idempotent, so the root and every nested module get + // a deterministic `ModuleId`/`SourceId` in first-encounter order. + let current_id = + state + .module_graph + .add_node(path.to_path_buf(), path.display().to_string(), Vec::new()); + // Register the module's raw text in the compilation-wide source map at + // its graph `SourceId`, before any scan or parse that can produce spans + // referencing that id (milestone 5: every span stays owned by its + // module's source). + let current_source_id = state + .module_graph + .node(current_id) + .map(|node| node.source) + .unwrap_or(crate::compiler::modules::SourceId(0)); + state.sources.add_source_at( + current_source_id.0, + path.display().to_string(), + source.to_string(), + ); + let (imports, decls) = scan_module_imports(source, flavor, path, options).map_err(|err| { + // Nested module sources surface their parse errors through the same + // path-prefixed diagnostic shape the compile parse uses. The root is + // scanned (and fails, if at all) in `load_units_for_source_file` + // before this point, so it never receives a prefix here. The scan + // parser numbers spans with its own local source id 0, so the span + // is always rebuilt against the owning module's graph source id — + // offsets from one module must never be interpreted in another. + match err { + SourcePathError::Source(SourceError::Parse(mut parse)) => { + parse.message = format!("{}: {}", path.display(), parse.message); + parse.span = None; + parse = parse.with_line_span_from_source(&state.sources, current_source_id.0); + SourcePathError::Source(SourceError::Parse(parse)) + } + other => other, + } + })?; + for (import_index, import) in imports.iter().enumerate() { + let spec = import.spec.clone(); + let span = decls + .get(import_index) + .map(|decl| decl.span) + .unwrap_or_else(|| crate::compiler::source_map::Span::new(0, 0, 0)); if is_builtin_host_namespace_spec(&spec) { + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::BuiltinNamespace, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target: None, + }, + ); continue; } if !is_module_specifier(&spec) { + // Plugin-managed host imports (non-RustScript flavors) stay on + // their dedicated resolution path. + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::HostNamespace, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target: None, + }, + ); continue; } let resolved = resolve_module_path(path, &spec, options)?; @@ -35,12 +103,37 @@ pub(super) fn collect_module_units( if key == path && is_virtual_host_namespace_spec(&spec, options) { // `use io;` / `use re;` inside files named `io.rss` / `re.rss` should // keep behaving as host-namespace imports instead of self-module cycles. + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::HostNamespace, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target: None, + }, + ); continue; } if state.visiting.contains(&key) { return Err(SourcePathError::ImportCycle(key)); } if state.seen.contains(&key) { + // Already loaded: keep the resolved edge pointing at the existing + // node instead of re-collecting the module. + let target = state.module_graph.module_id_for_identity(&key); + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::FileModule, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target, + }, + ); continue; } @@ -52,6 +145,17 @@ pub(super) fn collect_module_units( Ok(source) => source, Err(err) => { if should_treat_missing_module_as_host_namespace(&spec, options, &err) { + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::HostNamespace, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target: None, + }, + ); continue; } return Err(SourcePathError::Io(err)); @@ -68,22 +172,43 @@ pub(super) fn collect_module_units( )?; state.visiting.pop(); - let module_source = - strip_import_directives(&module_source_raw, SourceFlavor::RustScript, options)?; - let mut module_source_map = SourceMap::new(); - let module_source_id = - module_source_map.add_source(resolved.display().to_string(), module_source.clone()); - let parsed = frontends::parse_source(&module_source, SourceFlavor::RustScript, options) - .map_err(|err| { - SourceError::Parse( - err.with_line_span_from_source(&module_source_map, module_source_id), - ) - }) - .map_err(SourcePathError::Source)?; + let module_imports = parse_module_imports( + &module_source_raw, + SourceFlavor::RustScript, + &resolved, + options, + )?; + let module_source_id = state + .module_graph + .module_id_for_identity(&key) + .and_then(|module| state.module_graph.node(module)) + .map(|node| node.source.0) + .unwrap_or(0); + let mut parsed = frontends::parse_module_source_with_source_id( + &module_source_raw, + SourceFlavor::RustScript, + options, + module_source_id, + ) + .map_err(|mut err| { + // Nested module sources are parsed verbatim (no synthetic + // prelude, no textual rewrite), so the parse already reports the + // owning module's lines; rebuild the span against the + // compilation-wide map and prefix the module path. + err.span = None; + let mut parse = err.with_line_span_from_source(&state.sources, module_source_id); + parse.message = format!("{}: {}", resolved.display(), parse.message); + SourceError::Parse(parse) + })?; + let extern_names = parsed + .implicit_extern_names + .iter() + .map(String::as_str) + .collect::>(); let exports = parsed .functions .iter() - .filter(|func| func.exported) + .filter(|func| func.exported && !extern_names.contains(func.name.as_str())) .map(|func| { ( func.name.clone(), @@ -94,11 +219,36 @@ pub(super) fn collect_module_units( ) }) .collect::>(); + let target = state + .module_graph + .module_id_for_identity(&key) + .expect("module node should be registered during collection"); + record_module_symbols( + state, + target, + &resolved, + &module_imports, + &mut parsed, + options, + )?; state.units.push(ParsedUnit { parsed, - scope_prefix: Some(sanitize_scope_prefix(&resolved)), + scope_identity: Some(module_scope_prefix(&resolved, target)), source_name: resolved.display().to_string(), + module: target, + source_id: module_source_id, }); + state.module_graph.add_import( + current_id, + ResolvedImport { + kind: ImportTargetKind::FileModule, + spec, + clause: import.clause.clone(), + span, + line: import.line, + target: Some(target), + }, + ); state.module_exports.insert(key.clone(), exports); state.seen.insert(key); } @@ -115,37 +265,125 @@ fn module_source_override<'a>( }) } -pub(super) fn build_rustscript_import_prelude( - path: &Path, - imports: &[ModuleImport], - module_exports: &HashMap>, - options: &CompileSourceFileOptions, -) -> Result { - let declared = collect_imported_module_functions(path, imports, module_exports, options)?; - let mut prelude = String::new(); - for (name, signature) in declared { - let type_params = if signature.type_params.is_empty() { - String::new() - } else { - format!("<{}>", signature.type_params.join(", ")) +/// Build the exported-signature table keyed by [`SymbolId`]. +/// +/// The loader validates call sites against the exported arity and type +/// parameters at resolution time (the parse can no longer see them: module +/// sources are parsed verbatim without a synthetic prelude). +fn exported_signature_table( + state: &ModuleCollectState, + graph: &ModuleGraph, +) -> HashMap { + let mut table = HashMap::new(); + for node in graph.nodes() { + let Some(exports) = state.module_exports.get(&node.identity) else { + continue; }; - let args = (0..signature.arity) - .map(|idx| format!("arg{idx}")) - .collect::>() - .join(", "); - prelude.push_str(&format!("pub fn {name}{type_params}({args});\n")); + for entry in &node.exports { + if let Some(signature) = exports.get(&entry.name) { + table.insert(entry.symbol, signature.clone()); + } + } + } + table +} + +/// Namespace portion of a qualified call name (`au::helper` → `au`). +fn namespace_of(qualified: &str) -> &str { + qualified + .split_once("::") + .map(|(namespace, _)| namespace) + .unwrap_or(qualified) +} + +/// Clause-derived namespace alias of one import edge, mirroring the parser's +/// module-namespace alias rules: the `as` alias for namespace imports, the +/// spec stem for all-public imports, and no namespace for named imports. +fn namespace_alias_for_import(import: &ResolvedImport) -> Option { + match &import.clause { + ImportClause::Namespace(alias) => Some(alias.clone()), + ImportClause::AllPublic => Path::new(&import.spec) + .file_stem() + .and_then(|stem| stem.to_str()) + .map(|stem| stem.to_string()), + ImportClause::Named(_) | ImportClause::Prefix(_) => None, } - Ok(prelude) } -pub(super) fn collect_imported_module_functions( +/// File-module import targets that bind `namespace`, either through a clause +/// alias (`use a::util as au;` binds `au`) or through the spec stem +/// (host-form single-segment imports such as `use module;` whose namespace +/// the parser resolved as a host root). +fn file_module_targets_for_namespace( + graph: &ModuleGraph, + module: ModuleId, + namespace: &str, +) -> Vec { + let Some(node) = graph.node(module) else { + return Vec::new(); + }; + let mut targets = Vec::new(); + for import in &node.imports { + if import.kind != ImportTargetKind::FileModule { + continue; + } + let Some(target) = import.target else { + continue; + }; + let stem = Path::new(&import.spec) + .file_stem() + .and_then(|stem| stem.to_str()); + if (namespace_alias_for_import(import).as_deref() == Some(namespace) + || stem == Some(namespace)) + && !targets.contains(&target) + { + targets.push(target); + } + } + targets +} + +/// Whether any file-module import edge of `module` binds `qualified`'s +/// namespace. Host-form declarations whose namespace names a file module are +/// kept out of the module's declaration table; the resolution pass converts +/// their call sites to [`Expr::ModuleCall`] instead. +fn namespace_has_file_module_target( + graph: &ModuleGraph, + module: ModuleId, + qualified: &str, +) -> bool { + !file_module_targets_for_namespace(graph, module, namespace_of(qualified)).is_empty() +} + +/// One function binding introduced by an import edge, before it is recorded +/// in the module graph. +struct ImportBindingData { + /// Name the importing module binds (`as` alias for named imports). + local_name: String, + /// Name of the declaration in the source module. + source_name: String, + /// Source module once its graph node is known; `None` for host/builtin + /// namespaces that stay on their dedicated resolution paths. + source_module: Option, + /// Line of the `use` directive that introduced the binding. + line: usize, +} + +/// Collect the function bindings a module's imports introduce, structurally. +/// +/// Mirrors the legacy `collect_imported_module_functions` resolution rules +/// (builtin and non-module specifiers skipped, missing virtual host namespaces +/// tolerated) but preserves `as` aliases and resolves the source module in +/// the semantic graph, so the loader can record [`ImportedBinding`]s that +/// stay separate from local declarations. +fn collect_imported_bindings( path: &Path, imports: &[ModuleImport], module_exports: &HashMap>, + graph: &ModuleGraph, options: &CompileSourceFileOptions, -) -> Result, SourcePathError> { - let mut imported_functions = HashMap::::new(); - +) -> Result, SourcePathError> { + let mut bindings = Vec::new(); for import in imports { if is_builtin_host_namespace_spec(&import.spec) { continue; @@ -165,22 +403,22 @@ pub(super) fn collect_imported_module_functions( message: format!("module '{}' did not load", import.spec), }); }; + let source_module = graph.module_id_for_identity(&resolved); match &import.clause { ImportClause::AllPublic | ImportClause::Namespace(_) | ImportClause::Prefix(_) => { - for (name, signature) in exports { - merge_imported_function_signature( - &mut imported_functions, - name, - signature, - path, - import.line, - )?; + for name in exports.keys() { + bindings.push(ImportBindingData { + local_name: name.clone(), + source_name: name.clone(), + source_module, + line: import.line, + }); } } ImportClause::Named(named) => { for binding in named { - let signature = exports.get(&binding.imported).cloned().ok_or_else(|| { + let _signature = exports.get(&binding.imported).cloned().ok_or_else(|| { SourcePathError::InvalidImportSyntax { path: path.to_path_buf(), line: import.line, @@ -190,48 +428,796 @@ pub(super) fn collect_imported_module_functions( ), } })?; - merge_imported_function_signature( - &mut imported_functions, - &binding.imported, - &signature, - path, - import.line, - )?; + bindings.push(ImportBindingData { + local_name: binding.local.clone(), + source_name: binding.imported.clone(), + source_module, + line: import.line, + }); } } } } - - let mut declared = imported_functions.into_iter().collect::>(); - declared.sort_by(|(lhs_name, _), (rhs_name, _)| lhs_name.cmp(rhs_name)); - Ok(declared) + Ok(bindings) } -fn merge_imported_function_signature( - imported_functions: &mut HashMap, - name: &str, - signature: &ExportedFunctionSignature, +/// Attach milestone-3 declaration symbols and imported bindings to a parsed +/// unit's module node, then resolve imported call sites to their target +/// symbols (milestone 4). +/// +/// Runs once per module, right after its unit is parsed: imported-binding +/// mirror declarations and implicit externs are skipped, every remaining +/// function declaration receives a [`SymbolId`] owned by the module, public +/// declarations populate the module's export table, and every import-introduced +/// binding is recorded separately in the module's imported-binding table. +/// The same `symbol` is written back onto the parsed [`FunctionDecl`] so the +/// linker can collect it through `merge_units`. Finally, calls to imported +/// functions and module namespace members are resolved to [`Expr::ModuleCall`] +/// nodes carrying the target [`SymbolId`]. +pub(super) fn record_module_symbols( + state: &mut ModuleCollectState, + module: ModuleId, path: &Path, - line: usize, + imports: &[ModuleImport], + parsed: &mut FrontendIr, + options: &CompileSourceFileOptions, ) -> Result<(), SourcePathError> { - if let Some(existing) = imported_functions.get_mut(name) { - existing.arity = existing.arity.max(signature.arity); - if existing.type_params != signature.type_params { - if existing.type_params.is_empty() { - existing.type_params = signature.type_params.clone(); - } else if !signature.type_params.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, + let bindings = collect_imported_bindings( + path, + imports, + &state.module_exports, + &state.module_graph, + options, + )?; + // Implicit externs (module mode) mirror calls the loader must resolve or + // reject; they never become local declarations or flat entries. + let extern_names = parsed + .implicit_extern_names + .iter() + .cloned() + .collect::>(); + + let module_source_id = state + .module_graph + .node(module) + .map(|node| node.source.0) + .unwrap_or(0); + let decl_lines = collect_function_decl_lines(&parsed.stmts); + let signatures = exported_signature_table(state, &state.module_graph); + + for func in &mut parsed.functions { + if extern_names.contains(func.name.as_str()) { + // Implicit extern (module mode): the resolution pass resolves + // (or rejects) its call sites; never a local decl. + continue; + } + if func.name.contains("::") + && namespace_has_file_module_target(&state.module_graph, module, &func.name) + { + // Host-form declaration whose namespace names a file module + // (single-segment import forms such as `use module;`): the + // resolution pass converts its call sites to `ModuleCall`, so + // the declaration must not become a flat host entry. + continue; + } + // A local declaration whose name is also imported is recorded here + // and then rejected when the import binding is added below: no + // silent shadowing of imported names. + let decl_line = decl_lines + .get(&func.index) + .copied() + .map(|line| line as usize) + .unwrap_or(1); + let symbol = state + .module_graph + .add_declaration(module, &func.name, func.exported) + .map_err(|message| { + // The duplicate symbol diagnostic renders from the owning + // module source: same-named declarations collide inside one + // module only, and the span points at the redeclaration. + let span = state.sources.line_span(module_source_id, decl_line); + SourcePathError::Source(SourceError::Parse(ParseError { + span, + code: None, + line: decl_line, + message: format!("{}: {message}", path.display()), + })) + })?; + func.symbol = Some(symbol); + } + + for binding in bindings { + let Some(source_module) = binding.source_module else { + continue; + }; + let binding_line = binding.line.max(1); + let source_symbol = state + .module_graph + .symbol_for_export(source_module, &binding.source_name) + .ok_or_else(|| { + // Visibility failure: the import directive is the offending + // site, so the span points at the `use` line in the + // importing module's source. + let span = state.sources.line_span(module_source_id, binding_line); + SourcePathError::Source(SourceError::Parse(ParseError { + span, + code: None, + line: binding_line, message: format!( - "function '{name}' declared with conflicting type parameters across imported modules" + "{}: imported function '{}' is not exported by module {}", + path.display(), + binding.source_name, + source_module.0 ), - }); + })) + })?; + state + .module_graph + .add_imported_binding( + module, + ImportedBinding { + local_name: binding.local_name, + source_module, + source_symbol, + source_name: binding.source_name, + }, + ) + .map_err(|message| { + let span = state.sources.line_span(module_source_id, binding_line); + SourcePathError::Source(SourceError::Parse(ParseError { + span, + code: None, + line: binding_line, + message: format!("{}: {message}", path.display()), + })) + })?; + } + + resolve_imported_call_sites( + module, + path, + &state.module_graph, + &state.sources, + &signatures, + &extern_names, + parsed, + ) +} + +fn collect_function_decl_lines(stmts: &[Stmt]) -> HashMap { + let mut lines = HashMap::new(); + record_function_decl_lines(stmts, &mut lines); + lines +} + +fn record_function_decl_lines(stmts: &[Stmt], lines: &mut HashMap) { + for stmt in stmts { + match stmt { + Stmt::FuncDecl { index, line, .. } => { + lines.entry(*index).or_insert(*line); + } + Stmt::IfElse { + then_branch, + else_branch, + .. + } => { + record_function_decl_lines(then_branch, lines); + record_function_decl_lines(else_branch, lines); + } + Stmt::For { + init, post, body, .. + } => { + record_function_decl_lines(std::slice::from_ref(init.as_ref()), lines); + record_function_decl_lines(std::slice::from_ref(post.as_ref()), lines); + record_function_decl_lines(body, lines); + } + Stmt::While { body, .. } => record_function_decl_lines(body, lines), + _ => {} + } + } +} + +/// Resolution context for one module's imported-call pass. +struct CallResolutionContext<'a> { + functions_by_index: HashMap, + /// Direct call names bound by exactly one source module (keyed by the + /// name the importing module binds: `as` alias or source name). + plain_symbols: HashMap, + /// Direct call names bound from several modules with different symbols. + ambiguous_names: HashSet, + /// Exported arity/type-parameter table for signature validation. + signatures: &'a HashMap, + /// Implicit-extern names produced by the parser (module mode). + extern_names: &'a HashSet, + module: ModuleId, + path: &'a Path, + graph: &'a ModuleGraph, + sources: &'a SourceMap, + source_id: u32, +} + +impl<'a> CallResolutionContext<'a> { + /// Resolve one unit-local call name to its target symbol. + /// + /// Names with a namespace separator resolve through the module's + /// file-module import edges (clause alias or spec stem); plain names + /// resolve through the imported-binding table. Returns `Ok(None)` for + /// names that are neither (the caller decides how to report them). + fn target_for_call( + &self, + decl_name: &str, + arg_count: usize, + type_args: &[TypeSchema], + line: u32, + ) -> Result, SourcePathError> { + if let Some((namespace, member)) = decl_name.split_once("::") { + return self.target_for_namespace_call( + namespace, member, decl_name, arg_count, type_args, line, + ); + } + if self.ambiguous_names.contains(decl_name) { + return Err(ambiguous_imported_call_error( + self.path, + decl_name, + self.sources, + self.source_id, + line, + )); + } + if let Some(symbol) = self.plain_symbols.get(decl_name) { + self.validate_imported_signature(decl_name, *symbol, arg_count, type_args, line)?; + return Ok(Some(*symbol)); + } + Err(unknown_function_error( + self.path, + decl_name, + self.sources, + self.source_id, + line, + )) + } + + fn target_for_namespace_call( + &self, + namespace: &str, + member: &str, + qualified: &str, + arg_count: usize, + type_args: &[TypeSchema], + line: u32, + ) -> Result, SourcePathError> { + if member.contains("::") { + // Multi-level module member paths are not supported; the legacy + // pipeline reported the same call as an unknown namespace call. + return Err(unknown_namespace_call_error( + self.path, + qualified, + self.sources, + self.source_id, + line, + )); + } + let mut found = HashSet::new(); + for target in file_module_targets_for_namespace(self.graph, self.module, namespace) { + if let Some(symbol) = self.graph.symbol_for_export(target, member) { + found.insert(symbol); } } - return Ok(()); + match found.len() { + 0 => { + if self.extern_names.contains(qualified) { + // Multi-segment import form whose namespace or member did + // not resolve to a public export. + Err(unknown_namespace_call_error( + self.path, + qualified, + self.sources, + self.source_id, + line, + )) + } else { + // Host-form declaration through a file-module namespace + // whose module does not export the member: report like + // the legacy parse did for the unqualified name. + Err(unknown_function_error( + self.path, + member, + self.sources, + self.source_id, + line, + )) + } + } + 1 => { + let symbol = found.into_iter().next().expect("exactly one symbol"); + self.validate_imported_signature(qualified, symbol, arg_count, type_args, line)?; + Ok(Some(symbol)) + } + _ => Err(ambiguous_imported_call_error( + self.path, + qualified, + self.sources, + self.source_id, + line, + )), + } } - imported_functions.insert(name.to_string(), signature.clone()); + /// Validate a resolved call against the exported arity and type + /// parameters, mirroring the messages the synthetic prelude used to + /// produce at parse time. + fn validate_imported_signature( + &self, + call_name: &str, + symbol: SymbolId, + arg_count: usize, + type_args: &[TypeSchema], + line: u32, + ) -> Result<(), SourcePathError> { + let Some(signature) = self.signatures.get(&symbol) else { + return Ok(()); + }; + let parse_error = |message: String| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: self.sources.line_span(self.source_id, line as usize), + code: None, + line: line as usize, + message: format!("{}: {message}", self.path.display()), + })) + }; + if usize::from(signature.arity) != arg_count { + return Err(parse_error(format!( + "function '{call_name}' expects {} arguments", + signature.arity + ))); + } + if signature.type_params.is_empty() { + if type_args.is_empty() { + return Ok(()); + } + return Err(parse_error(format!( + "function '{call_name}' does not accept explicit type arguments" + ))); + } + if signature.type_params.len() != type_args.len() { + return Err(parse_error(format!( + "function '{call_name}' expects {} type arguments, got {}", + signature.type_params.len(), + type_args.len() + ))); + } + Ok(()) + } + + /// Resolve one function-value reference to its target symbol. + fn target_for_function_ref( + &self, + name: &str, + line: u32, + ) -> Result, SourcePathError> { + if self.ambiguous_names.contains(name) { + return Err(ambiguous_imported_call_error( + self.path, + name, + self.sources, + self.source_id, + line, + )); + } + if let Some(symbol) = self.plain_symbols.get(name) { + return Ok(Some(*symbol)); + } + Err(SourcePathError::Source(SourceError::Parse(ParseError { + span: self.sources.line_span(self.source_id, line as usize), + code: None, + line: line as usize, + message: format!("{}: unknown local '{}'", self.path.display(), name), + }))) + } +} + +/// Resolve every call to an imported function to its compiler-owned +/// [`SymbolId`] before unit merge. +/// +/// Module sources are parsed verbatim (no synthetic prelude, no textual +/// rewrite), so call sites reach this pass in the shapes the parser produced: +/// +/// - Direct calls (`helper(...)`) parse as implicit externs. A name bound +/// from exactly one source module maps to that module's symbol; a name +/// bound from several modules is ambiguous and becomes a diagnostic; an +/// unbound name is rejected as an unknown function. +/// - Namespace calls (`au::helper(...)`) parse either as implicit externs +/// carrying the qualified name (multi-segment import forms) or as +/// host-form calls whose namespace the parser treated as a host root +/// (single-segment import forms such as `use module;`). Both resolve +/// through the module's file-module import edges: the clause alias or the +/// spec stem maps the namespace to its target module, and the member must +/// be one of its public exports. +/// - Function values (`let f = helper;`) parse as +/// [`Expr::UnresolvedFunctionRef`] and resolve to +/// [`Expr::ModuleFunctionRef`]. +/// +/// Local calls (declarations that own a symbol) and host/builtin calls are +/// left untouched; the linker remaps them by symbol or keeps their reserved +/// builtin index. +fn resolve_imported_call_sites( + module: ModuleId, + path: &Path, + graph: &ModuleGraph, + sources: &SourceMap, + signatures: &HashMap, + extern_names: &HashSet, + parsed: &mut FrontendIr, +) -> Result<(), SourcePathError> { + let source_id = graph.node(module).map(|node| node.source.0).unwrap_or(0); + let mut plain_symbols = HashMap::::new(); + let mut ambiguous_names = HashSet::::new(); + if let Some(node) = graph.node(module) { + for binding in &node.imported_bindings { + match plain_symbols.entry(binding.local_name.clone()) { + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(binding.source_symbol); + } + std::collections::hash_map::Entry::Occupied(entry) + if *entry.get() != binding.source_symbol => + { + ambiguous_names.insert(binding.local_name.clone()); + } + std::collections::hash_map::Entry::Occupied(_) => {} + } + } + } + + let functions_by_index = parsed + .functions + .iter() + .map(|func| (func.index, func)) + .collect::>(); + + let ctx = CallResolutionContext { + functions_by_index, + plain_symbols, + ambiguous_names, + signatures, + extern_names, + module, + path, + graph, + sources, + source_id, + }; + + let resolve_stmt = |stmt: &mut Stmt| -> Result<(), SourcePathError> { + resolve_stmt_imported_calls(&ctx, stmt) + }; + for stmt in &mut parsed.stmts { + resolve_stmt(stmt)?; + } + for function_impl in parsed.function_impls.values_mut() { + for stmt in &mut function_impl.body_stmts { + resolve_stmt(stmt)?; + } + resolve_expr_imported_calls( + &ctx, + &mut function_impl.body_expr, + function_impl.body_expr_line.max(1), + )?; + } + Ok(()) +} + +fn unknown_function_error( + path: &Path, + name: &str, + sources: &SourceMap, + source_id: u32, + line: u32, +) -> SourcePathError { + SourcePathError::Source(SourceError::Parse(ParseError { + span: sources.line_span(source_id, line as usize), + code: None, + line: line as usize, + message: format!("{}: unknown function '{}'", path.display(), name), + })) +} + +/// Validate type arguments on a host import call whose parse-time validation +/// was deferred (non-builtin namespaces that may name file modules). +fn validate_deferred_host_type_args( + ctx: &CallResolutionContext<'_>, + host_name: &str, + type_args: &[TypeSchema], + line: u32, +) -> Result<(), SourcePathError> { + let expected = crate::compiler::parser::host_generic_type_arg_arity(host_name); + let parse_error = |message: String| { + SourcePathError::Source(SourceError::Parse(ParseError { + span: ctx.sources.line_span(ctx.source_id, line as usize), + code: None, + line: line as usize, + message: format!("{}: {message}", ctx.path.display()), + })) + }; + match expected { + Some(expected) if type_args.is_empty() || expected == type_args.len() => Ok(()), + Some(expected) => Err(parse_error(format!( + "function '{host_name}' expects {expected} type arguments, got {}", + type_args.len() + ))), + None if type_args.is_empty() => Ok(()), + None => Err(parse_error(format!( + "function '{host_name}' does not accept explicit type arguments" + ))), + } +} + +fn unknown_namespace_call_error( + path: &Path, + qualified: &str, + sources: &SourceMap, + source_id: u32, + line: u32, +) -> SourcePathError { + SourcePathError::Source(SourceError::Parse(ParseError { + span: sources.line_span(source_id, line as usize), + code: None, + line: line as usize, + message: format!( + "{}: unknown namespace call '{}'; the module does not export this function", + path.display(), + qualified + ), + })) +} + +fn ambiguous_imported_call_error( + path: &Path, + name: &str, + sources: &SourceMap, + source_id: u32, + line: u32, +) -> SourcePathError { + SourcePathError::Source(SourceError::Parse(ParseError { + span: sources.line_span(source_id, line as usize), + code: None, + line: line as usize, + message: format!( + "{}: call to '{name}' is ambiguous: the name is exported by multiple imported modules; qualify the call with a namespace alias or a named import", + path.display() + ), + })) +} + +fn resolve_stmt_imported_calls( + ctx: &CallResolutionContext<'_>, + stmt: &mut Stmt, +) -> Result<(), SourcePathError> { + let line = stmt_line(stmt); + match stmt { + Stmt::Noop { .. } | Stmt::Break { .. } | Stmt::Continue { .. } => {} + Stmt::Let { expr, .. } | Stmt::Assign { expr, .. } | Stmt::Expr { expr, .. } => { + resolve_expr_imported_calls(ctx, expr, line)?; + } + Stmt::ClosureLet { closure, .. } => { + resolve_expr_imported_calls(ctx, &mut closure.body, line)?; + } + Stmt::FuncDecl { .. } => {} + Stmt::IfElse { + condition, + then_branch, + else_branch, + .. + } => { + resolve_expr_imported_calls(ctx, condition, line)?; + for nested in then_branch { + resolve_stmt_imported_calls(ctx, nested)?; + } + for nested in else_branch { + resolve_stmt_imported_calls(ctx, nested)?; + } + } + Stmt::For { + init, + condition, + post, + body, + .. + } => { + resolve_stmt_imported_calls(ctx, init)?; + resolve_expr_imported_calls(ctx, condition, line)?; + resolve_stmt_imported_calls(ctx, post)?; + for nested in body { + resolve_stmt_imported_calls(ctx, nested)?; + } + } + Stmt::While { + condition, body, .. + } => { + resolve_expr_imported_calls(ctx, condition, line)?; + for nested in body { + resolve_stmt_imported_calls(ctx, nested)?; + } + } + Stmt::Drop { .. } => {} + } Ok(()) } + +fn resolve_expr_imported_calls( + ctx: &CallResolutionContext<'_>, + expr: &mut Expr, + line: u32, +) -> Result<(), SourcePathError> { + match expr { + Expr::Call(index, type_args, args) => { + for arg in args.iter_mut() { + resolve_expr_imported_calls(ctx, arg, line)?; + } + let Some(decl) = ctx.functions_by_index.get(index) else { + // Builtin calls use the reserved builtin index space and are + // not part of the unit's declaration table. + return Ok(()); + }; + if decl.symbol.is_some() { + // Local declaration or host import: resolved by the linker. + // Host imports whose type arguments were deferred at parse + // (non-builtin namespaces that may name file modules) are + // validated against the host generic arity here. + if decl.name.contains("::") { + validate_deferred_host_type_args(ctx, &decl.name, type_args, line)?; + } + return Ok(()); + } + let name = decl.name.as_str(); + if let Some(symbol) = ctx.target_for_call(name, args.len(), type_args, line)? { + *expr = Expr::ModuleCall(symbol, std::mem::take(type_args), std::mem::take(args)); + } else { + return Err(unknown_function_error( + ctx.path, + name, + ctx.sources, + ctx.source_id, + line, + )); + } + } + Expr::FunctionRef(index, _type_args) => { + let Some(decl) = ctx.functions_by_index.get(index) else { + return Ok(()); + }; + if decl.symbol.is_none() { + return Err(unknown_function_error( + ctx.path, + &decl.name, + ctx.sources, + ctx.source_id, + line, + )); + } + } + Expr::UnresolvedFunctionRef { name, type_args } => { + if let Some(symbol) = ctx.target_for_function_ref(name, line)? { + *expr = Expr::ModuleFunctionRef(symbol, std::mem::take(type_args)); + } else { + return Err(unknown_function_error( + ctx.path, + name, + ctx.sources, + ctx.source_id, + line, + )); + } + } + Expr::Null + | Expr::Int(_) + | Expr::Float(_) + | Expr::Bool(_) + | Expr::Bytes(_) + | Expr::String(_) + | Expr::ModuleCall(..) + | Expr::ModuleFunctionRef(..) + | Expr::Var(_) + | Expr::MoveVar(_) + | Expr::MoveField { .. } + | Expr::MoveIndex { .. } => {} + Expr::OptionalGet { + container, + key, + container_slot: _, + key_slot: _, + } => { + resolve_expr_imported_calls(ctx, container, line)?; + resolve_expr_imported_calls(ctx, key, line)?; + } + Expr::OptionUnwrapOr { + value, + value_slot: _, + fallback, + } => { + resolve_expr_imported_calls(ctx, value, line)?; + resolve_expr_imported_calls(ctx, fallback, line)?; + } + Expr::LocalCall(_, _, args) => { + for arg in args.iter_mut() { + resolve_expr_imported_calls(ctx, arg, line)?; + } + } + Expr::Closure(closure) => { + resolve_expr_imported_calls(ctx, &mut closure.body, line)?; + } + Expr::ClosureCall(closure, args) => { + resolve_expr_imported_calls(ctx, &mut closure.body, line)?; + for arg in args.iter_mut() { + resolve_expr_imported_calls(ctx, arg, line)?; + } + } + Expr::Add(lhs, rhs) + | Expr::Sub(lhs, rhs) + | Expr::Mul(lhs, rhs) + | Expr::Div(lhs, rhs) + | Expr::Mod(lhs, rhs) + | Expr::And(lhs, rhs) + | Expr::Or(lhs, rhs) + | Expr::Eq(lhs, rhs) + | Expr::Lt(lhs, rhs) + | Expr::Gt(lhs, rhs) => { + resolve_expr_imported_calls(ctx, lhs, line)?; + resolve_expr_imported_calls(ctx, rhs, line)?; + } + Expr::Neg(inner) + | Expr::Not(inner) + | Expr::ToOwned(inner) + | Expr::Borrow(inner) + | Expr::BorrowMut(inner) => { + resolve_expr_imported_calls(ctx, inner, line)?; + } + Expr::IfElse { + condition, + then_expr, + else_expr, + } => { + resolve_expr_imported_calls(ctx, condition, line)?; + resolve_expr_imported_calls(ctx, then_expr, line)?; + resolve_expr_imported_calls(ctx, else_expr, line)?; + } + Expr::Match { + value_slot: _, + result_slot: _, + value, + arms, + default, + } => { + resolve_expr_imported_calls(ctx, value, line)?; + for (_, arm_expr) in arms.iter_mut() { + resolve_expr_imported_calls(ctx, arm_expr, line)?; + } + resolve_expr_imported_calls(ctx, default, line)?; + } + Expr::Block { stmts, expr } => { + for stmt in stmts.iter_mut() { + resolve_stmt_imported_calls(ctx, stmt)?; + } + resolve_expr_imported_calls(ctx, expr, line)?; + } + } + Ok(()) +} + +/// Source line of one statement, used to attribute unresolved/ambiguous +/// imported-call diagnostics to the owning module source. +fn stmt_line(stmt: &Stmt) -> u32 { + match stmt { + Stmt::Noop { line } + | Stmt::Break { line } + | Stmt::Continue { line } + | Stmt::Drop { line, .. } + | Stmt::ClosureLet { line, .. } + | Stmt::FuncDecl { line, .. } + | Stmt::Let { line, .. } + | Stmt::Assign { line, .. } + | Stmt::Expr { line, .. } + | Stmt::IfElse { line, .. } + | Stmt::For { line, .. } + | Stmt::While { line, .. } => *line, + } +} diff --git a/src/compiler/source_loader/imports.rs b/src/compiler/source_loader/imports.rs index 41281466..b8f8615c 100644 --- a/src/compiler/source_loader/imports.rs +++ b/src/compiler/source_loader/imports.rs @@ -1,10 +1,14 @@ -use std::path::{Path, PathBuf}; +use std::path::{Component, Path, PathBuf}; use crate::builtins::is_builtin_namespace; use super::super::frontends::{is_ident_continue, is_ident_start}; -use super::super::{CompileSourceFileOptions, SourceFlavor, SourcePathError}; -use super::model::{ImportClause, ModuleImport, NamedImport}; +use super::super::modules::{UseDecl, use_path_to_spec}; +use super::super::{ + CompileSourceFileOptions, SharedParserOptions, SourceError, SourceFlavor, SourcePathError, + frontends, +}; +use super::model::ModuleImport; pub(super) fn parse_module_imports( source: &str, @@ -12,235 +16,94 @@ pub(super) fn parse_module_imports( path: &Path, options: &CompileSourceFileOptions, ) -> Result, SourcePathError> { - match flavor { - SourceFlavor::RustScript => parse_rustscript_imports(source, path), - SourceFlavor::JavaScript | SourceFlavor::Lua => options - .source_plugin_for_flavor(flavor) - .ok_or(SourcePathError::MissingFrontendPlugin(flavor))? - .parse_module_imports(source, path), - } + scan_module_imports(source, flavor, path, options).map(|(imports, _)| imports) } -fn parse_rustscript_imports( +/// Scan the module imports of one source. +/// +/// For RustScript this parses the source once with the real frontend parser +/// and consumes the structured `use` declaration nodes, so import discovery +/// shares the parser's spans, clauses, and syntax validation instead of +/// treating line-prefix stripping as the authoritative import parser. The +/// paired [`UseDecl`] list is returned alongside the legacy `ModuleImport` +/// list so graph construction can preserve spans. Other flavors keep their +/// plugin-based discovery and contribute no structured declarations. +pub(super) fn scan_module_imports( source: &str, + flavor: SourceFlavor, path: &Path, -) -> Result, SourcePathError> { - let mut imports = Vec::new(); - for (idx, raw_line) in source.lines().enumerate() { - let line_no = idx + 1; - let line = raw_line.trim(); - if line.starts_with("import ") { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line: line_no, - message: "RustScript uses 'use', not 'import'".to_string(), - }); + options: &CompileSourceFileOptions, +) -> Result<(Vec, Vec), SourcePathError> { + match flavor { + SourceFlavor::RustScript => { + let decls = parse_rustscript_use_declarations(source, path)?; + let imports = use_declarations_to_module_imports(path, &decls)?; + Ok((imports, decls)) } - if !line.starts_with("use ") { - continue; + SourceFlavor::JavaScript | SourceFlavor::Lua => { + let imports = options + .source_plugin_for_flavor(flavor) + .ok_or(SourcePathError::MissingFrontendPlugin(flavor))? + .parse_module_imports(source, path)?; + Ok((imports, Vec::new())) } - let tail = line["use ".len()..].trim(); - let (spec, clause) = parse_rustscript_use(path, line_no, tail)?; - imports.push(ModuleImport { - spec, - clause, - line: line_no, - }); } - Ok(imports) } -fn parse_rustscript_use( +/// Parse all `use` directives of a RustScript source into structured nodes. +/// +/// The whole source is parsed with the real frontend parser (with implicit +/// externs enabled and file-path host aliases recorded) so that discovery +/// tolerates calls to functions imported from other modules, which the +/// loader's semantic resolution pass resolves later. +fn parse_rustscript_use_declarations( + source: &str, path: &Path, - line: usize, - tail: &str, -) -> Result<(String, ImportClause), SourcePathError> { - let Some((directive_body, _)) = tail.split_once(';') else { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected ';' at end of use directive".to_string(), - }); - }; - let directive_body = directive_body.trim(); - if directive_body.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected module path after 'use'".to_string(), - }); - } - - if let Some(module_path) = directive_body.strip_suffix("::*") { - let spec = rustscript_use_module_to_spec(path, line, module_path.trim())?; - return Ok((spec, ImportClause::AllPublic)); - } - - if let Some(open_idx) = directive_body.find("::{") { - if !directive_body.ends_with('}') { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected '}' to close use list".to_string(), - }); - } - let module_path = directive_body[..open_idx].trim(); - let spec = rustscript_use_module_to_spec(path, line, module_path)?; - let inner = directive_body[open_idx + 3..directive_body.len() - 1].trim(); - if inner == "*" { - return Ok((spec, ImportClause::AllPublic)); - } - if inner.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "use list requires at least one symbol".to_string(), - }); - } - let named = - parse_named_imports(inner).ok_or_else(|| SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: - "invalid use list; expected comma-separated names with optional 'as' aliases" - .to_string(), - })?; - if named.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "use list requires at least one symbol".to_string(), - }); - } - return Ok((spec, ImportClause::Named(named))); - } - - if let Some((module_path, alias)) = directive_body.rsplit_once(" as ") { - let spec = rustscript_use_module_to_spec(path, line, module_path.trim())?; - let alias = alias.trim(); - if !is_valid_ident(alias) { +) -> Result, SourcePathError> { + for (idx, raw_line) in source.lines().enumerate() { + let line = raw_line.trim(); + if line.starts_with("import ") { return Err(SourcePathError::InvalidImportSyntax { path: path.to_path_buf(), - line, - message: "invalid namespace alias in use directive".to_string(), + line: idx + 1, + message: "RustScript uses 'use', not 'import'".to_string(), }); } - return Ok((spec, ImportClause::Namespace(alias.to_string()))); } - let spec = rustscript_use_module_to_spec(path, line, directive_body)?; - Ok((spec, ImportClause::AllPublic)) + let options = CompileSourceFileOptions::default(); + let dialect = frontends::parser_dialect_for_flavor(SourceFlavor::RustScript, &options) + .expect("RustScript parser dialect is always registered"); + let ir = frontends::parse_source_with_dialect( + source, + dialect, + SharedParserOptions { + source_id: 0, + allow_implicit_externs: true, + allow_implicit_semicolons: false, + enforce_mutable_bindings: true, + import_scan_mode: true, + }, + ) + .map_err(|err| SourcePathError::Source(SourceError::Parse(err)))?; + Ok(ir.use_declarations) } -fn rustscript_use_module_to_spec( +fn use_declarations_to_module_imports( path: &Path, - line: usize, - module_path: &str, -) -> Result { - if module_path.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected module path after 'use'".to_string(), - }); - } - let segments = module_path - .split("::") - .map(|segment| segment.trim()) - .collect::>(); - if segments.iter().any(|segment| segment.is_empty()) { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "invalid module path in use directive".to_string(), - }); - } - - let mut path_prefix = PathBuf::new(); - let mut cursor = 0usize; - while cursor < segments.len() { - match segments[cursor] { - "self" => cursor += 1, - "super" => { - path_prefix.push(".."); - cursor += 1; - } - "crate" => { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "crate:: paths are not supported; use relative module paths" - .to_string(), - }); - } - _ => break, - } - } - - if cursor >= segments.len() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected module name after path qualifiers".to_string(), - }); - } - - for segment in &segments[cursor..] { - if !is_valid_ident(segment) { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: format!("invalid module path segment '{segment}' in use directive"), - }); - } - path_prefix.push(segment); - } - - let mut spec = path_prefix.to_string_lossy().replace('\\', "/"); - if spec.is_empty() { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line, - message: "expected module path after 'use'".to_string(), - }); - } - if !spec.ends_with(".rss") { - spec.push_str(".rss"); - } - Ok(spec) -} - -fn parse_named_imports(input: &str) -> Option> { - let mut named = Vec::new(); - for part in input.split(',') { - let entry = part.trim(); - if entry.is_empty() { - continue; - } - - if let Some((imported, local)) = entry.split_once(" as ") { - let imported = imported.trim(); - let local = local.trim(); - if !is_valid_ident(imported) || !is_valid_ident(local) { - return None; - } - named.push(NamedImport { - imported: imported.to_string(), - local: local.to_string(), - }); - continue; - } - - if !is_valid_ident(entry) { - return None; - } - named.push(NamedImport { - imported: entry.to_string(), - local: entry.to_string(), - }); - } - - Some(named) + decls: &[UseDecl], +) -> Result, SourcePathError> { + decls + .iter() + .map(|decl| { + let spec = use_path_to_spec(path, decl.line, &decl.path)?; + Ok(ModuleImport { + spec, + clause: decl.clause.clone(), + line: decl.line, + }) + }) + .collect() } pub(super) fn is_valid_ident(input: &str) -> bool { @@ -278,7 +141,7 @@ pub(super) fn resolve_module_path( if path.extension().and_then(|value| value.to_str()) != Some("rss") { return Err(SourcePathError::NonRustScriptModule(path)); } - return Ok(path); + return Ok(module_identity(path)); } if options.module_override_source(spec).is_some() { let parent = base_path @@ -295,7 +158,7 @@ pub(super) fn resolve_module_path( if path.extension().and_then(|value| value.to_str()) != Some("rss") { return Err(SourcePathError::NonRustScriptModule(path)); } - return Ok(path); + return Ok(module_identity(path)); } let parent = base_path @@ -312,71 +175,67 @@ pub(super) fn resolve_module_path( if path.extension().and_then(|value| value.to_str()) != Some("rss") { return Err(SourcePathError::NonRustScriptModule(path)); } - Ok(path) + Ok(module_identity(path)) +} + +/// Resolve the module identity for a normalized path. +/// +/// Files that exist on disk use their canonical path so that lexically +/// distinct but equivalent paths (`.`, `..`, symlinks) collapse to one +/// module identity for `seen`/`visiting`/exports/overrides. Paths that do not +/// exist on disk (virtual source overrides, in-memory entry points) keep the +/// normalized lexical path as their explicit virtual identity. +pub(super) fn module_identity(path: PathBuf) -> PathBuf { + let normalized = normalize_module_path(path); + if normalized.is_file() + && let Ok(canonical) = normalized.canonicalize() + { + return canonical; + } + normalized +} + +fn normalize_module_path(path: PathBuf) -> PathBuf { + let mut normalized = PathBuf::new(); + for component in path.components() { + match component { + Component::CurDir => {} + Component::ParentDir => match normalized.components().next_back() { + Some(Component::Normal(_)) => { + normalized.pop(); + } + Some(Component::ParentDir) | None => normalized.push(component.as_os_str()), + Some(Component::RootDir | Component::Prefix(_)) => {} + Some(Component::CurDir) => { + unreachable!("normalized paths omit current-dir components") + } + }, + Component::RootDir | Component::Prefix(_) | Component::Normal(_) => { + normalized.push(component.as_os_str()); + } + } + } + normalized } +/// Prepare source text for the compile parse. +/// +/// RustScript no longer strips `use` directives: the parser consumes every +/// directive into a structured node (host-namespace forms keep their existing +/// dedicated handling), so line-prefix stripping is no longer an authority +/// for import discovery. Other flavors keep plugin-defined stripping. pub(super) fn strip_import_directives( source: &str, flavor: SourceFlavor, options: &CompileSourceFileOptions, ) -> Result { - let stripped = match flavor { - SourceFlavor::RustScript => source - .lines() - .map(|line| { - if line.trim_start().starts_with("use ") - && !is_direct_host_namespace_use_directive_line(line.trim_start()) - && !is_builtin_namespace_use_directive_line(line.trim_start()) - { - String::new() - } else { - line.to_string() - } - }) - .collect::>() - .join("\n"), - SourceFlavor::JavaScript | SourceFlavor::Lua => options + match flavor { + SourceFlavor::RustScript => Ok(source.to_string()), + SourceFlavor::JavaScript | SourceFlavor::Lua => Ok(options .source_plugin_for_flavor(flavor) .ok_or(SourcePathError::MissingFrontendPlugin(flavor))? - .strip_import_directives(source), - }; - Ok(stripped) -} - -fn is_direct_host_namespace_use_directive_line(line: &str) -> bool { - let trimmed = line.trim(); - if !trimmed.starts_with("use ") { - return false; - } - let Some((directive_body, _)) = trimmed["use ".len()..].split_once(';') else { - return false; - }; - let directive_body = directive_body.trim(); - if directive_body.contains("::{") || directive_body.ends_with("::*") { - return false; + .strip_import_directives(source)), } - if let Some((namespace, alias)) = directive_body.split_once(" as ") { - return is_virtual_host_namespace_spec( - namespace.trim(), - &CompileSourceFileOptions::default(), - ) && is_valid_ident(alias.trim()); - } - is_virtual_host_namespace_spec(directive_body, &CompileSourceFileOptions::default()) -} - -fn is_builtin_namespace_use_directive_line(line: &str) -> bool { - let trimmed = line.trim(); - if !trimmed.starts_with("use ") { - return false; - } - let Some((directive_body, _)) = trimmed["use ".len()..].split_once(';') else { - return false; - }; - let directive_body = directive_body.trim(); - if let Some((namespace, _alias)) = directive_body.split_once(" as ") { - return is_builtin_namespace(namespace.trim()); - } - is_builtin_namespace(directive_body) } pub(super) fn host_namespace_root_from_spec(spec: &str) -> Option { @@ -415,3 +274,145 @@ pub(super) fn should_treat_missing_module_as_host_namespace( std::io::ErrorKind::NotFound | std::io::ErrorKind::Unsupported ) && is_virtual_host_namespace_spec(spec, options) } + +#[cfg(test)] +mod tests { + use super::super::super::modules::UsePathSegment; + use super::super::SourceFlavor; + use super::super::model::ImportClause; + use super::{ + module_identity, normalize_module_path, parse_module_imports, scan_module_imports, + }; + use std::path::PathBuf; + + #[test] + fn normalize_module_path_preserves_unmatched_parent_components() { + assert_eq!( + normalize_module_path(PathBuf::from("../foo/../../bar")), + PathBuf::from("../../bar") + ); + assert_eq!( + normalize_module_path(PathBuf::from("foo/../../../bar")), + PathBuf::from("../../bar") + ); + } + + #[cfg(unix)] + #[test] + fn normalize_module_path_does_not_escape_absolute_root() { + assert_eq!( + normalize_module_path(PathBuf::from("/foo/../../bar")), + PathBuf::from("/bar") + ); + } + + #[test] + fn module_identity_keeps_normalized_virtual_path_for_missing_files() { + assert_eq!( + module_identity(PathBuf::from("/no/such/dir/../virtual/nested.rss")), + PathBuf::from("/no/such/virtual/nested.rss") + ); + } + + #[test] + fn module_identity_uses_canonical_path_for_existing_files() { + let unique = format!( + "pd-vm-module-identity-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp root should be created"); + let module = root.join("a.rss"); + std::fs::write(&module, "pub fn value() -> int { 1 }\n").expect("module should write"); + + let via_dot = module_identity(root.join("./a.rss")); + let via_parent = module_identity(root.join("sub/../a.rss")); + let canonical = module.canonicalize().expect("module should canonicalize"); + + assert_eq!(via_dot, canonical); + assert_eq!(via_parent, canonical); + assert_eq!(via_dot, via_parent); + + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn structured_scan_preserves_spans_clauses_and_lines() { + let source = "use self::nested as nested;\nuse sibling::{value as v, other};\nuse super::shared;\nuse io;\n"; + let path = PathBuf::from("/root/pkg/main.rss"); + let (imports, decls) = + scan_module_imports(source, SourceFlavor::RustScript, &path, &Default::default()) + .expect("scan should succeed"); + + assert_eq!(imports.len(), 4); + assert_eq!(imports[0].spec, "./nested.rss"); + assert_eq!(imports[1].spec, "sibling.rss"); + assert_eq!(imports[2].spec, "../shared.rss"); + assert_eq!(imports[3].spec, "io.rss"); + + assert_eq!(decls.len(), 4); + assert_eq!( + decls[0].path, + vec![ + UsePathSegment::Self_, + UsePathSegment::Ident("nested".to_string()) + ] + ); + assert!(matches!(&decls[0].clause, ImportClause::Namespace(alias) if alias == "nested")); + assert_eq!(decls[0].line, 1); + assert_eq!(decls[1].line, 2); + assert!( + decls[0].span.lo < decls[0].span.hi, + "span must cover the directive" + ); + assert!(matches!(&decls[1].clause, ImportClause::Named(named) if named.len() == 2)); + assert!(matches!(&decls[3].clause, ImportClause::AllPublic)); + } + + #[test] + fn structured_scan_handles_wildcard_and_alias_forms() { + let source = "use a::b::*;\nuse c::d::{x};\nuse e as f;\n"; + let path = PathBuf::from("/root/main.rss"); + let (imports, decls) = + scan_module_imports(source, SourceFlavor::RustScript, &path, &Default::default()) + .expect("scan should succeed"); + + assert_eq!(imports[0].spec, "a/b.rss"); + assert!(matches!(imports[0].clause, ImportClause::AllPublic)); + assert_eq!(imports[1].spec, "c/d.rss"); + assert!(matches!(&imports[1].clause, ImportClause::Named(named) if named.len() == 1)); + assert_eq!(imports[2].spec, "e.rss"); + assert!(matches!(&imports[2].clause, ImportClause::Namespace(alias) if alias == "f")); + assert_eq!(decls[1].path.len(), 2); + } + + #[test] + fn structured_scan_rejects_import_keyword() { + let source = "import \"./module.rss\";\n"; + let path = PathBuf::from("/root/main.rss"); + let err = + parse_module_imports(source, SourceFlavor::RustScript, &path, &Default::default()) + .expect_err("import keyword should be rejected"); + assert!( + err.to_string().contains("uses 'use', not 'import'"), + "unexpected error: {err}" + ); + } + + #[test] + fn structured_scan_rejects_crate_paths() { + let source = "use crate::x;\n"; + let path = PathBuf::from("/root/main.rss"); + let err = + parse_module_imports(source, SourceFlavor::RustScript, &path, &Default::default()) + .expect_err("crate:: paths should be rejected"); + assert!( + err.to_string().contains("crate:: paths are not supported"), + "unexpected error: {err}" + ); + } +} diff --git a/src/compiler/source_loader/line_map.rs b/src/compiler/source_loader/line_map.rs deleted file mode 100644 index 34853d48..00000000 --- a/src/compiler/source_loader/line_map.rs +++ /dev/null @@ -1,165 +0,0 @@ -use super::super::ir::{Expr, FrontendIr, Stmt}; - -pub(super) fn remap_frontend_ir_line_numbers(ir: &mut FrontendIr, prelude_lines: usize) { - let offset = u32::try_from(prelude_lines).unwrap_or(u32::MAX); - for stmt in &mut ir.stmts { - remap_stmt_line_numbers(stmt, offset); - } - for function in ir.function_impls.values_mut() { - for stmt in &mut function.body_stmts { - remap_stmt_line_numbers(stmt, offset); - } - remap_expr_line_numbers(&mut function.body_expr, offset); - } -} - -fn remap_line(line: &mut u32, offset: u32) { - *line = (*line).saturating_sub(offset).max(1); -} - -fn remap_stmt_line_numbers(stmt: &mut Stmt, offset: u32) { - match stmt { - Stmt::Noop { line } - | Stmt::Break { line } - | Stmt::Continue { line } - | Stmt::Drop { line, .. } - | Stmt::ClosureLet { line, .. } - | Stmt::FuncDecl { line, .. } => remap_line(line, offset), - Stmt::Let { expr, line, .. } - | Stmt::Assign { expr, line, .. } - | Stmt::Expr { expr, line } => { - remap_line(line, offset); - remap_expr_line_numbers(expr, offset); - } - Stmt::IfElse { - condition, - then_branch, - else_branch, - line, - } => { - remap_line(line, offset); - remap_expr_line_numbers(condition, offset); - for stmt in then_branch { - remap_stmt_line_numbers(stmt, offset); - } - for stmt in else_branch { - remap_stmt_line_numbers(stmt, offset); - } - } - Stmt::For { - init, - condition, - post, - body, - line, - } => { - remap_line(line, offset); - remap_stmt_line_numbers(init, offset); - remap_expr_line_numbers(condition, offset); - remap_stmt_line_numbers(post, offset); - for stmt in body { - remap_stmt_line_numbers(stmt, offset); - } - } - Stmt::While { - condition, - body, - line, - } => { - remap_line(line, offset); - remap_expr_line_numbers(condition, offset); - for stmt in body { - remap_stmt_line_numbers(stmt, offset); - } - } - } -} - -fn remap_expr_line_numbers(expr: &mut Expr, offset: u32) { - match expr { - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { - for arg in args { - remap_expr_line_numbers(arg, offset); - } - } - Expr::ClosureCall(closure, args) => { - remap_closure_line_numbers(closure, offset); - for arg in args { - remap_expr_line_numbers(arg, offset); - } - } - Expr::Closure(closure) => remap_closure_line_numbers(closure, offset), - Expr::OptionalGet { container, key, .. } => { - remap_expr_line_numbers(container, offset); - remap_expr_line_numbers(key, offset); - } - Expr::OptionUnwrapOr { - value, fallback, .. - } => { - remap_expr_line_numbers(value, offset); - remap_expr_line_numbers(fallback, offset); - } - Expr::Add(lhs, rhs) - | Expr::Sub(lhs, rhs) - | Expr::Mul(lhs, rhs) - | Expr::Div(lhs, rhs) - | Expr::Mod(lhs, rhs) - | Expr::And(lhs, rhs) - | Expr::Or(lhs, rhs) - | Expr::Eq(lhs, rhs) - | Expr::Lt(lhs, rhs) - | Expr::Gt(lhs, rhs) => { - remap_expr_line_numbers(lhs, offset); - remap_expr_line_numbers(rhs, offset); - } - Expr::Neg(inner) - | Expr::Not(inner) - | Expr::ToOwned(inner) - | Expr::Borrow(inner) - | Expr::BorrowMut(inner) => { - remap_expr_line_numbers(inner, offset); - } - Expr::IfElse { - condition, - then_expr, - else_expr, - } => { - remap_expr_line_numbers(condition, offset); - remap_expr_line_numbers(then_expr, offset); - remap_expr_line_numbers(else_expr, offset); - } - Expr::Match { - value, - arms, - default, - .. - } => { - remap_expr_line_numbers(value, offset); - for (_, arm_expr) in arms { - remap_expr_line_numbers(arm_expr, offset); - } - remap_expr_line_numbers(default, offset); - } - Expr::Block { stmts, expr } => { - for stmt in stmts { - remap_stmt_line_numbers(stmt, offset); - } - remap_expr_line_numbers(expr, offset); - } - Expr::Null - | Expr::Int(_) - | Expr::Float(_) - | Expr::Bool(_) - | Expr::Bytes(_) - | Expr::String(_) - | Expr::FunctionRef(..) - | Expr::Var(_) - | Expr::MoveVar(_) - | Expr::MoveField { .. } - | Expr::MoveIndex { .. } => {} - } -} - -fn remap_closure_line_numbers(closure: &mut crate::compiler::ir::ClosureExpr, offset: u32) { - remap_expr_line_numbers(&mut closure.body, offset); -} diff --git a/src/compiler/source_loader/model.rs b/src/compiler/source_loader/model.rs index 172ff65f..4199de19 100644 --- a/src/compiler/source_loader/model.rs +++ b/src/compiler/source_loader/model.rs @@ -2,7 +2,8 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use super::super::linker::ParsedUnit; - +use super::super::modules::ModuleGraph; +use super::super::source_map::SourceMap; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FrontendImportSyntax { RustScript, @@ -43,7 +44,10 @@ pub(super) struct ModuleCollectState { pub(super) seen: HashSet, pub(super) units: Vec, pub(super) module_exports: HashMap>, -} -pub(super) struct ImportRewriteResult { - pub(super) source: String, + pub(super) module_graph: ModuleGraph, + /// Compilation-wide source map keyed by the module graph's + /// [`SourceId`](super::super::modules::SourceId) space. Every module's + /// raw text is registered here at its graph source id so spans produced + /// during load and merge resolve to the owning source for diagnostics. + pub(super) sources: SourceMap, } diff --git a/src/compiler/source_loader/rewrite.rs b/src/compiler/source_loader/rewrite.rs deleted file mode 100644 index 4dd305b9..00000000 --- a/src/compiler/source_loader/rewrite.rs +++ /dev/null @@ -1,780 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; - -use super::super::frontends::{is_ident_continue, is_ident_start}; -use super::super::{CompileSourceFileOptions, SourceFlavor, SourcePathError}; - -use super::imports::{ - host_namespace_root_from_spec, is_builtin_host_namespace_spec, is_module_specifier, - is_valid_ident, is_virtual_host_namespace_spec, resolve_module_path, -}; -use super::model::{ExportedFunctionSignature, ImportClause, ImportRewriteResult, ModuleImport}; - -struct ImportCallResolution { - alias_calls: HashMap, - namespace_calls: HashMap>, - namespace_prefix_calls: HashMap, -} - -fn resolve_import_call_paths( - flavor: SourceFlavor, - path: &Path, - imports: &[ModuleImport], - module_exports: &HashMap>, - options: &CompileSourceFileOptions, -) -> Result { - let mut alias_calls = HashMap::::new(); - let mut namespace_calls = HashMap::>::new(); - let namespace_prefix_calls = HashMap::::new(); - for import in imports { - if is_builtin_host_namespace_spec(&import.spec) { - continue; - } - if !is_module_specifier(&import.spec) { - if let Some(host_root) = host_namespace_root_from_spec(&import.spec) - && is_virtual_host_namespace_spec(&import.spec, options) - && let Some(host_prefix) = virtual_host_namespace_prefix(flavor, &host_root) - { - match &import.clause { - ImportClause::AllPublic => {} - ImportClause::Named(named) => { - for binding in named { - alias_calls.insert( - binding.local.clone(), - format!("{host_prefix}::{}", binding.imported), - ); - } - } - ImportClause::Namespace(_namespace) => {} - ImportClause::Prefix(_) => {} - } - } - continue; - } - - let resolved = resolve_module_path(path, &import.spec, options)?; - let Some(exports) = module_exports.get(&resolved) else { - if let Some(host_root) = host_namespace_root_from_spec(&import.spec) - && is_virtual_host_namespace_spec(&import.spec, options) - && let Some(host_prefix) = virtual_host_namespace_prefix(flavor, &host_root) - { - match &import.clause { - ImportClause::AllPublic => {} - ImportClause::Named(named) => { - for binding in named { - alias_calls.insert( - binding.local.clone(), - format!("{host_prefix}::{}", binding.imported), - ); - } - } - ImportClause::Namespace(_namespace) => {} - ImportClause::Prefix(_) => {} - } - } - continue; - }; - - match &import.clause { - ImportClause::AllPublic => { - // Bare `use module;` keeps direct calls (`fn_name(...)`) and now also - // supports namespace-style calls (`module::fn_name(...)`) for ergonomics. - if let Some(namespace) = module_default_namespace(&import.spec) { - let entries = namespace_calls.entry(namespace).or_default(); - for name in exports.keys() { - entries.insert(name.clone()); - } - } - } - ImportClause::Named(named) => { - for binding in named { - if !exports.contains_key(&binding.imported) { - return Err(SourcePathError::InvalidImportSyntax { - path: path.to_path_buf(), - line: import.line, - message: format!( - "module '{}' has no public function '{}'", - import.spec, binding.imported - ), - }); - } - if binding.local != binding.imported { - alias_calls.insert(binding.local.clone(), binding.imported.clone()); - } - } - } - ImportClause::Namespace(namespace) => { - let entries = namespace_calls.entry(namespace.clone()).or_default(); - for name in exports.keys() { - entries.insert(name.clone()); - } - } - ImportClause::Prefix(prefix) => { - for name in exports.keys() { - alias_calls.insert(format!("{prefix}{name}"), name.clone()); - } - } - } - } - - Ok(ImportCallResolution { - alias_calls, - namespace_calls, - namespace_prefix_calls, - }) -} - -fn virtual_host_namespace_prefix(flavor: SourceFlavor, host_root: &str) -> Option { - match flavor { - SourceFlavor::RustScript => Some(host_root.to_string()), - SourceFlavor::JavaScript | SourceFlavor::Lua => None, - } -} - -pub(super) fn rewrite_imported_call_sites( - source: &str, - flavor: SourceFlavor, - path: &Path, - imports: &[ModuleImport], - module_exports: &HashMap>, - options: &CompileSourceFileOptions, -) -> Result { - let resolution = resolve_import_call_paths(flavor, path, imports, module_exports, options)?; - let alias_calls = resolution.alias_calls; - let namespace_calls = resolution.namespace_calls; - let namespace_prefix_calls = resolution.namespace_prefix_calls; - let namespace_wildcards = HashSet::::new(); - let prefix_aliases = Vec::::new(); - - let rewritten = rewrite_host_namespace_call_paths(source, flavor, &namespace_prefix_calls); - - if alias_calls.is_empty() - && namespace_calls.is_empty() - && namespace_wildcards.is_empty() - && prefix_aliases.is_empty() - { - return Ok(ImportRewriteResult { source: rewritten }); - } - - Ok(ImportRewriteResult { - source: rewrite_function_call_paths( - &rewritten, - flavor, - &alias_calls, - &namespace_calls, - &namespace_wildcards, - &prefix_aliases, - ), - }) -} - -fn rewrite_host_namespace_call_paths( - source: &str, - flavor: SourceFlavor, - namespace_prefix_calls: &HashMap, -) -> String { - if namespace_prefix_calls.is_empty() { - return source.to_string(); - } - - let bytes = source.as_bytes(); - let mut out = String::with_capacity(source.len()); - let mut i = 0usize; - let mut in_line_comment = false; - let mut in_block_comment = false; - let mut string_delim: Option = None; - let mut escaped = false; - - while i < bytes.len() { - let b = bytes[i]; - - if let Some(delim) = string_delim { - out.push(b as char); - if escaped { - escaped = false; - } else if b == b'\\' { - escaped = true; - } else if b == delim { - string_delim = None; - } - i += 1; - continue; - } - - if in_line_comment { - out.push(b as char); - if b == b'\n' { - in_line_comment = false; - } - i += 1; - continue; - } - - if in_block_comment { - out.push(b as char); - if b == b'*' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { - out.push('/'); - i += 2; - in_block_comment = false; - continue; - } - i += 1; - continue; - } - - if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { - out.push('/'); - out.push('/'); - i += 2; - in_line_comment = true; - continue; - } - - if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { - out.push('/'); - out.push('*'); - i += 2; - in_block_comment = true; - continue; - } - - if b == b'"' || b == b'\'' || b == b'`' { - out.push(b as char); - i += 1; - string_delim = Some(b); - escaped = false; - continue; - } - - if !is_ident_start(b as char) { - out.push(b as char); - i += 1; - continue; - } - - let start = i; - i += 1; - while i < bytes.len() && is_ident_continue(bytes[i] as char) { - i += 1; - } - let ident = &source[start..i]; - - if let Some(prefix) = namespace_prefix_calls.get(ident) - && namespace_call_target_is_function(source, i, flavor) - { - out.push_str(prefix); - continue; - } - - out.push_str(ident); - } - - out -} - -fn namespace_call_target_is_function(source: &str, index: usize, flavor: SourceFlavor) -> bool { - let bytes = source.as_bytes(); - let mut cursor = index; - if !consume_namespace_separator(bytes, &mut cursor, flavor) { - return false; - } - - loop { - while cursor < bytes.len() - && bytes[cursor].is_ascii_whitespace() - && bytes[cursor] != b'\n' - && bytes[cursor] != b'\r' - { - cursor += 1; - } - if cursor >= bytes.len() || !is_ident_start(bytes[cursor] as char) { - return false; - } - cursor += 1; - while cursor < bytes.len() && is_ident_continue(bytes[cursor] as char) { - cursor += 1; - } - - while cursor < bytes.len() - && bytes[cursor].is_ascii_whitespace() - && bytes[cursor] != b'\n' - && bytes[cursor] != b'\r' - { - cursor += 1; - } - if cursor < bytes.len() && bytes[cursor] == b'(' { - return true; - } - if !consume_namespace_separator(bytes, &mut cursor, flavor) { - return false; - } - } -} - -fn consume_namespace_separator(bytes: &[u8], cursor: &mut usize, flavor: SourceFlavor) -> bool { - if *cursor >= bytes.len() { - return false; - } - if flavor == SourceFlavor::RustScript { - if bytes[*cursor] != b':' { - return false; - } - *cursor += 1; - while *cursor < bytes.len() - && bytes[*cursor].is_ascii_whitespace() - && bytes[*cursor] != b'\n' - && bytes[*cursor] != b'\r' - { - *cursor += 1; - } - if *cursor >= bytes.len() || bytes[*cursor] != b':' { - return false; - } - *cursor += 1; - return true; - } - - if bytes[*cursor] == b'.' { - *cursor += 1; - return true; - } - false -} - -fn module_default_namespace(spec: &str) -> Option { - let stem = Path::new(spec).file_stem()?.to_str()?; - if is_valid_ident(stem) { - Some(stem.to_string()) - } else { - None - } -} - -fn skip_inline_whitespace(bytes: &[u8], mut index: usize) -> usize { - while index < bytes.len() - && bytes[index].is_ascii_whitespace() - && bytes[index] != b'\n' - && bytes[index] != b'\r' - { - index += 1; - } - index -} - -fn rustscript_turbofish_call_starts(bytes: &[u8], start: usize) -> bool { - let mut index = skip_inline_whitespace(bytes, start); - if index >= bytes.len() || bytes[index] != b':' { - return false; - } - index = skip_inline_whitespace(bytes, index + 1); - if index >= bytes.len() || bytes[index] != b':' { - return false; - } - index = skip_inline_whitespace(bytes, index + 1); - if index >= bytes.len() || bytes[index] != b'<' { - return false; - } - - let mut depth = 0usize; - while index < bytes.len() { - match bytes[index] { - b'<' => depth += 1, - b'>' => { - depth = depth.saturating_sub(1); - if depth == 0 { - index += 1; - break; - } - } - b'\n' | b'\r' => return false, - _ => {} - } - index += 1; - } - - if depth != 0 { - return false; - } - - index = skip_inline_whitespace(bytes, index); - index < bytes.len() && bytes[index] == b'(' -} - -fn call_starts_after_position(bytes: &[u8], start: usize, flavor: SourceFlavor) -> bool { - let index = skip_inline_whitespace(bytes, start); - if index < bytes.len() && bytes[index] == b'(' { - return true; - } - flavor == SourceFlavor::RustScript && rustscript_turbofish_call_starts(bytes, start) -} - -fn rewrite_function_call_paths( - source: &str, - flavor: SourceFlavor, - alias_calls: &HashMap, - namespace_calls: &HashMap>, - namespace_wildcards: &HashSet, - prefix_aliases: &[String], -) -> String { - let bytes = source.as_bytes(); - let mut out = String::with_capacity(source.len()); - let mut i = 0usize; - let mut in_line_comment = false; - let mut in_block_comment = false; - let mut string_delim: Option = None; - let mut escaped = false; - - while i < bytes.len() { - let b = bytes[i]; - - if let Some(delim) = string_delim { - out.push(b as char); - if escaped { - escaped = false; - } else if b == b'\\' { - escaped = true; - } else if b == delim { - string_delim = None; - } - i += 1; - continue; - } - - if in_line_comment { - out.push(b as char); - if b == b'\n' { - in_line_comment = false; - } - i += 1; - continue; - } - - if in_block_comment { - out.push(b as char); - if b == b'*' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { - out.push('/'); - i += 2; - in_block_comment = false; - continue; - } - i += 1; - continue; - } - - if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'/' { - out.push('/'); - out.push('/'); - i += 2; - in_line_comment = true; - continue; - } - - if b == b'/' && i + 1 < bytes.len() && bytes[i + 1] == b'*' { - out.push('/'); - out.push('*'); - i += 2; - in_block_comment = true; - continue; - } - - if b == b'"' || b == b'\'' || b == b'`' { - out.push(b as char); - i += 1; - string_delim = Some(b); - escaped = false; - continue; - } - - if is_ident_start(b as char) { - let start = i; - i += 1; - while i < bytes.len() && is_ident_continue(bytes[i] as char) { - i += 1; - } - let ident = &source[start..i]; - - let namespace_methods = namespace_calls.get(ident); - let namespace_wildcard = namespace_wildcards.contains(ident); - if namespace_methods.is_some() || namespace_wildcard { - let mut j = i; - while j < bytes.len() - && bytes[j].is_ascii_whitespace() - && bytes[j] != b'\n' - && bytes[j] != b'\r' - { - j += 1; - } - - let mut sep_end = None; - if flavor == SourceFlavor::RustScript { - if j < bytes.len() && bytes[j] == b':' { - let mut k = j + 1; - while k < bytes.len() - && bytes[k].is_ascii_whitespace() - && bytes[k] != b'\n' - && bytes[k] != b'\r' - { - k += 1; - } - if k < bytes.len() && bytes[k] == b':' { - sep_end = Some(k + 1); - } - } - } else if j < bytes.len() && bytes[j] == b'.' { - sep_end = Some(j + 1); - } - - if let Some(mut k) = sep_end { - while k < bytes.len() - && bytes[k].is_ascii_whitespace() - && bytes[k] != b'\n' - && bytes[k] != b'\r' - { - k += 1; - } - if k < bytes.len() && is_ident_start(bytes[k] as char) { - let member_start = k; - k += 1; - while k < bytes.len() && is_ident_continue(bytes[k] as char) { - k += 1; - } - let member = &source[member_start..k]; - if call_starts_after_position(bytes, k, flavor) - && (namespace_wildcard - || namespace_methods - .is_some_and(|methods| methods.contains(member))) - { - out.push_str(member); - i = k; - continue; - } - } - } - } - - if let Some(target) = alias_calls.get(ident) - && call_starts_after_position(bytes, i, flavor) - { - out.push_str(target); - continue; - } - - let mut rewritten_by_prefix = false; - for prefix in prefix_aliases { - if !ident.starts_with(prefix) { - continue; - } - let rem = &ident[prefix.len()..]; - if rem.is_empty() || !is_valid_ident(rem) { - continue; - } - if call_starts_after_position(bytes, i, flavor) { - out.push_str(rem); - rewritten_by_prefix = true; - break; - } - } - if rewritten_by_prefix { - continue; - } - - out.push_str(ident); - continue; - } - - out.push(b as char); - i += 1; - } - - out -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::path::{Path, PathBuf}; - - use super::*; - use crate::compiler::CompileSourceFileOptions; - use crate::compiler::source_loader::model::{ - ExportedFunctionSignature, ImportClause, ModuleImport, NamedImport, - }; - - #[test] - fn rustscript_namespace_import_calls_rewrite_to_direct_calls() { - let source = r#" -string::non_empty("rss"); -is_empty(""); -"#; - let path = Path::new("tests/main.rss"); - let imports = vec![ - ModuleImport { - spec: "strings.rss".to_string(), - clause: ImportClause::Namespace("string".to_string()), - line: 1, - }, - ModuleImport { - spec: "strings.rss".to_string(), - clause: ImportClause::Named(vec![NamedImport { - imported: "is_empty".to_string(), - local: "is_empty".to_string(), - }]), - line: 2, - }, - ]; - let mut module_exports = - HashMap::>::new(); - module_exports.insert( - PathBuf::from("tests").join("strings.rss"), - HashMap::from([ - ( - "is_empty".to_string(), - ExportedFunctionSignature { - arity: 1, - type_params: Vec::new(), - }, - ), - ( - "non_empty".to_string(), - ExportedFunctionSignature { - arity: 1, - type_params: Vec::new(), - }, - ), - ]), - ); - - let rewritten = rewrite_imported_call_sites( - source, - SourceFlavor::RustScript, - path, - &imports, - &module_exports, - &CompileSourceFileOptions::default(), - ) - .expect("rewrite should succeed"); - - assert_eq!( - rewritten.source.trim(), - r#" -non_empty("rss"); -is_empty(""); -"# - .trim() - ); - } - - #[test] - fn rustscript_namespace_import_turbofish_calls_rewrite_to_direct_calls() { - let source = r#"collections::dedup::(["rss", "rss"]);"#; - let path = Path::new("tests/main.rss"); - let imports = vec![ModuleImport { - spec: "collections.rss".to_string(), - clause: ImportClause::Namespace("collections".to_string()), - line: 1, - }]; - let mut module_exports = - HashMap::>::new(); - module_exports.insert( - PathBuf::from("tests").join("collections.rss"), - HashMap::from([( - "dedup".to_string(), - ExportedFunctionSignature { - arity: 1, - type_params: vec!["T".to_string()], - }, - )]), - ); - - let rewritten = rewrite_imported_call_sites( - source, - SourceFlavor::RustScript, - path, - &imports, - &module_exports, - &CompileSourceFileOptions::default(), - ) - .expect("rewrite should succeed"); - - assert_eq!( - rewritten.source.trim(), - r#"dedup::(["rss", "rss"]);"# - ); - } - - #[test] - fn rustscript_named_import_turbofish_calls_rewrite_to_direct_calls() { - let source = r#"dedup_items::(["rss", "rss"]);"#; - let path = Path::new("tests/main.rss"); - let imports = vec![ModuleImport { - spec: "collections.rss".to_string(), - clause: ImportClause::Named(vec![NamedImport { - imported: "dedup".to_string(), - local: "dedup_items".to_string(), - }]), - line: 1, - }]; - let mut module_exports = - HashMap::>::new(); - module_exports.insert( - PathBuf::from("tests").join("collections.rss"), - HashMap::from([( - "dedup".to_string(), - ExportedFunctionSignature { - arity: 1, - type_params: vec!["T".to_string()], - }, - )]), - ); - - let rewritten = rewrite_imported_call_sites( - source, - SourceFlavor::RustScript, - path, - &imports, - &module_exports, - &CompileSourceFileOptions::default(), - ) - .expect("rewrite should succeed"); - - assert_eq!( - rewritten.source.trim(), - r#"dedup::(["rss", "rss"]);"# - ); - } - - #[test] - fn rustscript_all_public_import_namespace_calls_rewrite_to_direct_calls() { - let source = "runtime::sleep(3);\n"; - let path = Path::new("tests/main.rss"); - let imports = vec![ModuleImport { - spec: "runtime.rss".to_string(), - clause: ImportClause::AllPublic, - line: 1, - }]; - let mut module_exports = - HashMap::>::new(); - module_exports.insert( - PathBuf::from("tests").join("runtime.rss"), - HashMap::from([( - "sleep".to_string(), - ExportedFunctionSignature { - arity: 1, - type_params: Vec::new(), - }, - )]), - ); - - let rewritten = rewrite_imported_call_sites( - source, - SourceFlavor::RustScript, - path, - &imports, - &module_exports, - &CompileSourceFileOptions::default(), - ) - .expect("rewrite should succeed"); - - assert_eq!(rewritten.source.trim(), "sleep(3);"); - } -} diff --git a/src/compiler/source_map.rs b/src/compiler/source_map.rs index 5839a0c0..622648a8 100644 --- a/src/compiler/source_map.rs +++ b/src/compiler/source_map.rs @@ -118,6 +118,33 @@ impl SourceMap { id } + /// Register a source at an explicit id (the semantic module graph's + /// `SourceId` space) so spans that + /// reference that id resolve to this text. Missing slots are filled with + /// empty placeholders; an already-occupied slot keeps its first text. + pub fn add_source_at( + &mut self, + id: SourceId, + name: impl Into, + text: impl Into, + ) -> SourceId { + let id_usize = id as usize; + while self.files.len() <= id_usize { + let placeholder = self.files.len() as SourceId; + self.files + .push(SourceFile::new(placeholder, String::new(), String::new())); + } + if self.files[id_usize].text.is_empty() && self.files[id_usize].name.is_empty() { + self.files[id_usize] = SourceFile::new(id, name.into(), text.into()); + } + id + } + + /// Display name of the source registered at `id`. + pub fn file_name(&self, id: SourceId) -> Option<&str> { + self.file(id).map(|file| file.name.as_str()) + } + pub fn file(&self, id: SourceId) -> Option<&SourceFile> { self.files.get(id as usize) } diff --git a/src/compiler/typing/collect.rs b/src/compiler/typing/collect.rs index ac048e9a..359051ee 100644 --- a/src/compiler/typing/collect.rs +++ b/src/compiler/typing/collect.rs @@ -519,7 +519,9 @@ fn collect_expr_types( | Expr::MoveVar(_) | Expr::MoveField { .. } | Expr::MoveIndex { .. } - | Expr::FunctionRef(..) => { + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => { let _ = context.infer_expr_type(expr, state); } Expr::OptionalGet { container, key, .. } => { @@ -570,7 +572,7 @@ fn collect_expr_types( ); let _ = context.infer_expr_type(expr, state); } - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => { for arg in args { collect_expr_types( arg, diff --git a/src/compiler/typing/context.rs b/src/compiler/typing/context.rs index 89fc6501..efdfb045 100644 --- a/src/compiler/typing/context.rs +++ b/src/compiler/typing/context.rs @@ -922,8 +922,13 @@ impl<'a> TypeContext<'a> { state.get(*root) } } - Expr::FunctionRef(..) | Expr::Closure(_) => BoundType::Callable, - Expr::Call(..) | Expr::LocalCall(..) => self.infer_call_like_expr_type(expr, state), + Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } + | Expr::Closure(_) => BoundType::Callable, + Expr::Call(..) | Expr::ModuleCall(..) | Expr::LocalCall(..) => { + self.infer_call_like_expr_type(expr, state) + } Expr::ClosureCall(_, _) => self.infer_call_like_expr_type(expr, state), Expr::Add(lhs, rhs) | Expr::Sub(lhs, rhs) @@ -1061,7 +1066,14 @@ impl<'a> TypeContext<'a> { .unwrap_or(BoundType::Unknown), }, Expr::ClosureCall(closure, args) => self.infer_closure_return(closure, args, state), - Expr::Closure(_) | Expr::FunctionRef(..) => BoundType::Callable, + Expr::Closure(_) + | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } => BoundType::Callable, + // Resolved module calls carry no per-unit type information; the + // prelude declaration they replaced also had an unknown return + // type, so this matches the legacy behavior. + Expr::ModuleCall(..) => BoundType::Unknown, _ => BoundType::Unknown, } } diff --git a/src/compiler/typing/helpers.rs b/src/compiler/typing/helpers.rs index d8ea130c..28e76a8e 100644 --- a/src/compiler/typing/helpers.rs +++ b/src/compiler/typing/helpers.rs @@ -1370,6 +1370,8 @@ pub(super) fn expr_contains_param_add(expr: &Expr, param_slots: &[LocalSlot]) -> | Expr::Bytes(_) | Expr::String(_) | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } | Expr::Var(_) | Expr::MoveVar(_) | Expr::MoveField { .. } @@ -1384,7 +1386,7 @@ pub(super) fn expr_contains_param_add(expr: &Expr, param_slots: &[LocalSlot]) -> expr_contains_param_add(value, param_slots) || expr_contains_param_add(fallback, param_slots) } - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => args + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => args .iter() .any(|arg| expr_contains_param_add(arg, param_slots)), Expr::ClosureCall(closure, args) => { @@ -1445,6 +1447,8 @@ pub(super) fn expr_uses_param(expr: &Expr, param_slots: &[LocalSlot]) -> bool { | Expr::Bytes(_) | Expr::String(_) | Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } | Expr::MoveField { .. } | Expr::MoveIndex { .. } => false, Expr::OptionalGet { container, key, .. } => { @@ -1453,7 +1457,7 @@ pub(super) fn expr_uses_param(expr: &Expr, param_slots: &[LocalSlot]) -> bool { Expr::OptionUnwrapOr { value, fallback, .. } => expr_uses_param(value, param_slots) || expr_uses_param(fallback, param_slots), - Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) => { + Expr::Call(_, _, args) | Expr::LocalCall(_, _, args) | Expr::ModuleCall(_, _, args) => { args.iter().any(|arg| expr_uses_param(arg, param_slots)) } Expr::ClosureCall(closure, args) => { @@ -1603,7 +1607,13 @@ pub(super) fn legalize_expr( let _ = legalize_expr(fallback, state, context); context.infer_expr_type(expr, state) } - Expr::FunctionRef(..) | Expr::Call(..) | Expr::LocalCall(..) | Expr::Closure(_) => { + Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } + | Expr::Call(..) + | Expr::ModuleCall(..) + | Expr::LocalCall(..) + | Expr::Closure(_) => { legalize_expr_children(expr, state, context); context.infer_call_like_expr_type(expr, state) } @@ -1709,6 +1719,11 @@ pub(super) fn legalize_expr_children( fold_builtin_call(expr, builtin, state); } } + Expr::ModuleCall(_, _, args) => { + for arg in args.iter_mut() { + let _ = legalize_expr(arg, state, context); + } + } Expr::LocalCall(_, _, args) => { for arg in args.iter_mut() { let _ = legalize_expr(arg, state, context); diff --git a/src/compiler/typing/validate.rs b/src/compiler/typing/validate.rs index fab96ea6..6a2b61fe 100644 --- a/src/compiler/typing/validate.rs +++ b/src/compiler/typing/validate.rs @@ -556,7 +556,13 @@ pub(super) fn validate_expr( )?, Expr::Var(slot) | Expr::MoveVar(slot) => state.get(*slot), Expr::MoveField { root, .. } | Expr::MoveIndex { root, .. } => state.get(*root), - Expr::FunctionRef(..) | Expr::Call(..) | Expr::LocalCall(..) | Expr::Closure(_) => { + Expr::FunctionRef(..) + | Expr::ModuleFunctionRef(..) + | Expr::UnresolvedFunctionRef { .. } + | Expr::Call(..) + | Expr::ModuleCall(..) + | Expr::LocalCall(..) + | Expr::Closure(_) => { validate_expr_children( expr, state, diff --git a/src/lib.rs b/src/lib.rs index d28b0a5c..6037cd4e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,8 @@ pub use assembler::{AsmParseError, Assembler, AssemblerError, BytecodeBuilder, a #[cfg(feature = "runtime")] pub use builtins::runtime::HostCallResult; #[cfg(feature = "runtime")] +pub use builtins::runtime::HttpConfig; +#[cfg(feature = "runtime")] pub use builtins::runtime::print::{PrintHostFunction, PrintlnHostFunction, format_value}; pub use builtins::{ BUILTIN_CATALOG, BuiltinFunction, BuiltinNamespaceMemberSpec, BuiltinNamespaceSpec, @@ -35,21 +37,29 @@ pub use builtins::{ pub use bytecode::{ CallableEnvironment, CallableKind, CallablePrototype, CallableTarget, CallableValue, CaptureBindingMode, ExportedCallable, FunctionRegion, HostImport, OpCode, Program, - RootCallableBinding, ScriptFunction, TypeMap, Value, ValueType, + RootCallableBinding, ScriptFunction, TypeMap, Value, ValueType, VmMap, }; pub fn builtin_call_index(name: &str) -> Option { use builtins::BuiltinFunction; BuiltinFunction::from_source_name(name).map(|builtin| builtin.call_index()) } -pub use compiler::diagnostics::{render_compile_error, render_source_error}; +#[cfg(feature = "runtime")] +pub use builtins::runtime::error::{RuntimeError, RuntimeErrorCode, RuntimeResult}; +#[cfg(feature = "runtime")] +pub use builtins::runtime::event::{EventPayload, EventSink}; +pub use compiler::diagnostics::{ + render_compile_error, render_source_error, render_source_path_error, +}; pub use compiler::source_map::{LineSpanMapping, LoweredSource, SourceId, SourceMap, Span}; pub use compiler::{ AssignmentKind, ClosureExpr, CompileError, CompileSourceFileOptions, CompiledProgram, - CompiledReplProgram, Compiler, Expr, FormatError, FrontendImportSyntax, FrontendIr, - FunctionDecl, ImportClause, InferredLocalTypeHint, LocalIrBuilder, LocalSlot, ModuleImport, - NamedImport, ParseError, ParserDialect, ReplLocalBinding, ReplLocalState, SharedParserOptions, - SourceError, SourceFlavor, SourcePathError, SourcePlugin, Stmt, UnknownInferredLocal, + CompiledReplProgram, Compiler, DeclSymbol, ExportEntry, Expr, FormatError, + FrontendImportSyntax, FrontendIr, FunctionDecl, ImportClause, ImportTargetKind, + ImportedBinding, InferredLocalTypeHint, LocalIrBuilder, LocalSlot, ModuleGraph, ModuleId, + ModuleImport, ModuleNode, NamedImport, ParseError, ParserDialect, ReplLocalBinding, + ReplLocalState, ResolvedImport, SharedParserOptions, SourceError, SourceFlavor, + SourcePathError, SourcePlugin, Stmt, SymbolId, UnknownInferredLocal, UseDecl, UsePathSegment, collect_inferred_local_type_hints, collect_inferred_local_type_hints_at_path_with_options, collect_inferred_local_type_hints_with_options, compile_source, compile_source_at_path_with_flavor_and_options, compile_source_file, @@ -78,12 +88,15 @@ pub use jit::{ pub use vm::diagnostics::render_vm_error; #[cfg(feature = "runtime")] pub use vm::{ - AotArtifactError, CallOutcome, CallReturn, DEFAULT_MAX_SCRIPT_CALL_DEPTH, EpochCheckpoint, - EpochHandle, FuelCheckpoint, HostArgsFunction, HostAsyncBridge, HostBindingPlan, HostFunction, - HostFunctionRegistry, HostOpId, HostStackFunction, IntoScriptValue, QueuedScriptInvocation, - ScriptArgs, ScriptCallback, ScriptResult, StaticHostArgsFunction, StaticHostFunction, - StaticHostStackFunction, Store, Vm, VmError, VmResult, VmStatus, VmYieldReason, + AotArtifactError, CallOutcome, CallReturn, CancellationReason, DEFAULT_MAX_SCRIPT_CALL_DEPTH, + EpochCheckpoint, EpochHandle, FuelCheckpoint, HostArgsFunction, HostAsyncBridge, + HostBindingPlan, HostFunction, HostFunctionRegistry, HostOpId, HostStackFunction, + IntoScriptValue, QueuedScriptInvocation, ScriptArgs, ScriptCallback, ScriptResult, + StaticHostArgsFunction, StaticHostFunction, StaticHostStackFunction, Store, Vm, VmError, + VmResult, VmStatus, VmYieldReason, }; +#[cfg(feature = "sqlite")] +pub use vm::{SqliteLimits, SqlitePolicy}; #[cfg(feature = "runtime")] pub use vmbc::{ DisassembleOptions, ValidationError, WireError, decode_program, disassemble_program, diff --git a/src/vm/aot/artifact.rs b/src/vm/aot/artifact.rs index 37e31f83..1f944ed5 100644 --- a/src/vm/aot/artifact.rs +++ b/src/vm/aot/artifact.rs @@ -3,7 +3,8 @@ use std::path::Path; use crate::bytecode::Program; use crate::vm::native::{ - helper_entry_offset, interrupt_helper_entry_offset, selected_codegen_backend, + detect_native_stack_layout, helper_entry_offset, interrupt_helper_entry_offset, + native_stack_layout_fingerprint, selected_codegen_backend, }; use crate::vm::{Vm, VmError}; @@ -12,7 +13,7 @@ use super::compile::CompiledProgram; const MAGIC: [u8; 4] = *b"PAT\0"; const VERSION: u16 = 7; -const ABI_VERSION: u16 = 6; +const ABI_VERSION: u16 = 7; const FLAG_INTERPRETER_BOUNDARY_ONLY: u16 = 1; const SUPPORTED_FLAGS: u16 = FLAG_INTERPRETER_BOUNDARY_ONLY; @@ -108,11 +109,12 @@ impl From for AotArtifactError { impl Vm { pub fn encode_aot_artifact(&mut self) -> Result, AotArtifactError> { - if self.aot_program.is_none() { + if self.engine.aot_program.is_none() { self.compile_aot()?; } let program_hash = self.ensure_program_cache_key(); let aot_program = self + .engine .aot_program .as_ref() .ok_or(AotArtifactError::MissingAotProgram)?; @@ -135,8 +137,8 @@ impl Vm { } else { CompiledProgram::from_code(decoded.code, decoded.resume_ips)? }; - self.aot_program = Some(compiled); - self.aot_exec_count = 0; + self.engine.aot_program = Some(compiled); + self.engine.aot_exec_count = 0; Ok(()) } @@ -159,8 +161,8 @@ impl Vm { } else { CompiledProgram::from_code(decoded.code, decoded.resume_ips)? }; - vm.aot_program = Some(compiled); - vm.aot_exec_count = 0; + vm.engine.aot_program = Some(compiled); + vm.engine.aot_exec_count = 0; Ok(vm) } @@ -201,7 +203,8 @@ fn encode_artifact( write_string("os", std::env::consts::OS, &mut out)?; write_string("backend", selected_codegen_backend(), &mut out)?; - write_u32("vm ip offset", std::mem::offset_of!(Vm, ip), &mut out)?; + let native_layout_fingerprint = native_stack_layout_fingerprint(detect_native_stack_layout()?); + out.extend_from_slice(&native_layout_fingerprint.to_le_bytes()); write_u32( "native helper offset", helper_entry_offset() as usize, @@ -281,10 +284,12 @@ fn decode_artifact( selected_codegen_backend().to_string(), cursor.read_string()?, )?; + let expected_layout_fingerprint = + native_stack_layout_fingerprint(detect_native_stack_layout()?); validate_runtime_field( - "vm ip offset", - std::mem::offset_of!(Vm, ip).to_string(), - cursor.read_u32()?.to_string(), + "native stack layout fingerprint", + format!("{expected_layout_fingerprint:#018x}"), + format!("{:#018x}", cursor.read_u64()?), )?; validate_runtime_field( "native helper offset", @@ -438,15 +443,30 @@ impl<'a> Cursor<'a> { #[cfg(test)] mod tests { use super::*; + use crate::vm::native::{detect_native_stack_layout, native_stack_layout_fingerprint}; use crate::{BytecodeBuilder, Program, Value, ValueType, VmStatus}; + fn native_layout_fingerprint_offset(encoded: &[u8]) -> usize { + let mut offset = 11; + for _ in 0..3 { + let len = u32::from_le_bytes( + encoded[offset..offset + 4] + .try_into() + .expect("runtime string length"), + ) as usize; + offset += 4 + len; + } + offset + } + #[test] fn aot_artifact_preserves_interpreter_boundary_mode() { let mut bc = BytecodeBuilder::new(); bc.ret(); let mut vm = Vm::new(Program::new(Vec::new(), bc.finish())); vm.compile_aot().expect("aot compile should succeed"); - vm.aot_program + vm.engine + .aot_program .as_mut() .expect("compiled program") .interpreter_boundary_only = true; @@ -468,6 +488,7 @@ mod tests { .expect("boundary artifact should load"); assert!( standalone + .engine .aot_program .as_ref() .expect("loaded aot program") @@ -501,6 +522,74 @@ mod tests { )); } + #[test] + fn aot_artifact_decode_rejects_previous_native_layout_abi() { + let mut bc = BytecodeBuilder::new(); + bc.ret(); + let mut vm = Vm::new(Program::new(Vec::new(), bc.finish())); + vm.compile_aot().expect("aot compile should succeed"); + let mut encoded = vm + .encode_aot_artifact() + .expect("artifact encode should succeed"); + encoded[6..8].copy_from_slice(&6_u16.to_le_bytes()); + + assert!(matches!( + vm.load_aot_artifact(&encoded), + Err(AotArtifactError::UnsupportedAbiVersion(6)) + )); + } + + #[test] + fn aot_artifact_records_complete_native_layout_fingerprint() { + let mut bc = BytecodeBuilder::new(); + bc.ret(); + let mut vm = Vm::new(Program::new(Vec::new(), bc.finish())); + vm.compile_aot().expect("aot compile should succeed"); + let encoded = vm + .encode_aot_artifact() + .expect("artifact encode should succeed"); + let fingerprint_offset = native_layout_fingerprint_offset(&encoded); + let stored = u64::from_le_bytes( + encoded[fingerprint_offset..fingerprint_offset + 8] + .try_into() + .expect("native layout fingerprint"), + ); + let expected = native_stack_layout_fingerprint( + detect_native_stack_layout().expect("native layout should be detected"), + ); + + assert_eq!(stored, expected); + vm.load_aot_artifact(&encoded) + .expect("matching native layout should load"); + } + + #[test] + fn aot_artifact_rejects_native_layout_fingerprint_mismatch() { + let mut bc = BytecodeBuilder::new(); + bc.ret(); + let mut vm = Vm::new(Program::new(Vec::new(), bc.finish())); + vm.compile_aot().expect("aot compile should succeed"); + let mut encoded = vm + .encode_aot_artifact() + .expect("artifact encode should succeed"); + let fingerprint_offset = native_layout_fingerprint_offset(&encoded); + let stored = u64::from_le_bytes( + encoded[fingerprint_offset..fingerprint_offset + 8] + .try_into() + .expect("native layout fingerprint"), + ); + encoded[fingerprint_offset..fingerprint_offset + 8] + .copy_from_slice(&stored.wrapping_add(1).to_le_bytes()); + + assert!(matches!( + vm.load_aot_artifact(&encoded), + Err(AotArtifactError::IncompatibleRuntime { + field: "native stack layout fingerprint", + .. + }) + )); + } + #[test] fn aot_artifact_decode_rejects_incompatible_program_hash() { let mut first_bc = BytecodeBuilder::new(); @@ -581,7 +670,7 @@ mod tests { .encode_aot_artifact() .expect("artifact encode should succeed"); assert_eq!(u16::from_le_bytes([encoded[4], encoded[5]]), 7); - assert_eq!(u16::from_le_bytes([encoded[6], encoded[7]]), 6); + assert_eq!(u16::from_le_bytes([encoded[6], encoded[7]]), 7); let mut old_format = encoded.clone(); old_format[4..6].copy_from_slice(&6u16.to_le_bytes()); @@ -590,10 +679,10 @@ mod tests { Err(AotArtifactError::UnsupportedVersion(6)) )); let mut old_abi = encoded.clone(); - old_abi[6..8].copy_from_slice(&5u16.to_le_bytes()); + old_abi[6..8].copy_from_slice(&6u16.to_le_bytes()); assert!(matches!( Vm::new_from_aot_artifact_with_jit_config(&old_abi, JitConfig::default()), - Err(AotArtifactError::UnsupportedAbiVersion(5)) + Err(AotArtifactError::UnsupportedAbiVersion(6)) )); let mut standalone = diff --git a/src/vm/aot/compile.rs b/src/vm/aot/compile.rs index c8896093..96718ba7 100644 --- a/src/vm/aot/compile.rs +++ b/src/vm/aot/compile.rs @@ -566,7 +566,7 @@ fn compile_ssa( let ctx_setup_elapsed = ctx_setup_started.elapsed(); let vm_ip_offset = - i32::try_from(std::mem::offset_of!(Vm, ip)).expect("Vm::ip offset must fit i32"); + i32::try_from(std::mem::offset_of!(Vm, instance.ip)).expect("Vm::ip offset must fit i32"); let code_len_i64 = i64::try_from(program.code.len()) .map_err(|_| AotCompileError::Codegen("program length does not fit i64".to_string()))?; diff --git a/src/vm/aot/runtime.rs b/src/vm/aot/runtime.rs index 13029006..abfcb7e4 100644 --- a/src/vm/aot/runtime.rs +++ b/src/vm/aot/runtime.rs @@ -8,32 +8,33 @@ use crate::vm::{ExecOutcome, Vm, VmError, VmResult}; impl Vm { pub fn compile_aot(&mut self) -> VmResult<()> { - self.aot_program = Some(compile_program(self.program())?); - self.aot_exec_count = 0; + self.engine.aot_program = Some(compile_program(self.program())?); + self.engine.aot_exec_count = 0; Ok(()) } pub fn clear_aot(&mut self) { - self.aot_program = None; - self.aot_exec_count = 0; + self.engine.aot_program = None; + self.engine.aot_exec_count = 0; } pub fn has_aot_program(&self) -> bool { - self.aot_program.is_some() + self.engine.aot_program.is_some() } pub fn aot_exec_count(&self) -> u64 { - self.aot_exec_count + self.engine.aot_exec_count } pub fn aot_resume_ips(&self) -> Option<&[usize]> { - self.aot_program + self.engine + .aot_program .as_ref() .map(|program| program.resume_ips.as_ref()) } pub fn dump_aot_info(&self) -> String { - let Some(program) = self.aot_program.as_ref() else { + let Some(program) = self.engine.aot_program.as_ref() else { return "whole-program aot: disabled\n".to_string(); }; @@ -43,7 +44,10 @@ impl Vm { " native codegen backend: {}\n", selected_codegen_backend() )); - out.push_str(&format!(" aot executions: {}\n", self.aot_exec_count)); + out.push_str(&format!( + " aot executions: {}\n", + self.engine.aot_exec_count + )); out.push_str(&format!(" code_bytes={}\n", program.code.len())); out.push_str(&format!( " lowering={}\n", @@ -58,38 +62,51 @@ impl Vm { } pub(crate) fn execute_aot_entry(&mut self) -> VmResult { - let Some(entry) = self.aot_program.as_ref().map(|program| program.entry) else { + if !self.host.allow_default_host_capabilities { + self.engine.aot_interpreter_boundary_hit = true; + return Ok(ExecOutcome::Continue); + } + let Some(entry) = self + .engine + .aot_program + .as_ref() + .map(|program| program.entry) + else { return Ok(ExecOutcome::Continue); }; clear_bridge_error(); unsafe { crate::vm::native::prepare_for_execution() }; let status = unsafe { entry(self as *mut Vm) }; - self.aot_exec_count = self.aot_exec_count.saturating_add(1); + self.engine.aot_exec_count = self.engine.aot_exec_count.saturating_add(1); match status { STATUS_CONTINUE | STATUS_LINKED_CONTINUE => Ok(ExecOutcome::Continue), STATUS_HALTED => Ok(ExecOutcome::Halted), STATUS_YIELDED => { - self.last_yield_reason = Some(super::super::VmYieldReason::Host); + self.instance.last_yield_reason = Some(super::super::VmYieldReason::Host); Ok(ExecOutcome::Yielded) } STATUS_WAITING => { - let op_id = self.waiting_host_op.map(|op| op.op_id).ok_or_else(|| { - VmError::JitNative( - "aot call bridge reported waiting without a pending op".to_string(), - ) - })?; + let op_id = self + .instance + .waiting_host_op + .map(|op| op.op_id) + .ok_or_else(|| { + VmError::JitNative( + "aot call bridge reported waiting without a pending op".to_string(), + ) + })?; Ok(ExecOutcome::Waiting(op_id)) } - STATUS_OUT_OF_FUEL => match self.interrupt_mode { + STATUS_OUT_OF_FUEL => match self.run_ctx.interrupt_mode { super::super::InterruptMode::Fuel => Err(VmError::OutOfFuel { needed: 1, - remaining: self.fuel_remaining, + remaining: self.run_ctx.fuel_remaining, }), super::super::InterruptMode::Epoch => Err(VmError::EpochDeadlineReached { current: self.current_epoch(), - deadline: self.epoch_deadline, + deadline: self.run_ctx.epoch_deadline, }), super::super::InterruptMode::None => Err(VmError::JitNative( "aot interruption checkpoint fired while interruption was disabled".to_string(), @@ -99,18 +116,18 @@ impl Vm { if let Some(err) = take_bridge_error() { return Err(err); } - if self.ip == self.program.code.len() { + if self.instance.ip == self.program.code.len() { return Err(VmError::BytecodeBounds); } Err(VmError::JitNative(format!( "aot entry reported failure without VmError (ip={} stack_len={} aot={})", - self.ip, - self.stack.len(), + self.instance.ip, + self.instance.stack.len(), self.has_aot_program() ))) } STATUS_TRACE_EXIT => { - self.aot_interpreter_boundary_hit = true; + self.engine.aot_interpreter_boundary_hit = true; Ok(ExecOutcome::Continue) } other => Err(VmError::JitNative(format!( diff --git a/src/vm/engine.rs b/src/vm/engine.rs new file mode 100644 index 00000000..33acefe9 --- /dev/null +++ b/src/vm/engine.rs @@ -0,0 +1,141 @@ +//! Backend engine state. +//! +//! [`Engine`] owns the code-generation backends and their caches: the trace +//! JIT engine, native traces and their counters, the optional AOT program, +//! the regex cache, program-derived decode caches, and code-generation +//! telemetry. It holds no per-run interpreter state and no host bindings, so +//! it can be shared across runs (and, by construction, reused by any number of +//! instances that never share stacks or resources). +//! +//! Native ABI note: the JIT/AOT code generators read a handful of fields by +//! machine offset through `std::mem::offset_of!(Vm, engine.)`. The +//! field set and the offsets are part of the native ABI; see +//! `crate::vm::native::layout`. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::builtins::runtime::regex::RegexCache; +use crate::bytecode::{DecodedInstructionData, Program}; +use crate::vm::aot; +use crate::vm::jit; +use crate::vm::native; + +/// Engine-owned backend configuration, caches, and code-generation telemetry. +/// +/// Thread safety: `Engine` is not shared between threads (`TraceJitEngine` is +/// not `Sync`); one VM facade owns one engine. Clone semantics: `Engine` is +/// intentionally not `Clone` — duplicating it would duplicate native traces +/// and JIT bookkeeping that are keyed to one execution identity. +pub(crate) struct Engine { + pub(crate) jit: jit::TraceJitEngine, + pub(crate) native_traces: Vec>, + pub(crate) native_trace_exec_count: u64, + pub(crate) aot_program: Option, + pub(crate) aot_exec_count: u64, + pub(crate) aot_interpreter_boundary_hit: bool, + pub(crate) jit_native_region_entry_count: u64, + pub(crate) jit_native_region_edge_count: u64, + pub(crate) jit_native_direct_link_count: u64, + pub(crate) jit_native_direct_links_enabled: bool, + pub(crate) jit_native_direct_cross_frame_enabled: bool, + pub(crate) jit_native_active_direct_trace_id: usize, + pub(crate) jit_native_direct_escape_streak: u16, + pub(crate) jit_native_direct_region_fallback: bool, + pub(crate) jit_native_compile_time_ns: u64, + pub(crate) jit_native_region_compile_time_ns: u64, + pub(crate) jit_trace_exit_count: u64, + pub(crate) jit_native_loop_back_count: u64, + pub(crate) jit_native_link_handoff_count: u64, + pub(crate) jit_native_link_dispatch_depth: u32, + pub(crate) jit_helper_fallback_count: u64, + pub(crate) jit_native_bridge_stats_enabled: bool, + pub(crate) jit_native_bridge_counts: HashMap<&'static str, u64>, + pub(crate) program_cache_key: u64, + pub(crate) program_cache_key_ready: bool, + pub(crate) regex_cache: RegexCache, + pub(crate) decoded_instruction_data: Arc, + pub(crate) operand_type_hints: Option>, + // Native ABI mirrors: the JIT/AOT code generators load these addresses by + // field offset from the `Vm` facade. They are derived from the program and + // from static helper entry points, and are documented as load-bearing for + // `crate::vm::native`. + pub(crate) program_constants_ptr: usize, + #[allow(dead_code)] + pub(crate) program_constants_len: usize, + #[allow(dead_code)] + pub(crate) native_helper_fn: usize, + #[allow(dead_code)] + pub(crate) native_interrupt_helper_fn: usize, +} + +impl Engine { + /// Builds an engine for one program and JIT configuration. + pub(crate) fn new(jit_config: jit::JitConfig, program: &Program) -> Self { + Self { + jit: jit::TraceJitEngine::new(jit_config), + native_traces: Vec::new(), + native_trace_exec_count: 0, + aot_program: None, + aot_exec_count: 0, + aot_interpreter_boundary_hit: false, + jit_native_region_entry_count: 0, + jit_native_region_edge_count: 0, + jit_native_direct_link_count: 0, + jit_native_direct_links_enabled: true, + jit_native_direct_cross_frame_enabled: false, + jit_native_active_direct_trace_id: usize::MAX, + jit_native_direct_escape_streak: 0, + jit_native_direct_region_fallback: false, + jit_native_compile_time_ns: 0, + jit_native_region_compile_time_ns: 0, + jit_trace_exit_count: 0, + jit_native_loop_back_count: 0, + jit_native_link_handoff_count: 0, + jit_native_link_dispatch_depth: 0, + jit_helper_fallback_count: 0, + jit_native_bridge_stats_enabled: false, + jit_native_bridge_counts: HashMap::new(), + program_cache_key: 0, + program_cache_key_ready: false, + regex_cache: RegexCache::default(), + decoded_instruction_data: program.shared_decoded_instruction_data(), + operand_type_hints: program.shared_operand_type_hints(), + program_constants_ptr: program.constants.as_ptr() as usize, + program_constants_len: program.constants.len(), + native_helper_fn: native::helper_entry_address(), + native_interrupt_helper_fn: native::interrupt_helper_entry_address(), + } + } + + /// Returns the program cache key, computing and caching it on first use. + /// The key identifies the program for backend cache lookups; it is stable + /// for the lifetime of the engine (the program is immutable). + pub(crate) fn ensure_program_cache_key(&mut self, program: &Program) -> u64 { + if !self.program_cache_key_ready { + self.program_cache_key = super::compute_program_cache_key(program); + self.program_cache_key_ready = true; + } + self.program_cache_key + } + + /// Rewinds run-scoped backend state between runs while retaining compiled + /// artifacts: hot-entry bookkeeping and call-site profiles are cleared, + /// and the AOT boundary flag is recomputed from the compiled program. + pub(crate) fn reset_runtime_state(&mut self, program: &Program) { + self.aot_interpreter_boundary_hit = self + .aot_program + .as_ref() + .is_some_and(|compiled| compiled.interpreter_boundary_only); + self.jit.reset_runtime_backoff(); + self.jit.clear_call_site_profiles(); + let _ = program; + } + + /// Invalidates code-generation caches that may reference run-scoped + /// behavior (used when drop-contract event accounting is toggled). + pub(crate) fn invalidate_codegen_caches(&mut self) { + self.native_traces.clear(); + self.native_trace_exec_count = 0; + } +} diff --git a/src/vm/epoch.rs b/src/vm/epoch.rs index 178a1202..0c8b2d93 100644 --- a/src/vm/epoch.rs +++ b/src/vm/epoch.rs @@ -57,56 +57,40 @@ impl EpochHandle { impl Vm { #[inline(always)] pub(in crate::vm) fn charge_epoch_tick(&mut self) -> VmResult<()> { - if !self.epoch_interruption_enabled() { - return Ok(()); - } - if self.fuel_ops_until_check > 1 { - self.fuel_ops_until_check -= 1; - return Ok(()); - } - - let current = self.current_epoch(); - if current >= self.epoch_deadline { - return Err(VmError::EpochDeadlineReached { - current, - deadline: self.epoch_deadline, - }); - } - self.fuel_ops_until_check = self.fuel_check_interval; - Ok(()) + self.run_ctx.charge_epoch_tick() } #[inline(always)] pub(super) fn mark_interrupt_yield(&mut self, reason: VmYieldReason) { - self.last_yield_reason = Some(reason); + self.instance.last_yield_reason = Some(reason); if matches!(reason, VmYieldReason::Epoch) { - self.epoch_rearm_pending = true; + self.run_ctx.epoch_rearm_pending = true; } } #[inline(always)] pub(super) fn rearm_epoch_after_yield_if_needed(&mut self) { - if !self.epoch_rearm_pending { + if !self.run_ctx.epoch_rearm_pending { return; } if !self.epoch_interruption_enabled() { - self.epoch_rearm_pending = false; + self.run_ctx.epoch_rearm_pending = false; return; } - self.epoch_deadline = self + self.run_ctx.epoch_deadline = self .current_epoch() - .saturating_add(self.epoch_deadline_delta); - self.epoch_rearm_pending = false; + .saturating_add(self.run_ctx.epoch_deadline_delta); + self.run_ctx.epoch_rearm_pending = false; self.reset_interrupt_countdown(); } pub(super) fn clear_epoch_deadline_internal(&mut self) { if self.epoch_interruption_enabled() { - self.interrupt_mode = InterruptMode::None; + self.run_ctx.interrupt_mode = InterruptMode::None; } - self.epoch_deadline = 0; - self.epoch_deadline_delta = 0; - self.epoch_rearm_pending = false; + self.run_ctx.epoch_deadline = 0; + self.run_ctx.epoch_deadline_delta = 0; + self.run_ctx.epoch_rearm_pending = false; self.reset_interrupt_countdown(); } @@ -118,29 +102,29 @@ impl Vm { } pub fn epoch_handle(&self) -> EpochHandle { - self.epoch_handle.clone() + self.run_ctx.epoch_handle.clone() } pub fn current_epoch(&self) -> u64 { - self.epoch_handle.current() + self.run_ctx.epoch_handle.current() } pub fn increment_epoch(&self) -> u64 { - self.epoch_handle.increment() + self.run_ctx.epoch_handle.increment() } pub fn increment_epoch_by(&self, delta: u64) -> u64 { - self.epoch_handle.increment_by(delta) + self.run_ctx.epoch_handle.increment_by(delta) } pub fn set_epoch_deadline(&mut self, ticks_beyond_current: u64) -> VmResult<()> { if self.fuel_metering_enabled() { return Err(self.interruption_mode_conflict(InterruptMode::Epoch)); } - self.interrupt_mode = InterruptMode::Epoch; - self.epoch_deadline = self.current_epoch().saturating_add(ticks_beyond_current); - self.epoch_deadline_delta = ticks_beyond_current; - self.epoch_rearm_pending = false; + self.run_ctx.interrupt_mode = InterruptMode::Epoch; + self.run_ctx.epoch_deadline = self.current_epoch().saturating_add(ticks_beyond_current); + self.run_ctx.epoch_deadline_delta = ticks_beyond_current; + self.run_ctx.epoch_rearm_pending = false; self.reset_interrupt_countdown(); Ok(()) } @@ -151,12 +135,12 @@ impl Vm { pub fn epoch_deadline(&self) -> Option { self.epoch_interruption_enabled() - .then_some(self.epoch_deadline) + .then_some(self.run_ctx.epoch_deadline) } pub fn epoch_deadline_delta(&self) -> Option { self.epoch_interruption_enabled() - .then_some(self.epoch_deadline_delta) + .then_some(self.run_ctx.epoch_deadline_delta) } pub fn set_epoch_check_interval(&mut self, interval: u32) -> VmResult<()> { @@ -166,7 +150,7 @@ impl Vm { if self.fuel_metering_enabled() { return Err(self.interruption_mode_conflict(InterruptMode::Epoch)); } - self.fuel_check_interval = interval; + self.run_ctx.fuel_check_interval = interval; self.reset_interrupt_countdown(); Ok(()) } @@ -179,31 +163,31 @@ impl Vm { EpochCheckpoint { deadline: self .epoch_interruption_enabled() - .then_some(self.epoch_deadline), - deadline_delta: self.epoch_deadline_delta, - rearm_pending: self.epoch_rearm_pending, + .then_some(self.run_ctx.epoch_deadline), + deadline_delta: self.run_ctx.epoch_deadline_delta, + rearm_pending: self.run_ctx.epoch_rearm_pending, check_interval: self.epoch_check_interval(), - ops_until_check: self.fuel_ops_until_check, + ops_until_check: self.run_ctx.fuel_ops_until_check, } } pub fn restore_epoch(&mut self, checkpoint: EpochCheckpoint) { self.clear_fuel_internal(); - self.interrupt_mode = if checkpoint.deadline.is_some() { + self.run_ctx.interrupt_mode = if checkpoint.deadline.is_some() { InterruptMode::Epoch } else { InterruptMode::None }; - self.epoch_deadline = checkpoint.deadline.unwrap_or(0); - self.epoch_deadline_delta = checkpoint.deadline_delta; - self.epoch_rearm_pending = checkpoint.rearm_pending; - self.fuel_check_interval = checkpoint.check_interval.max(1); - self.fuel_ops_until_check = checkpoint + self.run_ctx.epoch_deadline = checkpoint.deadline.unwrap_or(0); + self.run_ctx.epoch_deadline_delta = checkpoint.deadline_delta; + self.run_ctx.epoch_rearm_pending = checkpoint.rearm_pending; + self.run_ctx.fuel_check_interval = checkpoint.check_interval.max(1); + self.run_ctx.fuel_ops_until_check = checkpoint .ops_until_check - .clamp(1, self.fuel_check_interval); + .clamp(1, self.run_ctx.fuel_check_interval); } pub fn last_yield_reason(&self) -> Option { - self.last_yield_reason + self.instance.last_yield_reason } } diff --git a/src/vm/fuel.rs b/src/vm/fuel.rs index f7a3e9d1..4af4b090 100644 --- a/src/vm/fuel.rs +++ b/src/vm/fuel.rs @@ -19,61 +19,28 @@ impl FuelCheckpoint { impl Vm { pub(super) fn pending_fuel_debt(&self) -> u64 { - if !self.fuel_metering_enabled() { - return 0; - } - let executed_since_last_check = self - .fuel_check_interval - .saturating_sub(self.fuel_ops_until_check); - u64::from(executed_since_last_check) + self.run_ctx.pending_fuel_debt() } #[inline(always)] pub(in crate::vm) fn charge_fuel(&mut self, amount: u64) -> VmResult<()> { - if amount == 0 || !self.fuel_metering_enabled() { - return Ok(()); - } - - let remaining = self.fuel_remaining; - if remaining < amount { - return Err(VmError::OutOfFuel { - needed: amount, - remaining, - }); - } - self.fuel_remaining = remaining - amount; - Ok(()) + self.run_ctx.charge_fuel(amount) } #[inline(always)] pub(in crate::vm) fn charge_fuel_tick(&mut self) -> VmResult<()> { - if !self.fuel_metering_enabled() { - return Ok(()); - } - if self.fuel_ops_until_check > 1 { - self.fuel_ops_until_check -= 1; - return Ok(()); - } - - let amount = u64::from(self.fuel_check_interval); - self.charge_fuel(amount)?; - self.fuel_ops_until_check = self.fuel_check_interval; - Ok(()) + self.run_ctx.charge_fuel_tick() } pub(super) fn clear_fuel_internal(&mut self) { - if self.fuel_metering_enabled() { - self.interrupt_mode = InterruptMode::None; - } - self.fuel_remaining = 0; - self.reset_interrupt_countdown(); + self.run_ctx.clear_fuel_internal(); } pub fn set_fuel(&mut self, fuel: u64) { - self.clear_epoch_deadline_internal(); - self.interrupt_mode = InterruptMode::Fuel; - self.fuel_remaining = fuel; - self.reset_interrupt_countdown(); + self.run_ctx.clear_epoch_deadline_internal(); + self.run_ctx.interrupt_mode = InterruptMode::Fuel; + self.run_ctx.fuel_remaining = fuel; + self.run_ctx.reset_interrupt_countdown(); } pub fn clear_fuel(&mut self) { @@ -87,18 +54,21 @@ impl Vm { if self.epoch_interruption_enabled() { return Err(self.interruption_mode_conflict(InterruptMode::Fuel)); } - self.fuel_check_interval = interval; - self.reset_interrupt_countdown(); + self.run_ctx.fuel_check_interval = interval; + self.run_ctx.reset_interrupt_countdown(); Ok(()) } pub fn fuel_check_interval(&self) -> u32 { - self.fuel_check_interval + self.run_ctx.fuel_check_interval } pub fn get_fuel(&self) -> Option { - self.fuel_metering_enabled() - .then_some(self.fuel_remaining.saturating_sub(self.pending_fuel_debt())) + self.fuel_metering_enabled().then_some( + self.run_ctx + .fuel_remaining + .saturating_sub(self.pending_fuel_debt()), + ) } pub fn add_fuel(&mut self, fuel: u64) -> VmResult<()> { @@ -108,13 +78,14 @@ impl Vm { if self.epoch_interruption_enabled() { return Err(self.interruption_mode_conflict(InterruptMode::Fuel)); } - self.fuel_remaining = if self.fuel_metering_enabled() { - self.fuel_remaining + self.run_ctx.fuel_remaining = if self.fuel_metering_enabled() { + self.run_ctx + .fuel_remaining .checked_add(fuel) .ok_or(VmError::FuelOverflow)? } else { - self.interrupt_mode = InterruptMode::Fuel; - self.reset_interrupt_countdown(); + self.run_ctx.interrupt_mode = InterruptMode::Fuel; + self.run_ctx.reset_interrupt_countdown(); fuel }; Ok(()) @@ -140,9 +111,11 @@ impl Vm { pub fn fuel_checkpoint(&self) -> FuelCheckpoint { FuelCheckpoint { - remaining: self.fuel_metering_enabled().then_some(self.fuel_remaining), + remaining: self + .fuel_metering_enabled() + .then_some(self.run_ctx.fuel_remaining), check_interval: self.fuel_check_interval(), - ops_until_check: self.fuel_ops_until_check, + ops_until_check: self.run_ctx.fuel_ops_until_check, } } @@ -151,17 +124,17 @@ impl Vm { } pub fn restore_fuel(&mut self, checkpoint: FuelCheckpoint) { - self.clear_epoch_deadline_internal(); - self.interrupt_mode = if checkpoint.remaining.is_some() { + self.run_ctx.clear_epoch_deadline_internal(); + self.run_ctx.interrupt_mode = if checkpoint.remaining.is_some() { InterruptMode::Fuel } else { InterruptMode::None }; - self.fuel_remaining = checkpoint.remaining.unwrap_or(0); - self.fuel_check_interval = checkpoint.check_interval.max(1); - self.fuel_ops_until_check = checkpoint + self.run_ctx.fuel_remaining = checkpoint.remaining.unwrap_or(0); + self.run_ctx.fuel_check_interval = checkpoint.check_interval.max(1); + self.run_ctx.fuel_ops_until_check = checkpoint .ops_until_check - .clamp(1, self.fuel_check_interval); + .clamp(1, self.run_ctx.fuel_check_interval); } pub fn restore_checkpoint(&mut self, checkpoint: FuelCheckpoint) { diff --git a/src/vm/host.rs b/src/vm/host.rs index ecca479a..d139079c 100644 --- a/src/vm/host.rs +++ b/src/vm/host.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock, RwLock}; use std::task::{Context, Poll, Wake, Waker}; @@ -89,6 +90,10 @@ pub trait HostAsyncBridge: Send { fn poll_op(&mut self, op_id: HostOpId, cx: &mut Context<'_>) -> Poll>; fn cancel_op(&mut self, _op_id: HostOpId) {} + + fn cancel_op_with_reason(&mut self, op_id: HostOpId, _reason: CancellationReason) { + self.cancel_op(op_id); + } } pub type StaticHostFunction = fn(&mut Vm, &[Value]) -> VmResult; @@ -114,6 +119,7 @@ enum RegistryEntryKind { struct RegistryEntry { arity: u8, kind: RegistryEntryKind, + runtime_owned_pending: bool, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -121,6 +127,16 @@ pub struct HostBindingPlan { import_signature: Vec, registry_slots: Vec, resolved_calls: Vec, + runtime_owned_pending_slots: Vec, + allowed_builtin_calls: Vec, + allow_default_builtin_capabilities: bool, + allowed_host_function_slots: Vec, + allow_default_host_capabilities: bool, + capability_profile: Arc<()>, + capability_state: Arc<()>, + registry_state: Arc<()>, + registry_generation_token: Arc<()>, + registry_generation: u64, } #[derive(Clone)] @@ -128,6 +144,15 @@ pub struct HostFunctionRegistry { entries: Arc>, by_name: Arc>, plan_cache: Arc, Arc>>>, + allowed_builtin_calls: Arc>, + allow_default_builtin_capabilities: bool, + allowed_host_registry_slots: Arc>, + allow_default_host_capabilities: bool, + capability_profile: Arc<()>, + capability_state: Arc<()>, + registry_state: Arc<()>, + registry_generation_token: Arc<()>, + registry_generation: Arc, } impl Default for HostFunctionRegistry { @@ -137,30 +162,103 @@ impl Default for HostFunctionRegistry { } impl HostFunctionRegistry { - fn empty() -> Self { + pub fn empty() -> Self { Self { entries: Arc::new(Vec::new()), by_name: Arc::new(HashMap::new()), plan_cache: Arc::new(RwLock::new(HashMap::new())), + allowed_builtin_calls: Arc::new(Vec::new()), + allow_default_builtin_capabilities: true, + allowed_host_registry_slots: Arc::new(Vec::new()), + allow_default_host_capabilities: true, + capability_profile: Arc::new(()), + capability_state: Arc::new(()), + registry_state: Arc::new(()), + registry_generation_token: Arc::new(()), + registry_generation: Arc::new(AtomicU64::new(0)), } } pub fn new() -> Self { static DEFAULT_REGISTRY: OnceLock = OnceLock::new(); - DEFAULT_REGISTRY + let mut registry = DEFAULT_REGISTRY .get_or_init(|| { let mut registry = Self::empty(); crate::builtins::runtime::register_default_host_functions(&mut registry); + registry.allow_default_builtin_capabilities = true; + registry.allow_default_host_capabilities = true; registry }) - .clone() + .clone(); + registry.plan_cache = Arc::new(RwLock::new(HashMap::new())); + registry.capability_profile = Arc::new(()); + registry.capability_state = Arc::new(()); + registry.registry_state = Arc::new(()); + registry.registry_generation_token = Arc::new(()); + registry.registry_generation = Arc::new(AtomicU64::new(0)); + registry + } + + /// Returns the standard host registry with every registered host function present but + /// requiring an explicit capability grant before execution. + pub fn restricted() -> Self { + let mut registry = Self::new(); + registry.allow_default_builtin_capabilities = false; + registry.allow_default_host_capabilities = false; + registry.allowed_host_registry_slots = Arc::new(Vec::new()); + registry.capability_profile = Arc::new(()); + registry.capability_state = Arc::new(()); + registry.registry_state = Arc::new(()); + registry.registry_generation_token = Arc::new(()); + registry.registry_generation = Arc::new(AtomicU64::new(0)); + registry.invalidate_plan_cache(); + registry + } + + /// Explicitly permits a namespaced builtin when this registry is used as a capability plan. + pub fn allow_builtin(&mut self, name: impl AsRef) -> VmResult<()> { + let name = name.as_ref(); + if let Some(®istry_slot) = self.by_name.get(name) { + let slots = Arc::make_mut(&mut self.allowed_host_registry_slots); + if !slots.contains(®istry_slot) { + slots.push(registry_slot); + slots.sort_unstable(); + } + self.capability_state = Arc::new(()); + self.invalidate_plan_cache(); + return Ok(()); + } + let builtin = BuiltinFunction::from_namespaced_name(name) + .ok_or_else(|| VmError::HostError(format!("unknown namespaced builtin '{name}'")))?; + let calls = Arc::make_mut(&mut self.allowed_builtin_calls); + if !calls.contains(&builtin.call_index()) { + calls.push(builtin.call_index()); + calls.sort_unstable(); + } + self.capability_state = Arc::new(()); + self.invalidate_plan_cache(); + Ok(()) } fn invalidate_plan_cache(&mut self) { + self.registry_state = Arc::new(()); + self.registry_generation.fetch_add(1, Ordering::Relaxed); self.plan_cache = Arc::new(RwLock::new(HashMap::new())); } + pub(crate) fn mark_runtime_owned_pending(&mut self, name: &str) { + let slot = self + .by_name + .get(name) + .copied() + .expect("generated runtime host function should be registered"); + let entry = Arc::make_mut(&mut self.entries) + .get_mut(slot as usize) + .expect("generated runtime host function slot should exist"); + entry.runtime_owned_pending = true; + } + pub fn register(&mut self, name: impl Into, arity: u8, factory: F) where F: Fn() -> Box + Send + Sync + 'static, @@ -170,6 +268,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::Factory(Arc::new(factory)); self.invalidate_plan_cache(); return; @@ -179,6 +278,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::Factory(Arc::new(factory)), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -196,6 +296,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::Static(function); self.invalidate_plan_cache(); return; @@ -205,6 +306,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::Static(function), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -220,6 +322,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::StackFactory(Arc::new(factory)); self.invalidate_plan_cache(); return; @@ -229,6 +332,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::StackFactory(Arc::new(factory)), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -246,6 +350,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::StackStatic(function); self.invalidate_plan_cache(); return; @@ -255,6 +360,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::StackStatic(function), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -270,6 +376,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::ArgsFactory(Arc::new(factory)); self.invalidate_plan_cache(); return; @@ -279,6 +386,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::ArgsFactory(Arc::new(factory)), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -296,6 +404,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::ArgsStatic(function); self.invalidate_plan_cache(); return; @@ -305,6 +414,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::ArgsStatic(function), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -328,6 +438,7 @@ impl HostFunctionRegistry { && let Some(entry) = Arc::make_mut(&mut self.entries).get_mut(slot as usize) { entry.arity = arity; + entry.runtime_owned_pending = false; entry.kind = RegistryEntryKind::ArgsStaticNonYielding(function); self.invalidate_plan_cache(); return; @@ -337,6 +448,7 @@ impl HostFunctionRegistry { let slot = entries.len() as u16; entries.push(RegistryEntry { arity, + runtime_owned_pending: false, kind: RegistryEntryKind::ArgsStaticNonYielding(function), }); Arc::make_mut(&mut self.by_name).insert(name, slot); @@ -356,6 +468,17 @@ impl HostFunctionRegistry { self.plan_for_imports(imports) } + fn plan_matches_current(&self, plan: &HostBindingPlan) -> bool { + Arc::ptr_eq(&self.capability_profile, &plan.capability_profile) + && Arc::ptr_eq(&self.capability_state, &plan.capability_state) + && Arc::ptr_eq(&self.registry_state, &plan.registry_state) + && Arc::ptr_eq( + &self.registry_generation_token, + &plan.registry_generation_token, + ) + && self.registry_generation.load(Ordering::Relaxed) == plan.registry_generation + } + fn plan_for_imports(&self, imports: &[HostImport]) -> VmResult> { if let Some(plan) = self .plan_cache @@ -363,6 +486,7 @@ impl HostFunctionRegistry { .expect("host binding plan cache read lock should not be poisoned") .get(imports) .cloned() + && self.plan_matches_current(&plan) { return Ok(plan); } @@ -400,17 +524,48 @@ impl HostFunctionRegistry { resolved_calls.push(vm_slot); } + let allowed_host_function_slots = self + .allowed_host_registry_slots + .iter() + .filter_map(|registry_slot| { + registry_slots + .iter() + .position(|slot| slot == registry_slot) + .map(|slot| slot as u16) + }) + .collect(); + let runtime_owned_pending_slots = registry_slots + .iter() + .enumerate() + .filter_map(|(vm_slot, registry_slot)| { + self.entries + .get(*registry_slot as usize) + .filter(|entry| entry.runtime_owned_pending) + .map(|_| vm_slot as u16) + }) + .collect(); let import_key = imports.to_vec(); let computed = Arc::new(HostBindingPlan { import_signature: import_key.clone(), registry_slots, resolved_calls, + runtime_owned_pending_slots, + allowed_builtin_calls: self.allowed_builtin_calls.as_ref().clone(), + allow_default_builtin_capabilities: self.allow_default_builtin_capabilities, + allowed_host_function_slots, + allow_default_host_capabilities: self.allow_default_host_capabilities, + capability_profile: Arc::clone(&self.capability_profile), + capability_state: Arc::clone(&self.capability_state), + registry_state: Arc::clone(&self.registry_state), + registry_generation_token: Arc::clone(&self.registry_generation_token), + registry_generation: self.registry_generation.load(Ordering::Relaxed), }); let mut cache = self .plan_cache .write() .expect("host binding plan cache write lock should not be poisoned"); - Ok(cache.entry(import_key).or_insert_with(|| computed).clone()) + cache.insert(import_key, Arc::clone(&computed)); + Ok(computed) } pub fn bind_vm_with_plan(&self, vm: &mut Vm, plan: &HostBindingPlan) -> VmResult<()> { @@ -419,13 +574,37 @@ impl HostFunctionRegistry { "host binding plan does not match vm import signature".to_string(), )); } - if !vm.host_functions.is_empty() || !vm.host_function_symbols.is_empty() { + if !Arc::ptr_eq(&self.capability_profile, &plan.capability_profile) { + return Err(VmError::HostError( + "host binding plan belongs to a different capability profile".to_string(), + )); + } + if !Arc::ptr_eq(&self.capability_state, &plan.capability_state) { + return Err(VmError::HostError( + "host binding plan belongs to a different capability state".to_string(), + )); + } + if !Arc::ptr_eq(&self.registry_state, &plan.registry_state) { + return Err(VmError::HostError( + "host binding plan belongs to a different registry state".to_string(), + )); + } + if !Arc::ptr_eq( + &self.registry_generation_token, + &plan.registry_generation_token, + ) || self.registry_generation.load(Ordering::Relaxed) != plan.registry_generation + { + return Err(VmError::HostError( + "host binding plan is stale for this registry".to_string(), + )); + } + if !vm.host.host_functions.is_empty() || !vm.host.host_function_symbols.is_empty() { return Err(VmError::HostError( "host binding cache requires an unbound vm".to_string(), )); } - vm.host_functions.reserve(plan.registry_slots.len()); + vm.host.host_functions.reserve(plan.registry_slots.len()); for ®istry_slot in &plan.registry_slots { let entry = self .entries @@ -455,6 +634,13 @@ impl HostFunctionRegistry { } } } + vm.set_default_host_fallback_enabled(false); + vm.host.allowed_builtin_calls = plan.allowed_builtin_calls.clone(); + vm.host.allow_default_builtin_capabilities = plan.allow_default_builtin_capabilities; + vm.host.allowed_host_function_slots = plan.allowed_host_function_slots.clone(); + vm.host.allow_default_host_capabilities = plan.allow_default_host_capabilities; + vm.host.runtime_owned_pending_host_slots = + plan.runtime_owned_pending_slots.iter().copied().collect(); vm.install_resolved_calls(plan.resolved_calls.clone())?; Ok(()) } @@ -533,13 +719,6 @@ pub(crate) fn validate_non_yielding_host_value( #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) struct WaitingHostOp { pub(super) op_id: HostOpId, - pub(super) source: WaitingHostOpSource, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum WaitingHostOpSource { - HostBridge, - BuiltinIo, } struct NoopWake; @@ -562,48 +741,56 @@ fn builtin_for_binding_name(name: &str) -> Option { impl Vm { pub fn register_function(&mut self, function: Box) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions.push(VmHostFunction::Dynamic(function)); - self.resolved_calls_dirty = true; + let index = self.host.host_functions.len() as u16; + self.host + .host_functions + .push(VmHostFunction::Dynamic(function)); + self.host.resolved_calls_dirty = true; index } pub fn register_static_function(&mut self, function: StaticHostFunction) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions.push(VmHostFunction::Static(function)); - self.resolved_calls_dirty = true; + let index = self.host.host_functions.len() as u16; + self.host + .host_functions + .push(VmHostFunction::Static(function)); + self.host.resolved_calls_dirty = true; index } pub fn register_stack_function(&mut self, function: Box) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions + let index = self.host.host_functions.len() as u16; + self.host + .host_functions .push(VmHostFunction::StackDynamic(function)); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; index } pub fn register_static_stack_function(&mut self, function: StaticHostStackFunction) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions + let index = self.host.host_functions.len() as u16; + self.host + .host_functions .push(VmHostFunction::StackStatic(function)); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; index } pub fn register_args_function(&mut self, function: Box) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions + let index = self.host.host_functions.len() as u16; + self.host + .host_functions .push(VmHostFunction::ArgsDynamic(function)); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; index } pub fn register_static_args_function(&mut self, function: StaticHostArgsFunction) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions + let index = self.host.host_functions.len() as u16; + self.host + .host_functions .push(VmHostFunction::ArgsStatic(function)); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; index } @@ -617,49 +804,79 @@ impl Vm { &mut self, function: StaticHostArgsFunction, ) -> u16 { - let index = self.host_functions.len() as u16; - self.host_functions + let index = self.host.host_functions.len() as u16; + self.host + .host_functions .push(VmHostFunction::ArgsStaticNonYielding(function)); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; index } + fn clear_runtime_owned_pending_binding(&mut self, name: &str) { + let slot = builtin_for_binding_name(name) + .and_then(|builtin| { + self.host + .builtin_overrides + .get(&builtin.call_index()) + .copied() + }) + .or_else(|| self.host.host_function_symbols.get(name).copied()); + if let Some(slot) = slot { + self.host.runtime_owned_pending_host_slots.remove(&slot); + } + } + + pub(crate) fn mark_runtime_owned_pending_binding(&mut self, name: &str) { + let slot = builtin_for_binding_name(name) + .and_then(|builtin| { + self.host + .builtin_overrides + .get(&builtin.call_index()) + .copied() + }) + .or_else(|| self.host.host_function_symbols.get(name).copied()) + .expect("generated runtime host binding should exist"); + self.host.runtime_owned_pending_host_slots.insert(slot); + } + pub fn bind_function(&mut self, name: impl Into, function: Box) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot(builtin.call_index(), VmHostFunction::Dynamic(function)); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::Dynamic(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_static_function(&mut self, name: impl Into, function: StaticHostFunction) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot(builtin.call_index(), VmHostFunction::Static(function)); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::Static(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_static_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_stack_function( @@ -668,17 +885,18 @@ impl Vm { function: Box, ) { let name = name.into(); - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + self.clear_runtime_owned_pending_binding(&name); + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::StackDynamic(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_stack_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_static_stack_function( @@ -687,6 +905,7 @@ impl Vm { function: StaticHostStackFunction, ) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot( builtin.call_index(), @@ -694,17 +913,17 @@ impl Vm { ); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::StackStatic(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_static_stack_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_args_function( @@ -713,6 +932,7 @@ impl Vm { function: Box, ) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot( builtin.call_index(), @@ -720,17 +940,17 @@ impl Vm { ); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::ArgsDynamic(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_args_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_static_args_function( @@ -739,6 +959,7 @@ impl Vm { function: StaticHostArgsFunction, ) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot( builtin.call_index(), @@ -746,17 +967,17 @@ impl Vm { ); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::ArgsStatic(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_static_args_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } /// Binds a static args-only host function that always returns one value synchronously. @@ -771,6 +992,7 @@ impl Vm { function: StaticHostArgsFunction, ) { let name = name.into(); + self.clear_runtime_owned_pending_binding(&name); if let Some(builtin) = builtin_for_binding_name(&name) { self.bind_builtin_overrideslot( builtin.call_index(), @@ -778,17 +1000,17 @@ impl Vm { ); return; } - if let Some(&index) = self.host_function_symbols.get(&name) - && let Some(slot) = self.host_functions.get_mut(index as usize) + if let Some(&index) = self.host.host_function_symbols.get(&name) + && let Some(slot) = self.host.host_functions.get_mut(index as usize) { *slot = VmHostFunction::ArgsStaticNonYielding(function); - self.resolved_calls_dirty = true; + self.host.resolved_calls_dirty = true; return; } let index = self.register_static_non_yielding_args_function(function); - self.host_function_symbols.insert(name, index); - self.resolved_calls_dirty = true; + self.host.host_function_symbols.insert(name, index); + self.host.resolved_calls_dirty = true; } pub fn bind_builtin_override( @@ -800,6 +1022,7 @@ impl Vm { let builtin = BuiltinFunction::from_namespaced_name(&name).ok_or_else(|| { VmError::HostError(format!("unknown namespaced builtin override '{name}'")) })?; + self.clear_runtime_owned_pending_binding(&name); self.bind_builtin_overrideslot(builtin.call_index(), VmHostFunction::Dynamic(function)); Ok(()) } @@ -813,46 +1036,171 @@ impl Vm { let builtin = BuiltinFunction::from_namespaced_name(&name).ok_or_else(|| { VmError::HostError(format!("unknown namespaced builtin override '{name}'")) })?; + self.clear_runtime_owned_pending_binding(&name); self.bind_builtin_overrideslot(builtin.call_index(), VmHostFunction::Static(function)); Ok(()) } fn bind_builtin_overrideslot(&mut self, builtin_call_index: u16, function: VmHostFunction) { - if let Some(&host_slot) = self.builtin_overrides.get(&builtin_call_index) - && let Some(slot) = self.host_functions.get_mut(host_slot as usize) + if let Some(&host_slot) = self.host.builtin_overrides.get(&builtin_call_index) + && let Some(slot) = self.host.host_functions.get_mut(host_slot as usize) { *slot = function; return; } - let host_slot = self.host_functions.len() as u16; - self.host_functions.push(function); - self.builtin_overrides.insert(builtin_call_index, host_slot); + let host_slot = self.host.host_functions.len() as u16; + self.host.host_functions.push(function); + self.host + .builtin_overrides + .insert(builtin_call_index, host_slot); } pub fn set_async_bridge(&mut self, bridge: Box) { self.cancel_waiting_host_op(); - self.async_bridge = Some(bridge); + self.host.async_bridge = Some(bridge); } pub fn clear_async_bridge(&mut self) { self.cancel_waiting_host_op(); - self.async_bridge = None; + self.host.async_bridge = None; } pub fn set_runtime_print_sink(&mut self, sink: F) where F: FnMut(String) + Send + 'static, { - self.runtime_print_sink = Some(Box::new(sink)); + self.host.runtime_print_sink = Some(Box::new(sink)); } pub fn clear_runtime_print_sink(&mut self) { - self.runtime_print_sink = None; + self.host.runtime_print_sink = None; + } + + pub fn set_runtime_input(&mut self, value: Value) -> VmResult<()> { + self.run_ctx + .runtime_context + .set_input(value) + .map_err(|error| VmError::HostError(error.to_string())) + } + + pub fn clear_runtime_input(&mut self) { + self.run_ctx.runtime_context.clear_input(); + } + + pub fn set_runtime_event_sink(&mut self, sink: S) -> VmResult<()> + where + S: crate::builtins::runtime::event::EventSink + 'static, + { + self.run_ctx + .runtime_context + .set_event_sink(sink) + .map_err(|error| VmError::HostError(error.to_string())) + } + + pub fn clear_runtime_event_sink(&mut self) { + self.run_ctx.runtime_context.clear_event_sink(); + } + + pub(crate) fn runtime_input_value(&self) -> VmResult { + crate::builtins::runtime::context::runtime_input(&self.run_ctx.runtime_context) + } + + pub(crate) fn emit_runtime_event(&mut self, value: Value) -> VmResult<()> { + crate::builtins::runtime::context::runtime_emit(&mut self.run_ctx.runtime_context, value) + } + + /// Configure a bounded event sink without exposing runtime implementation types. + pub fn set_runtime_value_event_sink(&mut self, mut sink: F) -> VmResult<()> + where + F: FnMut(Value) -> VmResult<()> + Send + 'static, + { + self.run_ctx + .runtime_context + .set_event_sink( + move |payload: crate::builtins::runtime::event::EventPayload| { + sink(payload.into_value()).map_err(|error| { + crate::builtins::runtime::error::RuntimeError::new( + crate::builtins::runtime::error::RuntimeErrorCode::EventSinkRejected, + "runtime::emit", + error.to_string(), + ) + }) + }, + ) + .map_err(|error| VmError::HostError(error.to_string())) + } + + #[cfg(feature = "sqlite")] + pub fn configure_sqlite(&mut self, policy: crate::vm::SqlitePolicy) { + crate::builtins::runtime::cancel_operations_by_owner( + self, + crate::builtins::runtime::cancellation::OperationOwner::Sqlite, + crate::builtins::runtime::cancellation::CancellationReason::ResourceClosed, + ); + crate::builtins::runtime::close_resources_by_type( + self, + crate::builtins::runtime::resource::ResourceTypeId::SQLITE_CONNECTION, + crate::builtins::runtime::cancellation::CancellationReason::ResourceClosed, + ); + self.host.sqlite_policy = policy; + } + + #[cfg(feature = "sqlite")] + pub fn clear_sqlite_configuration(&mut self) { + crate::builtins::runtime::cancel_operations_by_owner( + self, + crate::builtins::runtime::cancellation::OperationOwner::Sqlite, + crate::builtins::runtime::cancellation::CancellationReason::ResourceClosed, + ); + crate::builtins::runtime::close_resources_by_type( + self, + crate::builtins::runtime::resource::ResourceTypeId::SQLITE_CONNECTION, + crate::builtins::runtime::cancellation::CancellationReason::ResourceClosed, + ); + self.host.sqlite_policy = crate::vm::SqlitePolicy::default(); + } + + pub fn configure_http(&mut self, config: crate::builtins::runtime::HttpConfig) { + self.host.http_state.configure(config); + } + + pub fn set_http_max_in_flight(&mut self, max_in_flight: usize) { + self.host.http_state.max_in_flight = max_in_flight; + } + + pub fn http_max_in_flight(&self) -> usize { + self.host.http_state.max_in_flight + } + + pub fn clear_http_configuration(&mut self) { + crate::builtins::runtime::cancel_operations_by_owner( + self, + crate::builtins::runtime::cancellation::OperationOwner::Http, + crate::builtins::runtime::cancellation::CancellationReason::Requested, + ); + self.host.http_state.clear_configuration(); + } + + pub fn http_is_configured(&self) -> bool { + self.host.http_state.is_configured() + } + + /// Enables or disables implicit binding of built-in host functions. + /// + /// Disabling this makes the VM use only explicitly registered host functions. The default + /// remains enabled for backwards compatibility until a registry is bound. + pub fn set_default_host_fallback_enabled(&mut self, enabled: bool) { + self.host.allow_default_host_fallback = enabled; + self.host.resolved_calls_dirty = true; + } + + pub fn default_host_fallback_enabled(&self) -> bool { + self.host.allow_default_host_fallback } pub(crate) fn write_runtime_print(&mut self, rendered: String) -> VmResult<()> { - let Some(sink) = self.runtime_print_sink.as_mut() else { + let Some(sink) = self.host.runtime_print_sink.as_mut() else { return Err(VmError::HostError( "runtime print sink is not configured".to_string(), )); @@ -862,28 +1210,48 @@ impl Vm { } pub fn allocate_host_op_id(&mut self) -> HostOpId { - let op_id = self.next_host_op_id; - self.next_host_op_id = self.next_host_op_id.wrapping_add(1).max(1); - op_id + self.host + .runtime_operations + .allocate_id() + .expect("host operation id space should not be exhausted") + .raw() } pub fn waiting_host_op_id(&self) -> Option { - self.waiting_host_op.map(|op| op.op_id) + self.instance.waiting_host_op.map(|op| op.op_id) + } + + pub fn cancel_waiting_host_op(&mut self) { + self.cancel_waiting_host_op_with_reason( + crate::builtins::runtime::cancellation::CancellationReason::Requested, + ); } - pub(super) fn cancel_waiting_host_op(&mut self) { - let Some(waiting) = self.waiting_host_op.take() else { + pub(crate) fn cancel_waiting_host_op_with_reason( + &mut self, + reason: crate::builtins::runtime::cancellation::CancellationReason, + ) { + let Some(waiting) = self.instance.waiting_host_op.take() else { return; }; - match waiting.source { - WaitingHostOpSource::HostBridge => { - if let Some(bridge) = self.async_bridge.as_mut() { - bridge.cancel_op(waiting.op_id); - } - } - WaitingHostOpSource::BuiltinIo => { - crate::builtins::runtime::cancel_builtin_io_op(self, waiting.op_id); + let Ok(operation_id) = + crate::builtins::runtime::cancellation::OperationId::from_raw(waiting.op_id) + else { + return; + }; + let owner = self + .host + .runtime_operations + .get(operation_id) + .ok() + .map(|operation| operation.owner()); + if owner == Some(crate::builtins::runtime::cancellation::OperationOwner::HostBridge) { + if let Some(bridge) = self.host.async_bridge.as_mut() { + bridge.cancel_op_with_reason(waiting.op_id, reason); } + let _ = self.host.runtime_operations.cancel(operation_id, reason); + } else { + crate::builtins::runtime::cancel_builtin_io_op_with_reason(self, waiting.op_id, reason); } } @@ -892,41 +1260,93 @@ impl Vm { op_id: HostOpId, values: impl Into, ) -> VmResult<()> { + let waiting = self.instance.waiting_host_op.ok_or_else(|| { + VmError::HostError(format!( + "host op {op_id} completed but vm is not waiting on any op", + )) + })?; + if waiting.op_id != op_id { + return Err(VmError::HostError(format!( + "host op {op_id} completed while vm waits on {}", + waiting.op_id + ))); + } + let operation_id = crate::builtins::runtime::cancellation::OperationId::from_raw(op_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + let operation = self + .host + .runtime_operations + .get(operation_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + if operation.owner() != crate::builtins::runtime::cancellation::OperationOwner::HostBridge { + return Err(VmError::HostError(format!( + "host bridge cannot complete runtime-owned operation {op_id}", + ))); + } + self.host + .runtime_operations + .complete(operation_id) + .map_err(|error| VmError::HostError(error.to_string()))?; self.complete_waiting_host_op(op_id, values.into()) } pub fn poll_waiting_host_op(&mut self, cx: &mut Context<'_>) -> Poll> { - let Some(waiting) = self.waiting_host_op else { + let Some(waiting) = self.instance.waiting_host_op else { return Poll::Ready(Ok(())); }; - - let poll_result = match waiting.source { - WaitingHostOpSource::HostBridge => { - let bridge_ptr = match self.async_bridge.as_mut() { - Some(bridge) => bridge.as_mut() as *mut dyn HostAsyncBridge, - None => { - return Poll::Ready(Err(VmError::HostError(format!( - "vm waiting on host op {} without an async bridge", - waiting.op_id - )))); - } - }; - - unsafe { (&mut *bridge_ptr).poll_op(waiting.op_id, cx) } - } - WaitingHostOpSource::BuiltinIo => { - crate::builtins::runtime::poll_builtin_io_op(self, waiting.op_id, cx) - } + let operation_id = + match crate::builtins::runtime::cancellation::OperationId::from_raw(waiting.op_id) { + Ok(operation_id) => operation_id, + Err(error) => return Poll::Ready(Err(VmError::HostError(error.to_string()))), + }; + let operation = match self.host.runtime_operations.get(operation_id) { + Ok(operation) => operation, + Err(error) => return Poll::Ready(Err(VmError::HostError(error.to_string()))), + }; + let host_bridge_owned = + operation.owner() == crate::builtins::runtime::cancellation::OperationOwner::HostBridge; + + let poll_result = if host_bridge_owned { + let bridge_ptr = match self.host.async_bridge.as_mut() { + Some(bridge) => bridge.as_mut() as *mut dyn HostAsyncBridge, + None => { + return Poll::Ready(Err(VmError::HostError(format!( + "vm waiting on host op {} without an async bridge", + waiting.op_id + )))); + } + }; + unsafe { (&mut *bridge_ptr).poll_op(waiting.op_id, cx) } + } else { + crate::builtins::runtime::poll_builtin_io_op(self, waiting.op_id, cx) }; match poll_result { Poll::Pending => Poll::Pending, Poll::Ready(Ok(values)) => { + if host_bridge_owned { + self.host + .runtime_operations + .complete(operation_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + } self.complete_waiting_host_op(waiting.op_id, values)?; Poll::Ready(Ok(())) } Poll::Ready(Err(err)) => { - self.waiting_host_op = None; + if host_bridge_owned { + let runtime_error = crate::builtins::runtime::error::RuntimeError::new( + crate::builtins::runtime::error::RuntimeErrorCode::OperationFailed, + "runtime::host_bridge", + err.to_string(), + ) + .with_value(waiting.op_id); + let _ = self + .host + .runtime_operations + .fail(operation_id, runtime_error); + } + self.instance.waiting_host_op = None; Poll::Ready(Err(err)) } } @@ -958,6 +1378,39 @@ impl Vm { } } + pub fn wait_for_host_op_blocking_with_cancel(&mut self, mut should_cancel: F) -> VmResult<()> + where + F: FnMut() -> bool, + { + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + loop { + if should_cancel() { + let cancellation_result = self + .run_ctx + .cancel(crate::builtins::runtime::cancellation::CancellationReason::Requested); + self.cancel_waiting_host_op(); + cancellation_result?; + return Err(VmError::HostError("host operation cancelled".to_string())); + } + match self.poll_waiting_host_op(&mut cx) { + Poll::Ready(result) => return result, + Poll::Pending => { + #[cfg(not(target_arch = "wasm32"))] + { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + #[cfg(target_arch = "wasm32")] + { + return Err(VmError::HostError( + "blocking host-op wait is unsupported on wasm32 runtime".to_string(), + )); + } + } + } + } + } + pub(super) fn execute_host_call( &mut self, index: u16, @@ -966,6 +1419,12 @@ impl Vm { ) -> VmResult { let argc = argc_u8 as usize; if let Some(builtin) = BuiltinFunction::from_call_index(index) { + if builtin.requires_explicit_host_capability() + && !self.host.allow_default_builtin_capabilities + && !self.host.allowed_builtin_calls.contains(&index) + { + return Err(VmError::UnboundImport(builtin.name().to_string())); + } if !builtin.accepts_arity(argc_u8) { return Err(VmError::InvalidCallArity { import: builtin.name().to_string(), @@ -973,7 +1432,7 @@ impl Vm { got: argc_u8, }); } - if self.builtin_overrides.contains_key(&index) { + if self.host.builtin_overrides.contains_key(&index) { return self.execute_builtin_override_call(index, argc_u8, call_ip); } if let Some(outcome) = @@ -994,13 +1453,28 @@ impl Vm { .get(usize::from(index)) .map(|import| import.return_type); let resolved_index = self.resolve_call_target(index, argc_u8)?; - if let Some(function) = - self.host_functions - .get(resolved_index as usize) - .and_then(|function| match function { - VmHostFunction::ArgsStaticNonYielding(function) => Some(*function), - _ => None, - }) + if !self.host.allow_default_host_capabilities + && !self + .host + .allowed_host_function_slots + .contains(&resolved_index) + { + let import_name = self + .program + .imports + .get(usize::from(index)) + .map(|import| import.name.clone()) + .unwrap_or_else(|| format!("host slot {resolved_index}")); + return Err(VmError::UnboundImport(import_name)); + } + if let Some(function) = self + .host + .host_functions + .get(resolved_index as usize) + .and_then(|function| match function { + VmHostFunction::ArgsStaticNonYielding(function) => Some(*function), + _ => None, + }) { return self.execute_static_non_yielding_args_host_function( function, @@ -1029,6 +1503,7 @@ impl Vm { call_ip: usize, ) -> VmResult { let resolved_index = self + .host .builtin_overrides .get(&builtin_call_index) .copied() @@ -1054,32 +1529,36 @@ impl Vm { call_ip: usize, ) -> VmResult { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; // Builtin dispatch reads arguments from the current stack tail while mutating the VM. - // The builtin runtime must not mutate `self.stack` until this borrowed slice is consumed. + // The builtin runtime must not mutate `self.instance.stack` until this borrowed slice is consumed. let outcome = unsafe { - let args = std::slice::from_raw_parts_mut(self.stack.as_mut_ptr().add(arg_start), argc); + let args = std::slice::from_raw_parts_mut( + self.instance.stack.as_mut_ptr().add(arg_start), + argc, + ); crate::builtins::runtime::execute_builtin_call(self, builtin, args) }?; match outcome { crate::builtins::runtime::BuiltinCallOutcome::Return(values) => { - self.stack.truncate(arg_start); - values.push_onto_stack(&mut self.stack); + self.instance.stack.truncate(arg_start); + values.push_onto_stack(&mut self.instance.stack); Ok(HostCallExecOutcome::Returned) } crate::builtins::runtime::BuiltinCallOutcome::Halt => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); Ok(HostCallExecOutcome::Halted) } crate::builtins::runtime::BuiltinCallOutcome::Pending(op_id) => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::BuiltinIo)?; - self.ip = resume_ip; + self.set_waiting_registered_op(op_id)?; + self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } } @@ -1092,13 +1571,14 @@ impl Vm { call_ip: usize, ) -> VmResult> { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; let (lhs, rhs) = self.operand_value_types(call_ip); let result = { - let args = &self.stack[arg_start..]; + let args = &self.instance.stack[arg_start..]; match builtin { BuiltinFunction::Len => match (lhs, args) { ( @@ -1166,8 +1646,8 @@ impl Vm { let Some(value) = result else { return Ok(None); }; - self.stack.truncate(arg_start); - self.stack.push(value); + self.instance.stack.truncate(arg_start); + self.instance.stack.push(value); self.record_typed_builtin_fast_path(); Ok(Some(HostCallExecOutcome::Returned)) } @@ -1178,12 +1658,13 @@ impl Vm { argc: usize, ) -> VmResult> { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; let result = { - let args = &self.stack[arg_start..]; + let args = &self.instance.stack[arg_start..]; match (builtin, args) { (BuiltinFunction::Len, [value]) => Self::fast_path_len_result(value), (BuiltinFunction::Get, [container, key]) => { @@ -1198,8 +1679,8 @@ impl Vm { let Some(value) = result else { return Ok(None); }; - self.stack.truncate(arg_start); - self.stack.push(value); + self.instance.stack.truncate(arg_start); + self.instance.stack.push(value); self.record_projection_fast_path(); Ok(Some(HostCallExecOutcome::Returned)) } @@ -1460,14 +1941,16 @@ impl Vm { call_ip: usize, ) -> VmResult { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; - let mut saved_stack = std::mem::take(&mut self.stack); - self.call_depth += 1; + let mut saved_stack = std::mem::take(&mut self.instance.stack); + self.instance.call_depth += 1; let function_ptr = - self.host_functions + self.host + .host_functions .get_mut(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))? as *mut VmHostFunction; let outcome = unsafe { @@ -1482,15 +1965,15 @@ impl Vm { | VmHostFunction::ArgsStaticNonYielding(_) => unreachable!(), } }; - self.call_depth = self.call_depth.saturating_sub(1); + self.instance.call_depth = self.instance.call_depth.saturating_sub(1); - let mut host_stack = std::mem::take(&mut self.stack); + let mut host_stack = std::mem::take(&mut self.instance.stack); let outcome = match outcome { Ok(outcome) => outcome, Err(err) => { saved_stack.truncate(arg_start); saved_stack.append(&mut host_stack); - self.stack = saved_stack; + self.instance.stack = saved_stack; return Err(err); } }; @@ -1500,28 +1983,28 @@ impl Vm { saved_stack.truncate(arg_start); saved_stack.append(&mut host_stack); values.push_onto_stack(&mut saved_stack); - self.stack = saved_stack; + self.instance.stack = saved_stack; Ok(HostCallExecOutcome::Returned) } CallOutcome::Halt => { saved_stack.truncate(arg_start); saved_stack.append(&mut host_stack); - self.stack = saved_stack; + self.instance.stack = saved_stack; Ok(HostCallExecOutcome::Halted) } CallOutcome::Yield => { saved_stack.append(&mut host_stack); - self.stack = saved_stack; - self.ip = call_ip; + self.instance.stack = saved_stack; + self.instance.ip = call_ip; Ok(HostCallExecOutcome::Yielded) } CallOutcome::Pending(op_id) => { saved_stack.truncate(arg_start); saved_stack.append(&mut host_stack); - self.stack = saved_stack; + self.instance.stack = saved_stack; let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; - self.ip = resume_ip; + self.set_waiting_bound_host_op(resolved_index, op_id)?; + self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } } @@ -1529,6 +2012,7 @@ impl Vm { fn bound_host_function_uses_args_slice(&self, resolved_index: u16) -> VmResult { let function = self + .host .host_functions .get(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))?; @@ -1542,6 +2026,7 @@ impl Vm { fn bound_host_function_uses_stack_borrow(&self, resolved_index: u16) -> VmResult { let function = self + .host .host_functions .get(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))?; @@ -1559,17 +2044,18 @@ impl Vm { expected_return_type: Option, ) -> VmResult { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; - self.call_depth += 1; - let outcome = function(&self.stack[arg_start..]); - self.call_depth = self.call_depth.saturating_sub(1); + self.instance.call_depth += 1; + let outcome = function(&self.instance.stack[arg_start..]); + self.instance.call_depth = self.instance.call_depth.saturating_sub(1); let value = require_non_yielding_host_value(outcome?)?; let value = validate_non_yielding_host_value(value, expected_return_type)?; - self.stack.truncate(arg_start); - self.stack.push(value); + self.instance.stack.truncate(arg_start); + self.instance.stack.push(value); Ok(HostCallExecOutcome::Returned) } @@ -1581,14 +2067,16 @@ impl Vm { expected_return_type: Option, ) -> VmResult { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; - self.call_depth += 1; + self.instance.call_depth += 1; let outcome = { - let args = &self.stack[arg_start..]; + let args = &self.instance.stack[arg_start..]; let function = self + .host .host_functions .get_mut(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))?; @@ -1602,36 +2090,36 @@ impl Vm { | VmHostFunction::StackStatic(_) => unreachable!(), } }; - self.call_depth = self.call_depth.saturating_sub(1); + self.instance.call_depth = self.instance.call_depth.saturating_sub(1); let (outcome, non_yielding) = outcome; let outcome = outcome?; if non_yielding { let value = require_non_yielding_host_value(outcome)?; let value = validate_non_yielding_host_value(value, expected_return_type)?; - self.stack.truncate(arg_start); - self.stack.push(value); + self.instance.stack.truncate(arg_start); + self.instance.stack.push(value); return Ok(HostCallExecOutcome::Returned); } match outcome { CallOutcome::Return(values) => { - self.stack.truncate(arg_start); - values.push_onto_stack(&mut self.stack); + self.instance.stack.truncate(arg_start); + values.push_onto_stack(&mut self.instance.stack); Ok(HostCallExecOutcome::Returned) } CallOutcome::Halt => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); Ok(HostCallExecOutcome::Halted) } CallOutcome::Yield => { - self.ip = call_ip; + self.instance.ip = call_ip; Ok(HostCallExecOutcome::Yielded) } CallOutcome::Pending(op_id) => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; - self.ip = resume_ip; + self.set_waiting_bound_host_op(resolved_index, op_id)?; + self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } } @@ -1644,20 +2132,23 @@ impl Vm { call_ip: usize, ) -> VmResult { let arg_start = self + .instance .stack .len() .checked_sub(argc) .ok_or(VmError::StackUnderflow)?; - self.call_depth += 1; + self.instance.call_depth += 1; let function_ptr = - self.host_functions + self.host + .host_functions .get_mut(resolved_index as usize) .ok_or(VmError::InvalidCall(resolved_index))? as *mut VmHostFunction; // Stack-borrowed host functions opt into the same raw stack-tail borrowing model used - // by builtin dispatch. They must not re-enter the VM or otherwise mutate `self.stack` + // by builtin dispatch. They must not re-enter the VM or otherwise mutate `self.instance.stack` // while the borrowed slice is alive. let outcome = unsafe { - let args = std::slice::from_raw_parts(self.stack.as_ptr().add(arg_start), argc); + let args = + std::slice::from_raw_parts(self.instance.stack.as_ptr().add(arg_start), argc); match &mut *function_ptr { VmHostFunction::StackDynamic(function) => function.call(self, args), VmHostFunction::StackStatic(function) => function(self, args), @@ -1668,28 +2159,28 @@ impl Vm { | VmHostFunction::ArgsStaticNonYielding(_) => unreachable!(), } }; - self.call_depth = self.call_depth.saturating_sub(1); + self.instance.call_depth = self.instance.call_depth.saturating_sub(1); let outcome = outcome?; match outcome { CallOutcome::Return(values) => { - self.stack.truncate(arg_start); - values.push_onto_stack(&mut self.stack); + self.instance.stack.truncate(arg_start); + values.push_onto_stack(&mut self.instance.stack); Ok(HostCallExecOutcome::Returned) } CallOutcome::Halt => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); Ok(HostCallExecOutcome::Halted) } CallOutcome::Yield => { - self.ip = call_ip; + self.instance.ip = call_ip; Ok(HostCallExecOutcome::Yielded) } CallOutcome::Pending(op_id) => { - self.stack.truncate(arg_start); + self.instance.stack.truncate(arg_start); let resume_ip = self.call_resume_ip(call_ip)?; - self.set_waiting_host_op(op_id, WaitingHostOpSource::HostBridge)?; - self.ip = resume_ip; + self.set_waiting_bound_host_op(resolved_index, op_id)?; + self.instance.ip = resume_ip; Ok(HostCallExecOutcome::Pending(op_id)) } } @@ -1715,12 +2206,91 @@ impl Vm { Ok(resume_ip) } - pub(super) fn set_waiting_host_op( - &mut self, - op_id: HostOpId, - source: WaitingHostOpSource, - ) -> VmResult<()> { - if let Some(active) = self.waiting_host_op + fn set_waiting_registered_op(&mut self, op_id: HostOpId) -> VmResult<()> { + let operation_id = crate::builtins::runtime::cancellation::OperationId::from_raw(op_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + let operation = self + .host + .runtime_operations + .get(operation_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + if operation.owner() == crate::builtins::runtime::cancellation::OperationOwner::HostBridge { + return Err(VmError::HostError(format!( + "builtin pending operation {op_id} is owned by the host bridge", + ))); + } + self.set_waiting_operation(op_id) + } + + fn set_waiting_bound_host_op(&mut self, resolved_index: u16, op_id: HostOpId) -> VmResult<()> { + if self + .host + .runtime_owned_pending_host_slots + .contains(&resolved_index) + { + self.set_waiting_registered_op(op_id) + } else { + self.set_waiting_host_op(op_id) + } + } + + pub(super) fn set_waiting_host_op(&mut self, op_id: HostOpId) -> VmResult<()> { + let result = (|| { + let operation_id = crate::builtins::runtime::cancellation::OperationId::from_raw(op_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + self.host + .runtime_operations + .retire_external_id(operation_id) + .map_err(|error| VmError::HostError(error.to_string()))?; + match self.host.runtime_operations.get(operation_id) { + Ok(operation) + if operation.owner() + == crate::builtins::runtime::cancellation::OperationOwner::HostBridge => {} + Ok(_) => { + return Err(VmError::HostError(format!( + "host bridge operation id {op_id} collides with a runtime-owned operation", + ))); + } + Err(_) => { + self.host + .runtime_operations + .register_retired_external( + operation_id, + crate::builtins::runtime::cancellation::OperationOwner::HostBridge, + Some(&self.run_ctx.cancellation), + None, + None, + ) + .map_err(|error| VmError::HostError(error.to_string()))?; + } + } + self.set_waiting_operation(op_id) + })(); + + if result.is_err() { + let reason = crate::builtins::runtime::cancellation::CancellationReason::ResourceClosed; + if let Some(bridge) = self.host.async_bridge.as_mut() { + bridge.cancel_op_with_reason(op_id, reason); + } + if let Ok(operation_id) = + crate::builtins::runtime::cancellation::OperationId::from_raw(op_id) + && self + .host + .runtime_operations + .get(operation_id) + .is_ok_and(|operation| { + operation.owner() + == crate::builtins::runtime::cancellation::OperationOwner::HostBridge + }) + { + let _ = self.host.runtime_operations.cancel(operation_id, reason); + } + } + result + } + + fn set_waiting_operation(&mut self, op_id: HostOpId) -> VmResult<()> { + if let Some(active) = self.instance.waiting_host_op && active.op_id != op_id { return Err(VmError::HostError(format!( @@ -1728,7 +2298,7 @@ impl Vm { active.op_id, op_id ))); } - self.waiting_host_op = Some(WaitingHostOp { op_id, source }); + self.instance.waiting_host_op = Some(WaitingHostOp { op_id }); Ok(()) } @@ -1737,7 +2307,7 @@ impl Vm { op_id: HostOpId, values: CallReturn, ) -> VmResult<()> { - let waiting = self.waiting_host_op.ok_or_else(|| { + let waiting = self.instance.waiting_host_op.ok_or_else(|| { VmError::HostError(format!( "host op {} completed but vm is not waiting on any op", op_id @@ -1749,8 +2319,8 @@ impl Vm { op_id, waiting.op_id ))); } - self.waiting_host_op = None; - values.push_onto_stack(&mut self.stack); + self.instance.waiting_host_op = None; + values.push_onto_stack(&mut self.instance.stack); Ok(()) } @@ -1763,21 +2333,24 @@ impl Vm { ))); } for &index in &resolved_calls { - if index as usize >= self.host_functions.len() { + if index as usize >= self.host.host_functions.len() { return Err(VmError::InvalidCall(index)); } } - self.resolved_calls = resolved_calls; - self.resolved_calls_dirty = false; + self.host.resolved_calls = resolved_calls; + self.host.resolved_calls_dirty = false; Ok(()) } pub(super) fn ensure_call_bindings(&mut self) -> VmResult<()> { - if self.program.imports.is_empty() || !self.resolved_calls_dirty { + if self.program.imports.is_empty() || !self.host.resolved_calls_dirty { return Ok(()); } - if self.host_function_symbols.is_empty() && self.host_functions.is_empty() { + if self.host.allow_default_host_fallback + && self.host.host_function_symbols.is_empty() + && self.host.host_functions.is_empty() + { let import_names = self .program .imports @@ -1789,49 +2362,54 @@ impl Vm { } } - let use_legacy_order = self.host_function_symbols.is_empty(); + let use_legacy_order = self.host.host_function_symbols.is_empty(); let mut resolved = Vec::with_capacity(self.program.imports.len()); let imports = self.program.imports.clone(); for (index, import) in imports.iter().enumerate() { if use_legacy_order { - if index >= self.host_functions.len() { + if index >= self.host.host_functions.len() { return Err(VmError::InvalidCall(index as u16)); } resolved.push(index as u16); continue; } - let bound = if let Some(bound) = self.host_function_symbols.get(&import.name).copied() { - bound - } else if crate::builtins::runtime::bind_default_host_function(self, &import.name) { - self.host_function_symbols - .get(&import.name) - .copied() - .ok_or_else(|| VmError::UnboundImport(import.name.clone()))? - } else { - return Err(VmError::UnboundImport(import.name.clone())); - }; + let bound = + if let Some(bound) = self.host.host_function_symbols.get(&import.name).copied() { + bound + } else if self.host.allow_default_host_fallback + && crate::builtins::runtime::bind_default_host_function(self, &import.name) + { + self.host + .host_function_symbols + .get(&import.name) + .copied() + .ok_or_else(|| VmError::UnboundImport(import.name.clone()))? + } else { + return Err(VmError::UnboundImport(import.name.clone())); + }; resolved.push(bound); } - self.resolved_calls = resolved; - self.resolved_calls_dirty = false; + self.host.resolved_calls = resolved; + self.host.resolved_calls_dirty = false; Ok(()) } pub(super) fn sync_jit_non_yielding_host_imports(&mut self) { let imports = self + .host .resolved_calls .iter() .map(|&slot| { matches!( - self.host_functions.get(usize::from(slot)), + self.host.host_functions.get(usize::from(slot)), Some(VmHostFunction::ArgsStaticNonYielding(_)) ) }) .collect(); - if self.jit.set_non_yielding_host_imports(imports) { - self.native_traces.clear(); + if self.engine.jit.set_non_yielding_host_imports(imports) { + self.engine.native_traces.clear(); } } @@ -1854,7 +2432,8 @@ impl Vm { }); } - self.resolved_calls + self.host + .resolved_calls .get(index as usize) .copied() .ok_or(VmError::InvalidCall(index)) diff --git a/src/vm/host_runtime.rs b/src/vm/host_runtime.rs new file mode 100644 index 00000000..8d3712da --- /dev/null +++ b/src/vm/host_runtime.rs @@ -0,0 +1,103 @@ +//! Host runtime shell. +//! +//! [`HostRuntime`] owns the host-facing capability surface: bound host +//! functions and their symbol table, capability allow-lists, builtin +//! overrides, resolved call slots, the opaque resource arena, the pending +//! operation registry, and the IO/HTTP/SQLite subsystem state plus the async +//! bridge and print sink. Interpreter state and run budgets live outside this +//! struct (see [`Instance`](super::instance::Instance) and +//! [`RunContext`](super::run_context::RunContext)). +//! +//! The unified host-lifecycle plan migrates individual subsystems behind this +//! shell; for now it groups their ownership and their reset/drop behavior. + +use std::collections::{HashMap, HashSet}; + +use crate::builtins::runtime::HttpState; +use crate::builtins::runtime::cancellation::{ + CancellationReason, DEFAULT_MAX_PENDING_OPERATIONS, OperationRegistry, +}; +use crate::builtins::runtime::resource::{DEFAULT_MAX_RESOURCES, ResourceArena}; + +#[cfg(feature = "sqlite")] +use crate::vm::SqlitePolicy; +use crate::vm::host::{HostAsyncBridge, VmHostFunction}; + +/// Embedder-supplied print sink for `print`/`debug` output. +pub(crate) type RuntimePrintSink = dyn FnMut(String) + Send; + +/// Host-owned capabilities, resources, operations, and subsystem state. +/// +/// Thread safety: `HostRuntime` is `!Sync` (host functions, resources, and +/// operations are mutable and not shareable) and not shared; one facade owns +/// one host runtime. Clone semantics: not `Clone` — host bindings and resource +/// handles must not be duplicated across VMs. +pub(crate) struct HostRuntime { + pub(super) host_functions: Vec, + pub(crate) host_function_symbols: HashMap, + pub(crate) allow_default_host_fallback: bool, + pub(crate) allowed_builtin_calls: Vec, + pub(crate) allow_default_builtin_capabilities: bool, + pub(crate) allowed_host_function_slots: Vec, + pub(crate) allow_default_host_capabilities: bool, + pub(crate) builtin_overrides: HashMap, + pub(crate) runtime_owned_pending_host_slots: HashSet, + pub(crate) resolved_calls: Vec, + pub(crate) resolved_calls_dirty: bool, + pub(crate) runtime_resources: ResourceArena, + pub(crate) runtime_operations: OperationRegistry, + #[cfg(feature = "sqlite")] + pub(crate) sqlite_policy: SqlitePolicy, + pub(crate) http_state: HttpState, + pub(crate) async_bridge: Option>, + pub(crate) runtime_print_sink: Option>, +} + +impl HostRuntime { + /// Creates an empty host runtime with default capability and resource + /// limits and no bound functions. + pub(crate) fn new() -> Self { + Self { + host_functions: Vec::new(), + host_function_symbols: HashMap::new(), + allow_default_host_fallback: true, + allowed_builtin_calls: Vec::new(), + allow_default_builtin_capabilities: true, + allowed_host_function_slots: Vec::new(), + allow_default_host_capabilities: true, + builtin_overrides: HashMap::new(), + runtime_owned_pending_host_slots: HashSet::new(), + resolved_calls: Vec::new(), + resolved_calls_dirty: true, + runtime_resources: ResourceArena::with_limit(DEFAULT_MAX_RESOURCES) + .expect("default runtime resource limit should be valid"), + runtime_operations: OperationRegistry::with_limit(DEFAULT_MAX_PENDING_OPERATIONS) + .expect("default runtime operation limit should be valid"), + #[cfg(feature = "sqlite")] + sqlite_policy: SqlitePolicy::default(), + http_state: HttpState::default(), + async_bridge: None, + runtime_print_sink: None, + } + } + + /// Closes run-scoped host state between runs: pending operations are + /// cancelled, resources are closed, and the IO subsystem is recreated. + /// Host bindings, capability allow-lists, and the async bridge are + /// preserved (documented reusable state). + pub(crate) fn reset_for_reuse(&mut self) { + let _ = self + .runtime_operations + .cancel_all(CancellationReason::VmReset); + let _ = self + .runtime_resources + .close_all(CancellationReason::VmReset); + self.http_state.reset_for_reuse(); + } +} + +impl Default for HostRuntime { + fn default() -> Self { + Self::new() + } +} diff --git a/src/vm/instance.rs b/src/vm/instance.rs new file mode 100644 index 00000000..baecdfb0 --- /dev/null +++ b/src/vm/instance.rs @@ -0,0 +1,270 @@ +//! Interpreter instance state. +//! +//! [`Instance`] owns everything that describes one execution position inside a +//! program: the instruction pointer, operand stack, locals, frames, capture +//! cells, callable ownership, queued callback traffic, waiting/yield state, +//! and instance-only counters. It has no program reference of its own; the +//! immutable [`Program`](crate::bytecode::Program) and the backend +//! [`Engine`](super::engine::Engine) live beside it, so one program can drive +//! many independent instances and a reset only touches this struct. +//! +//! Lifecycle: [`Instance::new`] starts a fresh halted instance; [`Instance::reset`] +//! rewinds run state while keeping configuration and host bindings (owned by +//! the facade); [`Instance::drop_cleanup`] releases interpreter-owned values +//! with drop-contract accounting. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, Weak}; + +use crate::bytecode::{CallableValue, Program, SharedCaptureCell, Value}; +use crate::vm::host::WaitingHostOp; +use crate::vm::map_iter::MapIteratorState; +use crate::vm::{DEFAULT_MAX_SCRIPT_CALL_DEPTH, VmYieldReason}; + +#[allow(dead_code)] +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum FrameContinuation { + Halt, + ResumeBytecode { return_ip: usize }, + ReturnToHost, +} + +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub(crate) struct ExecutionFrame { + pub(crate) continuation: FrameContinuation, + pub(crate) operand_stack_base: usize, + pub(crate) local_base: usize, + pub(crate) local_count: usize, + pub(crate) prototype_id: Option, +} + +impl ExecutionFrame { + pub(crate) fn root(local_count: usize) -> Self { + Self { + continuation: FrameContinuation::Halt, + operand_stack_base: 0, + local_base: 0, + local_count, + prototype_id: None, + } + } +} + +#[derive(Clone, Debug)] +pub(crate) struct QueuedCallable { + pub(crate) callable: Value, + pub(crate) args: Vec, + pub(crate) subscription: Option>, +} + +/// Interpreter-owned execution state. +/// +/// Thread safety: `Instance` is `!Sync` (it owns mutable interpreter state) +/// and is not shared; the VM facade owns exactly one instance. It is not +/// clonable: cloning would silently duplicate stack/frame/wait state. +pub(crate) struct Instance { + pub(crate) ip: usize, + pub(crate) stack: Vec, + pub(crate) locals: Vec, + pub(crate) capture_cells: HashMap, + pub(crate) shared_capture_slots: HashSet, + pub(crate) execution_frames: Vec, + pub(crate) active_local_base_cache: usize, + pub(crate) active_operand_stack_base_cache: usize, + pub(crate) call_depth: usize, + pub(crate) max_script_call_depth: usize, + pub(crate) host_return: Option, + pub(crate) queued_callables: VecDeque, + pub(crate) completed_callable_results: VecDeque, + pub(crate) owned_callables: Vec>, + pub(crate) callback_registry_flags: Vec>, + pub(crate) draining_queued_callables: bool, + pub(crate) shutdown: bool, + pub(super) waiting_host_op: Option, + pub(crate) last_yield_reason: Option, + pub(crate) map_iterators: Vec>>, + pub(crate) drop_contract_events_enabled: bool, + pub(crate) drop_contract_events: u64, + pub(crate) operand_hint_hit_count: u64, + pub(crate) operand_hint_miss_count: u64, + pub(crate) typed_builtin_fast_path_count: u64, + pub(crate) projection_fast_path_count: u64, + pub(crate) generic_builtin_call_count: u64, + pub(crate) scalar_superinstruction_count: u64, + pub(crate) local_type_hint_hit_count: u64, +} + +impl Instance { + /// Creates a halted instance positioned at program entry. + pub(crate) fn new(program: &Program) -> Self { + let local_count = program.local_count; + Self { + ip: 0, + stack: Vec::new(), + locals: vec![Value::Null; local_count], + capture_cells: HashMap::new(), + shared_capture_slots: HashSet::new(), + execution_frames: vec![ExecutionFrame::root(local_count)], + active_local_base_cache: 0, + active_operand_stack_base_cache: 0, + call_depth: 0, + max_script_call_depth: DEFAULT_MAX_SCRIPT_CALL_DEPTH, + host_return: None, + queued_callables: VecDeque::new(), + completed_callable_results: VecDeque::new(), + owned_callables: Vec::new(), + callback_registry_flags: Vec::new(), + draining_queued_callables: false, + shutdown: false, + waiting_host_op: None, + last_yield_reason: None, + map_iterators: Vec::new(), + drop_contract_events_enabled: false, + drop_contract_events: 0, + operand_hint_hit_count: 0, + operand_hint_miss_count: 0, + typed_builtin_fast_path_count: 0, + projection_fast_path_count: 0, + generic_builtin_call_count: 0, + scalar_superinstruction_count: 0, + local_type_hint_hit_count: 0, + } + } + + /// Rewinds run-scoped interpreter state for a fresh execution of the same + /// program. Host bindings, backend configuration, and compiled artifacts + /// (owned outside this struct) are preserved. + pub(crate) fn reset(&mut self, program: &Program) { + self.invalidate_callback_registries(); + self.ip = 0; + self.drop_contract_events = 0; + self.last_yield_reason = None; + self.clear_stack_with_drop_contract(); + self.capture_cells.clear(); + self.shared_capture_slots.clear(); + self.clear_locals_with_drop_contract(); + self.owned_callables.clear(); + self.locals.resize(program.local_count, Value::Null); + self.initialize_root_callable_bindings(program); + self.call_depth = 0; + self.execution_frames.clear(); + self.execution_frames + .push(ExecutionFrame::root(program.local_count)); + self.active_local_base_cache = 0; + self.active_operand_stack_base_cache = 0; + self.host_return = None; + self.queued_callables.clear(); + self.completed_callable_results.clear(); + self.owned_callables.clear(); + self.draining_queued_callables = false; + self.shutdown = false; + self.waiting_host_op = None; + self.map_iterators.clear(); + self.clear_interpreter_metrics(); + } + + /// Releases interpreter-owned values with drop-contract accounting. Used by + /// the facade's `Drop` (and by `shutdown`). + pub(crate) fn drop_cleanup(&mut self) { + self.clear_stack_with_drop_contract(); + self.capture_cells.clear(); + self.shared_capture_slots.clear(); + self.clear_locals_with_drop_contract(); + } + + pub(crate) fn invalidate_callback_registries(&mut self) { + for active in self + .callback_registry_flags + .drain(..) + .filter_map(|flag| flag.upgrade()) + { + active.store(false, std::sync::atomic::Ordering::Release); + } + } + + pub(crate) fn register_callback_registry(&mut self, active: &Arc) { + self.callback_registry_flags.push(Arc::downgrade(active)); + } + + pub(crate) fn initialize_root_callable_bindings(&mut self, program: &Program) { + let bindings = program.root_callable_bindings.clone(); + for binding in bindings { + let Some(kind) = program + .callable_prototypes + .get(binding.prototype_id as usize) + .map(|prototype| prototype.kind) + else { + continue; + }; + if binding.local_slot as usize >= self.locals.len() { + continue; + } + let callable = Arc::new(CallableValue { + prototype_id: binding.prototype_id, + kind, + env: None, + }); + self.owned_callables.push(Arc::downgrade(&callable)); + self.locals[binding.local_slot as usize] = Value::Callable(callable); + } + } + + pub(crate) fn clear_interpreter_metrics(&mut self) { + self.operand_hint_hit_count = 0; + self.operand_hint_miss_count = 0; + self.typed_builtin_fast_path_count = 0; + self.projection_fast_path_count = 0; + self.generic_builtin_call_count = 0; + self.scalar_superinstruction_count = 0; + self.local_type_hint_hit_count = 0; + } + + pub(crate) fn clear_stack_with_drop_contract(&mut self) { + let drained = self.stack.drain(..).collect::>(); + for value in drained { + self.drop_value_with_contract(value); + } + } + + pub(crate) fn clear_locals_with_drop_contract(&mut self) { + for slot in 0..self.locals.len() { + let previous = std::mem::replace(&mut self.locals[slot], Value::Null); + self.drop_value_with_contract(previous); + } + } + + pub(crate) fn drop_value_with_contract(&mut self, value: Value) { + if self.drop_contract_events_enabled { + self.count_value_drop_contract(&value); + } + } + + pub(crate) fn count_value_drop_contract(&mut self, value: &Value) { + match value { + Value::Null => {} + Value::Array(values) => { + self.drop_contract_events = self.drop_contract_events.saturating_add(1); + for item in values.iter() { + self.count_value_drop_contract(item); + } + } + Value::Map(entries) => { + self.drop_contract_events = self.drop_contract_events.saturating_add(1); + for (key, value) in entries.iter() { + self.count_value_drop_contract(key); + self.count_value_drop_contract(value); + } + } + Value::Int(_) + | Value::Float(_) + | Value::Bool(_) + | Value::String(_) + | Value::Bytes(_) + | Value::Callable(_) => { + self.drop_contract_events = self.drop_contract_events.saturating_add(1); + } + } + } +} diff --git a/src/vm/jit/diagnostics.rs b/src/vm/jit/diagnostics.rs index 9fe0dafe..47eb71e0 100644 --- a/src/vm/jit/diagnostics.rs +++ b/src/vm/jit/diagnostics.rs @@ -3,11 +3,12 @@ use super::{JitMetrics, JitSnapshot, native}; impl Vm { pub(super) fn jit_diagnostics_snapshot(&self) -> JitSnapshot { - self.jit.snapshot(self.jit_diagnostics_metrics()) + self.engine.jit.snapshot(self.jit_diagnostics_metrics()) } pub(super) fn jit_diagnostics_dump(&self, include_machine_code: bool) -> String { let mut out = self + .engine .jit .dump_text(self.program.debug.as_ref(), self.jit_diagnostics_metrics()); out.push_str(&format!( @@ -16,35 +17,36 @@ impl Vm { )); out.push_str(&format!( " native trace executions: {}\n", - self.native_trace_exec_count + self.engine.native_trace_exec_count )); out.push_str(&format!( " native trace handoffs: {}\n", - self.jit_native_link_handoff_count + self.engine.jit_native_link_handoff_count )); out.push_str(&format!( " native region entries: {}\n", - self.jit_native_region_entry_count + self.engine.jit_native_region_entry_count )); out.push_str(&format!( " native internal region edges: {}\n", - self.jit_native_region_edge_count + self.engine.jit_native_region_edge_count )); out.push_str(&format!( " native direct side links: {}\n", - self.jit_native_direct_link_count + self.engine.jit_native_direct_link_count )); out.push_str(&format!( " native compile time: {} ns (regions={} ns)\n", - self.jit_native_compile_time_ns, self.jit_native_region_compile_time_ns + self.engine.jit_native_compile_time_ns, self.engine.jit_native_region_compile_time_ns )); out.push_str(&format!( " native code bytes: {} (regions={})\n", self.jit_native_code_bytes(), self.jit_native_region_code_bytes() )); - if self.jit_native_bridge_stats_enabled { + if self.engine.jit_native_bridge_stats_enabled { let mut bridge_entries: Vec<(&'static str, u64)> = self + .engine .jit_native_bridge_counts .iter() .map(|(name, count)| (*name, *count)) @@ -62,14 +64,14 @@ impl Vm { out.push_str(&format!(" bridge {}: {}\n", name, count)); } } - let native_trace_count = self.native_traces.iter().flatten().count(); + let native_trace_count = self.engine.native_traces.iter().flatten().count(); if native_trace_count == 0 { out.push_str(" native traces: 0\n"); return out; } out.push_str(&format!(" native traces: {}\n", native_trace_count)); - for (id, native) in self.native_traces.iter().enumerate() { + for (id, native) in self.engine.native_traces.iter().enumerate() { if let Some(native) = native { out.push_str(&format!( " native trace#{} entry=0x{:X} code_bytes={} lowering={}\n", @@ -109,10 +111,10 @@ impl Vm { JitMetrics { boxed_load_site_count: 0, boxed_store_site_count: 0, - trace_exit_count: self.jit_trace_exit_count, - native_loop_back_count: self.jit_native_loop_back_count, - helper_fallback_count: self.jit_helper_fallback_count, - native_trace_exec_count: self.native_trace_exec_count, + trace_exit_count: self.engine.jit_trace_exit_count, + native_loop_back_count: self.engine.jit_native_loop_back_count, + helper_fallback_count: self.engine.jit_helper_fallback_count, + native_trace_exec_count: self.engine.native_trace_exec_count, script_call_observations: 0, monomorphic_call_sites: 0, polymorphic_call_sites: 0, diff --git a/src/vm/jit/runtime.rs b/src/vm/jit/runtime.rs index 751ef231..e4d946a0 100644 --- a/src/vm/jit/runtime.rs +++ b/src/vm/jit/runtime.rs @@ -257,20 +257,27 @@ pub(crate) extern "C" fn pd_vm_native_resume_linked_trace(vm: *mut Vm) -> i32 { return native::STATUS_ERROR; }; - if vm_ref.jit_native_link_dispatch_depth > 0 { + if vm_ref.engine.jit_native_link_dispatch_depth > 0 { return native::STATUS_TRACE_EXIT; } - vm_ref.jit_native_link_dispatch_depth = vm_ref.jit_native_link_dispatch_depth.saturating_add(1); + vm_ref.engine.jit_native_link_dispatch_depth = vm_ref + .engine + .jit_native_link_dispatch_depth + .saturating_add(1); match vm_ref.continue_linked_native_trace_from_exit() { Ok(status) => { - vm_ref.jit_native_link_dispatch_depth = - vm_ref.jit_native_link_dispatch_depth.saturating_sub(1); + vm_ref.engine.jit_native_link_dispatch_depth = vm_ref + .engine + .jit_native_link_dispatch_depth + .saturating_sub(1); status } Err(err) => { - vm_ref.jit_native_link_dispatch_depth = - vm_ref.jit_native_link_dispatch_depth.saturating_sub(1); + vm_ref.engine.jit_native_link_dispatch_depth = vm_ref + .engine + .jit_native_link_dispatch_depth + .saturating_sub(1); native::store_bridge_error(err); native::STATUS_ERROR } @@ -283,9 +290,9 @@ impl Vm { return None; } let entry_callable_prototypes = self.active_local_callable_prototypes(); - self.jit.compiled_trace_for_entry_with_callables( + self.engine.jit.compiled_trace_for_entry_with_callables( self.active_frame_key(), - self.ip, + self.instance.ip, self.active_operand_stack_len(), entry_callable_prototypes.as_deref(), ) @@ -299,21 +306,21 @@ impl Vm { all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")) ))] fn continue_linked_native_trace_from_exit(&mut self) -> VmResult { - self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); + self.engine.jit_trace_exit_count = self.engine.jit_trace_exit_count.saturating_add(1); let mut current_trace_id = { - let ip = self.ip; + let ip = self.instance.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); let mut next_trace_id = self.compiled_trace_for_active_entry(); if next_trace_id.is_none() && !self.active_frame_has_shared_capture_cells() - && !self.jit.callable_frame_is_blocked(frame_key) + && !self.engine.jit.callable_frame_is_blocked(frame_key) { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; - next_trace_id = self.jit.observe_exit_entry_with_local_types( + next_trace_id = self.engine.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, @@ -352,15 +359,16 @@ impl Vm { loop { native::clear_bridge_error(); - let region_edges_before = self.jit_native_region_edge_count; - let direct_links_before = self.jit_native_direct_link_count; + let region_edges_before = self.engine.jit_native_region_edge_count; + let direct_links_before = self.engine.jit_native_direct_link_count; let status = unsafe { entry(self as *mut Vm) }; - self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1); + self.engine.native_trace_exec_count = + self.engine.native_trace_exec_count.saturating_add(1); if !is_region - && self.jit_native_active_direct_trace_id != usize::MAX - && self.jit_native_active_direct_trace_id != current_trace_id + && self.engine.jit_native_active_direct_trace_id != usize::MAX + && self.engine.jit_native_active_direct_trace_id != current_trace_id { - current_trace_id = self.jit_native_active_direct_trace_id; + current_trace_id = self.engine.jit_native_active_direct_trace_id; let state = self.native_trace_state(current_trace_id)?; entry = state.0; root_ip = state.1; @@ -371,13 +379,15 @@ impl Vm { } self.record_native_direct_escape(status, direct_links_before); if is_region { - self.jit_native_region_entry_count = - self.jit_native_region_entry_count.saturating_add(1); - if self.jit_native_region_edge_count > region_edges_before { - self.jit.record_native_region_progress(current_trace_id); + self.engine.jit_native_region_entry_count = + self.engine.jit_native_region_entry_count.saturating_add(1); + if self.engine.jit_native_region_edge_count > region_edges_before { + self.engine + .jit + .record_native_region_progress(current_trace_id); } } - self.jit.mark_trace_executed(current_trace_id); + self.engine.jit.mark_trace_executed(current_trace_id); let mut trace_exit_key = None; let mut instruction_failure_exit = false; let status = if let Some(exit_id) = native::decode_jit_trace_exit_status(status) { @@ -393,8 +403,9 @@ impl Vm { exit_id: SsaExitId::new(exit_id), } }; - instruction_failure_exit = self.jit.trace_exit_is_instruction_failure(key); - self.jit + instruction_failure_exit = self.engine.jit.trace_exit_is_instruction_failure(key); + self.engine + .jit .record_trace_exit(key) .map_err(|err| VmError::JitNative(err.message()))?; trace_exit_key = Some(key); @@ -445,37 +456,39 @@ impl Vm { return Ok(native::STATUS_LINKED_CONTINUE); } native::STATUS_TRACE_EXIT => { - self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); + self.engine.jit_trace_exit_count = + self.engine.jit_trace_exit_count.saturating_add(1); if instruction_failure_exit { return Ok(native::STATUS_LINKED_CONTINUE); } if !has_yielding_call && terminal == JitTraceTerminal::LoopBack - && self.ip == root_ip + && self.instance.ip == root_ip { - self.jit.record_native_loop_back(current_trace_id); - self.jit_native_loop_back_count = - self.jit_native_loop_back_count.saturating_add(1); + self.engine.jit.record_native_loop_back(current_trace_id); + self.engine.jit_native_loop_back_count = + self.engine.jit_native_loop_back_count.saturating_add(1); continue; } - if self.jit.record_native_side_exit(current_trace_id) - && !self.jit_native_direct_links_enabled + if self.engine.jit.record_native_side_exit(current_trace_id) + && !self.engine.jit_native_direct_links_enabled { self.block_jit_callable_frame(current_trace_id); return Ok(native::STATUS_LINKED_CONTINUE); } if !has_yielding_call && !self.active_frame_has_shared_capture_cells() { - let ip = self.ip; + let ip = self.instance.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); let mut next_trace_id = self.compiled_trace_for_active_entry(); - if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) + if next_trace_id.is_none() + && !self.engine.jit.callable_frame_is_blocked(frame_key) { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; - next_trace_id = self.jit.observe_exit_entry_with_local_types( + next_trace_id = self.engine.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, @@ -534,19 +547,19 @@ impl Vm { } fn active_native_interrupt_settings(&self) -> Option { - match self.interrupt_mode { + match self.run_ctx.interrupt_mode { super::super::InterruptMode::None => None, super::super::InterruptMode::Fuel => Some(native::NativeInterruptSettings::fuel( - self.fuel_check_interval, + self.run_ctx.fuel_check_interval, )), super::super::InterruptMode::Epoch => Some(native::NativeInterruptSettings::epoch( - self.fuel_check_interval, + self.run_ctx.fuel_check_interval, )), } } fn clear_native_direct_links(&self) { - for native in self.native_traces.iter().flatten() { + for native in self.engine.native_traces.iter().flatten() { for slot in native.direct_slots.values() { slot.clear(); } @@ -554,15 +567,16 @@ impl Vm { } fn record_native_direct_escape(&mut self, _status: i32, direct_links_before: u64) { - if !self.jit_native_direct_links_enabled - || self.jit_native_direct_link_count == direct_links_before + if !self.engine.jit_native_direct_links_enabled + || self.engine.jit_native_direct_link_count == direct_links_before { return; } - self.jit_native_direct_escape_streak = 0; - if self.jit_native_active_direct_trace_id != usize::MAX { - self.jit - .record_native_loop_back(self.jit_native_active_direct_trace_id); + self.engine.jit_native_direct_escape_streak = 0; + if self.engine.jit_native_active_direct_trace_id != usize::MAX { + self.engine + .jit + .record_native_loop_back(self.engine.jit_native_active_direct_trace_id); } } @@ -571,7 +585,9 @@ impl Vm { key: TraceExitKey, child_trace_id: usize, ) -> VmResult<()> { - if !self.jit_native_direct_links_enabled || self.jit_native_direct_region_fallback { + if !self.engine.jit_native_direct_links_enabled + || self.engine.jit_native_direct_region_fallback + { return Ok(()); } self.publish_native_direct_slot(key.parent_trace_id, key.exit_id.raw(), child_trace_id) @@ -590,15 +606,21 @@ impl Vm { slot_id: u32, child_trace_id: usize, ) -> VmResult<()> { - if self.jit.trace_has_entry_callable_guards(child_trace_id) { + if self + .engine + .jit + .trace_has_entry_callable_guards(child_trace_id) + { return Ok(()); } - if !self.jit_native_direct_cross_frame_enabled { + if !self.engine.jit_native_direct_cross_frame_enabled { let parent_frame_key = self + .engine .jit .trace_clone(parent_trace_id) .map(|trace| trace.frame_key); let child_frame_key = self + .engine .jit .trace_clone(child_trace_id) .map(|trace| trace.frame_key); @@ -608,6 +630,7 @@ impl Vm { } self.ensure_native_trace(child_trace_id, native::NativeCompileProfile::Jit)?; let child_entry = self + .engine .native_traces .get(child_trace_id) .and_then(Option::as_ref) @@ -616,6 +639,7 @@ impl Vm { })? .tail_entry as *const u8; let Some(slot) = self + .engine .native_traces .get(parent_trace_id) .and_then(Option::as_ref) @@ -652,28 +676,34 @@ impl Vm { all(target_arch = "aarch64", any(target_os = "linux", target_os = "macos")) ))] fn maybe_publish_native_region(&mut self, key: TraceExitKey, child_trace_id: usize) { - if self.jit_native_direct_links_enabled && !self.jit_native_direct_region_fallback { + if self.engine.jit_native_direct_links_enabled + && !self.engine.jit_native_direct_region_fallback + { return; } if self + .engine .jit .trace_has_entry_callable_guards(key.parent_trace_id) - || self.jit.trace_has_entry_callable_guards(child_trace_id) + || self + .engine + .jit + .trace_has_entry_callable_guards(child_trace_id) { return; } - let Some(candidate) = self.jit.region_candidate(key, child_trace_id) else { + let Some(candidate) = self.engine.jit.region_candidate(key, child_trace_id) else { return; }; - if candidate.generation != self.jit.region_generation() { + if candidate.generation != self.engine.jit.region_generation() { return; } - let Some(parent) = self.jit.trace_clone(key.parent_trace_id) else { - self.jit.record_region_compile_failure(&candidate); + let Some(parent) = self.engine.jit.trace_clone(key.parent_trace_id) else { + self.engine.jit.record_region_compile_failure(&candidate); return; }; - let Some(child) = self.jit.trace_clone(child_trace_id) else { - self.jit.record_region_compile_failure(&candidate); + let Some(child) = self.engine.jit.trace_clone(child_trace_id) else { + self.engine.jit.record_region_compile_failure(&candidate); return; }; let back_import = scalar_cycle_import(&candidate.import) @@ -684,7 +714,8 @@ impl Vm { .iter() .filter(|exit| exit.exit_ip == parent.root_ip) .find_map(|exit| { - self.jit + self.engine + .jit .side_trace_import(child.id, exit.id, parent.id) .ok() }) @@ -699,7 +730,7 @@ impl Vm { ) { Ok(fused) => fused, Err(_) => { - self.jit.record_region_compile_failure(&candidate); + self.engine.jit.record_region_compile_failure(&candidate); return; } }; @@ -713,13 +744,14 @@ impl Vm { compile_profile, drop_contract_events_enabled, ); - self.jit_native_region_compile_time_ns = self + self.engine.jit_native_region_compile_time_ns = self + .engine .jit_native_region_compile_time_ns .saturating_add(elapsed_ns(compile_started)); let compiled = match compile_result { Ok(compiled) => compiled, Err(_) => { - self.jit.record_region_compile_failure(&candidate); + self.engine.jit.record_region_compile_failure(&candidate); return; } }; @@ -741,37 +773,38 @@ impl Vm { exit_keys: Arc::new(fused.exit_keys), }; let Some(parent_native) = self + .engine .native_traces .get_mut(key.parent_trace_id) .and_then(Option::as_mut) else { - self.jit.record_region_compile_failure(&candidate); + self.engine.jit.record_region_compile_failure(&candidate); return; }; - if !self.jit.publish_region(&candidate) { + if !self.engine.jit.publish_region(&candidate) { return; } parent_native.region = Some(region); } fn clear_native_region_owners(&mut self) { - for native in self.native_traces.iter_mut().flatten() { + for native in self.engine.native_traces.iter_mut().flatten() { native.region = None; } } pub(crate) fn disconnect_native_regions(&mut self) { - self.jit.invalidate_regions(); + self.engine.jit.invalidate_regions(); self.clear_native_region_owners(); } fn block_jit_trace(&mut self, trace_id: usize) { - self.jit.block_trace(trace_id); + self.engine.jit.block_trace(trace_id); self.clear_native_region_owners(); } fn block_jit_callable_frame(&mut self, trace_id: usize) { - self.jit.block_callable_frame(trace_id); + self.engine.jit.block_callable_frame(trace_id); self.clear_native_region_owners(); } @@ -780,26 +813,26 @@ impl Vm { self.ensure_program_cache_key(); } self.clear_native_direct_links(); - self.native_traces.clear(); - self.native_trace_exec_count = 0; - self.jit_native_region_entry_count = 0; - self.jit_native_region_edge_count = 0; - self.jit_native_direct_link_count = 0; - self.jit_native_active_direct_trace_id = usize::MAX; - self.jit_native_direct_escape_streak = 0; - self.jit_native_direct_region_fallback = false; - self.jit_native_compile_time_ns = 0; - self.jit_native_region_compile_time_ns = 0; - self.jit_trace_exit_count = 0; - self.jit_native_loop_back_count = 0; - self.jit_native_link_handoff_count = 0; - self.jit_native_link_dispatch_depth = 0; - self.jit_helper_fallback_count = 0; - self.jit.set_config(config); + self.engine.native_traces.clear(); + self.engine.native_trace_exec_count = 0; + self.engine.jit_native_region_entry_count = 0; + self.engine.jit_native_region_edge_count = 0; + self.engine.jit_native_direct_link_count = 0; + self.engine.jit_native_active_direct_trace_id = usize::MAX; + self.engine.jit_native_direct_escape_streak = 0; + self.engine.jit_native_direct_region_fallback = false; + self.engine.jit_native_compile_time_ns = 0; + self.engine.jit_native_region_compile_time_ns = 0; + self.engine.jit_trace_exit_count = 0; + self.engine.jit_native_loop_back_count = 0; + self.engine.jit_native_link_handoff_count = 0; + self.engine.jit_native_link_dispatch_depth = 0; + self.engine.jit_helper_fallback_count = 0; + self.engine.jit.set_config(config); } pub fn jit_config(&self) -> &super::JitConfig { - self.jit.config() + self.engine.jit.config() } pub fn jit_snapshot(&self) -> super::JitSnapshot { @@ -807,15 +840,16 @@ impl Vm { } pub fn jit_exit_profiles(&self) -> Vec { - self.jit.exit_profiles() + self.engine.jit.exit_profiles() } pub fn jit_call_site_profiles(&self) -> Vec { - self.jit.call_site_profiles() + self.engine.jit.call_site_profiles() } pub fn jit_native_code_bytes(&self) -> usize { - self.native_traces + self.engine + .native_traces .iter() .flatten() .map(|native| native.code.len()) @@ -823,7 +857,8 @@ impl Vm { } pub fn jit_native_region_code_bytes(&self) -> usize { - self.native_traces + self.engine + .native_traces .iter() .flatten() .filter_map(|native| native.region.as_ref()) @@ -832,11 +867,11 @@ impl Vm { } pub fn jit_native_compile_time_ns(&self) -> u64 { - self.jit_native_compile_time_ns + self.engine.jit_native_compile_time_ns } pub fn jit_native_region_compile_time_ns(&self) -> u64 { - self.jit_native_region_compile_time_ns + self.engine.jit_native_region_compile_time_ns } pub fn dump_jit_info(&self) -> String { @@ -909,15 +944,16 @@ impl Vm { ) = self.native_trace_state(current_trace_id)?; native::clear_bridge_error(); loop { - let region_edges_before = self.jit_native_region_edge_count; - let direct_links_before = self.jit_native_direct_link_count; + let region_edges_before = self.engine.jit_native_region_edge_count; + let direct_links_before = self.engine.jit_native_direct_link_count; let status = unsafe { entry(self as *mut Vm) }; - self.native_trace_exec_count = self.native_trace_exec_count.saturating_add(1); + self.engine.native_trace_exec_count = + self.engine.native_trace_exec_count.saturating_add(1); if !is_region - && self.jit_native_active_direct_trace_id != usize::MAX - && self.jit_native_active_direct_trace_id != current_trace_id + && self.engine.jit_native_active_direct_trace_id != usize::MAX + && self.engine.jit_native_active_direct_trace_id != current_trace_id { - current_trace_id = self.jit_native_active_direct_trace_id; + current_trace_id = self.engine.jit_native_active_direct_trace_id; let state = self.native_trace_state(current_trace_id)?; entry = state.0; root_ip = state.1; @@ -928,13 +964,15 @@ impl Vm { } self.record_native_direct_escape(status, direct_links_before); if is_region { - self.jit_native_region_entry_count = - self.jit_native_region_entry_count.saturating_add(1); - if self.jit_native_region_edge_count > region_edges_before { - self.jit.record_native_region_progress(current_trace_id); + self.engine.jit_native_region_entry_count = + self.engine.jit_native_region_entry_count.saturating_add(1); + if self.engine.jit_native_region_edge_count > region_edges_before { + self.engine + .jit + .record_native_region_progress(current_trace_id); } } - self.jit.mark_trace_executed(current_trace_id); + self.engine.jit.mark_trace_executed(current_trace_id); let mut trace_exit_key = None; let mut instruction_failure_exit = false; let status = if let Some(exit_id) = native::decode_jit_trace_exit_status(status) { @@ -950,8 +988,9 @@ impl Vm { exit_id: SsaExitId::new(exit_id), } }; - instruction_failure_exit = self.jit.trace_exit_is_instruction_failure(key); - self.jit + instruction_failure_exit = self.engine.jit.trace_exit_is_instruction_failure(key); + self.engine + .jit .record_trace_exit(key) .map_err(|err| VmError::JitNative(err.message()))?; trace_exit_key = Some(key); @@ -1017,13 +1056,19 @@ impl Vm { return Ok(ExecOutcome::Continue); } native::STATUS_TRACE_EXIT => { - self.jit_trace_exit_count = self.jit_trace_exit_count.saturating_add(1); + self.engine.jit_trace_exit_count = + self.engine.jit_trace_exit_count.saturating_add(1); if instruction_failure_exit { return Ok(ExecOutcome::Continue); } - if self.jit.trace_clone(current_trace_id).is_some_and(|trace| { - trace.op_names.last().map(String::as_str) == Some("callable_boundary") - }) { + if self + .engine + .jit + .trace_clone(current_trace_id) + .is_some_and(|trace| { + trace.op_names.last().map(String::as_str) == Some("callable_boundary") + }) + { self.block_jit_trace(current_trace_id); return Ok(ExecOutcome::Continue); } @@ -1031,25 +1076,26 @@ impl Vm { // calls, keep executing in native mode without bouncing through the interpreter. if !has_yielding_call && terminal == JitTraceTerminal::LoopBack - && self.ip == root_ip + && self.instance.ip == root_ip { - self.jit.record_native_loop_back(current_trace_id); - self.jit_native_loop_back_count = - self.jit_native_loop_back_count.saturating_add(1); + self.engine.jit.record_native_loop_back(current_trace_id); + self.engine.jit_native_loop_back_count = + self.engine.jit_native_loop_back_count.saturating_add(1); continue; } - if self.jit.record_native_side_exit(current_trace_id) - && !self.jit_native_direct_links_enabled + if self.engine.jit.record_native_side_exit(current_trace_id) + && !self.engine.jit_native_direct_links_enabled { self.block_jit_callable_frame(current_trace_id); return Ok(ExecOutcome::Continue); } if !has_yielding_call && !self.active_frame_has_shared_capture_cells() { - let ip = self.ip; + let ip = self.instance.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); let mut next_trace_id = self.compiled_trace_for_active_entry(); - if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) + if next_trace_id.is_none() + && !self.engine.jit.callable_frame_is_blocked(frame_key) { next_trace_id = { let entry_local_types = (frame_key != ROOT_FRAME_KEY) @@ -1057,7 +1103,7 @@ impl Vm { let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; - self.jit.observe_exit_entry_with_local_types( + self.engine.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, @@ -1121,17 +1167,19 @@ impl Vm { if self.active_frame_has_shared_capture_cells() { return Ok(ExecOutcome::Continue); } - let ip = self.ip; + let ip = self.instance.ip; let frame_key = self.active_frame_key(); let stack_depth = self.active_operand_stack_len(); let mut next_trace_id = self.compiled_trace_for_active_entry(); - if next_trace_id.is_none() && !self.jit.callable_frame_is_blocked(frame_key) { + if next_trace_id.is_none() + && !self.engine.jit.callable_frame_is_blocked(frame_key) + { next_trace_id = { let entry_local_types = (frame_key != ROOT_FRAME_KEY).then(|| self.active_local_types()); let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; - self.jit.observe_exit_entry_with_local_types( + self.engine.jit.observe_exit_entry_with_local_types( frame_key, ip, stack_depth, @@ -1186,26 +1234,31 @@ impl Vm { return Ok(ExecOutcome::Continue); } native::STATUS_YIELDED => { - self.last_yield_reason = Some(super::super::VmYieldReason::Host); + self.instance.last_yield_reason = Some(super::super::VmYieldReason::Host); return Ok(ExecOutcome::Yielded); } native::STATUS_WAITING => { - let op_id = self.waiting_host_op.map(|op| op.op_id).ok_or_else(|| { - VmError::JitNative( - "native call bridge reported waiting without a pending op".to_string(), - ) - })?; + let op_id = self + .instance + .waiting_host_op + .map(|op| op.op_id) + .ok_or_else(|| { + VmError::JitNative( + "native call bridge reported waiting without a pending op" + .to_string(), + ) + })?; return Ok(ExecOutcome::Waiting(op_id)); } native::STATUS_OUT_OF_FUEL => { - return match self.interrupt_mode { + return match self.run_ctx.interrupt_mode { super::super::InterruptMode::Fuel => Err(VmError::OutOfFuel { - needed: u64::from(self.fuel_check_interval), - remaining: self.fuel_remaining, + needed: u64::from(self.run_ctx.fuel_check_interval), + remaining: self.run_ctx.fuel_remaining, }), super::super::InterruptMode::Epoch => Err(VmError::EpochDeadlineReached { current: self.current_epoch(), - deadline: self.epoch_deadline, + deadline: self.run_ctx.epoch_deadline, }), super::super::InterruptMode::None => Err(VmError::JitNative( "native interruption checkpoint fired while interruption was disabled" @@ -1215,19 +1268,20 @@ impl Vm { } native::STATUS_ERROR => { let err = native::take_bridge_error().unwrap_or_else(|| { - let trace_meta = self.jit.trace_clone(current_trace_id).map(|trace| { - format!( - "trace_id={} root_ip={} terminal={:?} ops={}", - trace.id, - trace.root_ip, - trace.terminal, - trace.op_names.len() - ) - }); + let trace_meta = + self.engine.jit.trace_clone(current_trace_id).map(|trace| { + format!( + "trace_id={} root_ip={} terminal={:?} ops={}", + trace.id, + trace.root_ip, + trace.terminal, + trace.op_names.len() + ) + }); VmError::JitNative(format!( "jit bridge reported failure without VmError (ip={} stack_len={} {})", - self.ip, - self.stack.len(), + self.instance.ip, + self.instance.stack.len(), trace_meta.unwrap_or_else(|| "trace=".to_string()) )) }); @@ -1252,6 +1306,7 @@ impl Vm { ))] fn native_trace_state(&self, trace_id: usize) -> VmResult { let native = self + .engine .native_traces .get(trace_id) .and_then(Option::as_ref) @@ -1259,7 +1314,7 @@ impl Vm { VmError::JitNative(format!("native trace entry for id {} missing", trace_id)) })?; if let Some(region) = native.region.as_ref().filter(|region| { - self.jit.published_region().is_some_and(|published| { + self.engine.jit.published_region().is_some_and(|published| { published.generation == region.generation && published.key == region.key && published.child_trace_id == region.child_trace_id @@ -1300,10 +1355,10 @@ impl Vm { trace_id: usize, compile_profile: native::NativeCompileProfile, ) -> Option { - let native = self.native_traces.get(trace_id)?.as_ref()?; + let native = self.engine.native_traces.get(trace_id)?.as_ref()?; (native.interrupt_settings == self.active_native_interrupt_settings() && compile_profile_satisfies(native.compile_profile, compile_profile) - && native.drop_contract_events_enabled == self.drop_contract_events_enabled) + && native.drop_contract_events_enabled == self.instance.drop_contract_events_enabled) .then(|| self.native_trace_state(trace_id).ok()) .flatten() } @@ -1337,7 +1392,11 @@ impl Vm { compile_profile: native::NativeCompileProfile, interrupt_settings: Option, ) -> VmResult<()> { - if let Some(native) = self.native_traces.get(trace_id).and_then(Option::as_ref) + if let Some(native) = self + .engine + .native_traces + .get(trace_id) + .and_then(Option::as_ref) && native.interrupt_settings == interrupt_settings && compile_profile_satisfies(native.compile_profile, compile_profile) && native.drop_contract_events_enabled == self.drop_contract_events_enabled() @@ -1345,6 +1404,7 @@ impl Vm { return Ok(()); } if self + .engine .native_traces .get(trace_id) .and_then(Option::as_ref) @@ -1353,12 +1413,12 @@ impl Vm { self.disconnect_native_regions(); } self.clear_native_direct_links(); - if let Some(slot) = self.native_traces.get_mut(trace_id) { + if let Some(slot) = self.engine.native_traces.get_mut(trace_id) { *slot = None; } let program_cache_key = self.ensure_program_cache_key(); - let trace = self.jit.trace_clone(trace_id).ok_or_else(|| { + let trace = self.engine.jit.trace_clone(trace_id).ok_or_else(|| { VmError::JitNative(format!("trace {} missing for native compile", trace_id)) })?; let drop_contract_events_enabled = self.drop_contract_events_enabled(); @@ -1393,10 +1453,10 @@ impl Vm { .collect(); let mut code = cached.code.to_vec(); code.extend_from_slice(&dispatcher.code); - if self.native_traces.len() <= trace_id { - self.native_traces.resize_with(trace_id + 1, || None); + if self.engine.native_traces.len() <= trace_id { + self.engine.native_traces.resize_with(trace_id + 1, || None); } - self.native_traces[trace_id] = Some(NativeTrace { + self.engine.native_traces[trace_id] = Some(NativeTrace { _keepalive: cached.keepalive, _direct_keepalives: direct_keepalives, entry, @@ -1423,7 +1483,8 @@ impl Vm { compile_profile, drop_contract_events_enabled, ); - self.jit_native_compile_time_ns = self + self.engine.jit_native_compile_time_ns = self + .engine .jit_native_compile_time_ns .saturating_add(elapsed_ns(compile_started)); let compiled = compile_result?; @@ -1463,10 +1524,10 @@ impl Vm { let mut code = compiled.code; code.extend_from_slice(&dispatcher.code); let code = Arc::<[u8]>::from(code.into_boxed_slice()); - if self.native_traces.len() <= trace_id { - self.native_traces.resize_with(trace_id + 1, || None); + if self.engine.native_traces.len() <= trace_id { + self.engine.native_traces.resize_with(trace_id + 1, || None); } - self.native_traces[trace_id] = Some(NativeTrace { + self.engine.native_traces[trace_id] = Some(NativeTrace { _keepalive: keepalive, _direct_keepalives: direct_keepalives, entry, @@ -1487,21 +1548,24 @@ impl Vm { } pub fn jit_native_trace_count(&self) -> usize { - self.native_traces.iter().flatten().count() + self.engine.native_traces.iter().flatten().count() } pub fn jit_native_exec_count(&self) -> u64 { - self.native_trace_exec_count + self.engine.native_trace_exec_count } pub(crate) fn jit_native_inherited_target(&self) -> usize { - if !self.jit_native_direct_links_enabled || self.active_frame_has_shared_capture_cells() { + if !self.engine.jit_native_direct_links_enabled + || self.active_frame_has_shared_capture_cells() + { return 0; } let Some(trace_id) = self.compiled_trace_for_active_entry() else { return 0; }; - self.native_traces + self.engine + .native_traces .get(trace_id) .and_then(Option::as_ref) .map(|native| native.tail_entry as usize) @@ -1510,24 +1574,25 @@ impl Vm { pub fn set_jit_native_direct_links_enabled(&mut self, enabled: bool) { let cross_frame_enabled = enabled; - if self.jit_native_direct_links_enabled == enabled - && self.jit_native_direct_cross_frame_enabled == cross_frame_enabled + if self.engine.jit_native_direct_links_enabled == enabled + && self.engine.jit_native_direct_cross_frame_enabled == cross_frame_enabled { return; } self.clear_native_direct_links(); self.disconnect_native_regions(); - self.native_traces.clear(); - self.jit_native_direct_links_enabled = enabled; - self.jit_native_direct_cross_frame_enabled = cross_frame_enabled; - self.jit_native_direct_link_count = 0; - self.jit_native_active_direct_trace_id = usize::MAX; - self.jit_native_direct_escape_streak = 0; - self.jit_native_direct_region_fallback = false; + self.engine.native_traces.clear(); + self.engine.jit_native_direct_links_enabled = enabled; + self.engine.jit_native_direct_cross_frame_enabled = cross_frame_enabled; + self.engine.jit_native_direct_link_count = 0; + self.engine.jit_native_active_direct_trace_id = usize::MAX; + self.engine.jit_native_direct_escape_streak = 0; + self.engine.jit_native_direct_region_fallback = false; } pub fn jit_native_region_count(&self) -> usize { - self.native_traces + self.engine + .native_traces .iter() .flatten() .filter(|native| native.region.is_some()) @@ -1535,19 +1600,20 @@ impl Vm { } pub fn jit_native_region_entry_count(&self) -> u64 { - self.jit_native_region_entry_count + self.engine.jit_native_region_entry_count } pub fn jit_native_internal_region_edge_count(&self) -> u64 { - self.jit_native_region_edge_count + self.engine.jit_native_region_edge_count } pub fn jit_native_direct_link_count(&self) -> u64 { - self.jit_native_direct_link_count + self.engine.jit_native_direct_link_count } pub fn jit_native_active_direct_link_slot_count(&self) -> usize { - self.native_traces + self.engine + .native_traces .iter() .flatten() .flat_map(|native| native.direct_slots.values()) @@ -1556,19 +1622,21 @@ impl Vm { } pub fn jit_helper_fallback_count(&self) -> u64 { - self.jit_helper_fallback_count + self.engine.jit_helper_fallback_count } pub fn jit_native_link_handoff_count(&self) -> u64 { - self.jit_native_link_handoff_count + self.engine.jit_native_link_handoff_count } fn record_jit_helper_fallback(&mut self) { - self.jit_helper_fallback_count = self.jit_helper_fallback_count.saturating_add(1); + self.engine.jit_helper_fallback_count = + self.engine.jit_helper_fallback_count.saturating_add(1); } fn record_jit_link_handoff(&mut self) { - self.jit_native_link_handoff_count = self.jit_native_link_handoff_count.saturating_add(1); + self.engine.jit_native_link_handoff_count = + self.engine.jit_native_link_handoff_count.saturating_add(1); } } diff --git a/src/vm/mod.rs b/src/vm/mod.rs index ba148aec..0709c6cd 100644 --- a/src/vm/mod.rs +++ b/src/vm/mod.rs @@ -1,21 +1,27 @@ -use std::collections::{HashMap, HashSet, VecDeque}; +use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Mutex}; pub(crate) mod aot; pub mod diagnostics; +mod engine; mod epoch; mod fuel; mod host; +mod host_runtime; +mod instance; pub(crate) mod jit; mod map_iter; pub(crate) mod native; +pub mod program; +mod run_context; mod store; mod superinstructions; #[cfg(test)] mod tests; pub use self::aot::AotArtifactError; +use self::engine::Engine; pub use self::epoch::{EpochCheckpoint, EpochHandle}; pub use self::fuel::FuelCheckpoint; pub use self::host::{ @@ -23,7 +29,54 @@ pub use self::host::{ HostFunctionRegistry, HostOpId, HostStackFunction, StaticHostArgsFunction, StaticHostFunction, StaticHostStackFunction, }; -use self::host::{HostCallExecOutcome, VmHostFunction, WaitingHostOp}; +use self::host::{HostCallExecOutcome, VmHostFunction}; +use self::host_runtime::HostRuntime; +use self::instance::{ExecutionFrame, FrameContinuation, Instance, QueuedCallable}; +use self::run_context::{InterruptMode, RunContext}; +pub use crate::builtins::runtime::cancellation::CancellationReason; + +#[cfg(feature = "sqlite")] +#[derive(Clone, Copy, Debug)] +pub struct SqliteLimits { + pub max_connections: usize, + pub max_statements: usize, + pub max_rows: usize, + pub max_columns: usize, + pub max_result_bytes: usize, + pub max_statement_bytes: usize, + pub max_parameters: usize, + pub max_parameter_bytes: usize, + pub max_pending_operations: usize, + pub max_transaction_ms: u64, + pub busy_timeout_ms: u64, +} + +#[cfg(feature = "sqlite")] +impl Default for SqliteLimits { + fn default() -> Self { + Self { + max_connections: 16, + max_statements: 128, + max_rows: 1_000, + max_columns: 128, + max_result_bytes: 4 * 1024 * 1024, + max_statement_bytes: 1024 * 1024, + max_parameters: 128, + max_parameter_bytes: 1024 * 1024, + max_pending_operations: 32, + max_transaction_ms: 5_000, + busy_timeout_ms: 5_000, + } + } +} + +#[cfg(feature = "sqlite")] +#[derive(Clone, Debug, Default)] +pub struct SqlitePolicy { + pub database_root: Option, + pub allow_unsafe_sql: bool, + pub limits: SqliteLimits, +} pub use crate::bytecode::{ CallableTarget, CallableValue, HostImport, OpCode, Program, Value, ValueType, }; @@ -228,25 +281,6 @@ pub struct InterpreterMetrics { pub local_type_hint_hit_count: u64, } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -#[repr(u8)] -enum InterruptMode { - None = 0, - Fuel = 1, - Epoch = 2, -} - -impl InterruptMode { - fn label(self) -> &'static str { - match self { - Self::None => "none", - Self::Fuel => "fuel", - Self::Epoch => "epoch", - } - } -} -type RuntimePrintSink = dyn FnMut(String) + Send; - type PackedOperandTypes = u8; const NO_OPERAND_TYPE_HINT: PackedOperandTypes = 0; @@ -283,129 +317,12 @@ pub struct VmExecutionFrameSnapshot { pub prototype_id: Option, } -#[allow(dead_code)] -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum FrameContinuation { - Halt, - ResumeBytecode { return_ip: usize }, - ReturnToHost, -} - -#[allow(dead_code)] -#[derive(Clone, Debug)] -pub(crate) struct ExecutionFrame { - pub(crate) continuation: FrameContinuation, - pub(crate) operand_stack_base: usize, - pub(crate) local_base: usize, - pub(crate) local_count: usize, - pub(crate) prototype_id: Option, -} - -impl ExecutionFrame { - fn root(local_count: usize) -> Self { - Self { - continuation: FrameContinuation::Halt, - operand_stack_base: 0, - local_base: 0, - local_count, - prototype_id: None, - } - } -} - -#[derive(Clone, Debug)] -struct QueuedCallable { - callable: Value, - args: Vec, - subscription: Option>, -} - pub struct Vm { program: Arc, - #[allow(dead_code)] - program_constants_ptr: usize, - #[allow(dead_code)] - program_constants_len: usize, - #[allow(dead_code)] - native_helper_fn: usize, - #[allow(dead_code)] - native_interrupt_helper_fn: usize, - program_cache_key: u64, - program_cache_key_ready: bool, - ip: usize, - stack: Vec, - locals: Vec, - capture_cells: HashMap, - shared_capture_slots: HashSet, - operand_type_hints: Option>, - decoded_instruction_data: Arc, - host_functions: Vec, - host_function_symbols: HashMap, - builtin_overrides: HashMap, - resolved_calls: Vec, - resolved_calls_dirty: bool, - call_depth: usize, - max_script_call_depth: usize, - execution_frames: Vec, - active_local_base_cache: usize, - active_operand_stack_base_cache: usize, - host_return: Option, - queued_callables: VecDeque, - completed_callable_results: VecDeque, - owned_callables: Vec>, - callback_registry_flags: Vec>, - draining_queued_callables: bool, - shutdown: bool, - aot_program: Option, - aot_exec_count: u64, - aot_interpreter_boundary_hit: bool, - jit: jit::TraceJitEngine, - native_traces: Vec>, - native_trace_exec_count: u64, - jit_native_region_entry_count: u64, - jit_native_region_edge_count: u64, - jit_native_direct_link_count: u64, - jit_native_direct_links_enabled: bool, - jit_native_direct_cross_frame_enabled: bool, - jit_native_active_direct_trace_id: usize, - jit_native_direct_escape_streak: u16, - jit_native_direct_region_fallback: bool, - jit_native_compile_time_ns: u64, - jit_native_region_compile_time_ns: u64, - jit_trace_exit_count: u64, - jit_native_loop_back_count: u64, - jit_native_link_handoff_count: u64, - jit_native_link_dispatch_depth: u32, - jit_helper_fallback_count: u64, - jit_native_bridge_stats_enabled: bool, - jit_native_bridge_counts: HashMap<&'static str, u64>, - async_bridge: Option>, - runtime_print_sink: Option>, - waiting_host_op: Option, - next_host_op_id: HostOpId, - pub(crate) io_state: crate::builtins::runtime::IoState, - regex_cache: crate::builtins::runtime::regex::RegexCache, - map_iterators: Vec>>, - epoch_handle: EpochHandle, - #[allow(dead_code)] - epoch_counter_ptr: usize, - interrupt_mode: InterruptMode, - fuel_remaining: u64, - fuel_check_interval: u32, - fuel_ops_until_check: u32, - epoch_deadline: u64, - epoch_deadline_delta: u64, - epoch_rearm_pending: bool, - last_yield_reason: Option, - drop_contract_events_enabled: bool, - drop_contract_events: u64, - operand_hint_hit_count: u64, - operand_hint_miss_count: u64, - typed_builtin_fast_path_count: u64, - projection_fast_path_count: u64, - generic_builtin_call_count: u64, - scalar_superinstruction_count: u64, - local_type_hint_hit_count: u64, + pub(crate) engine: Engine, + pub(crate) instance: Instance, + pub(crate) run_ctx: RunContext, + pub(crate) host: HostRuntime, } pub(crate) enum ExecOutcome { @@ -660,126 +577,21 @@ impl Vm { } pub fn new_shared_with_jit_config(program: Arc, jit_config: jit::JitConfig) -> Self { - let program_constants_ptr = program.constants.as_ptr(); - let program_constants_len = program.constants.len(); - let local_count = program.local_count; - let operand_type_hints = program.shared_operand_type_hints(); - let decoded_instruction_data = program.shared_decoded_instruction_data(); - let epoch_handle = EpochHandle::default(); - let epoch_counter_ptr = epoch_handle.as_ptr() as usize; - let mut vm = Self { + let engine = Engine::new(jit_config, &program); + let mut instance = Instance::new(&program); + instance.initialize_root_callable_bindings(&program); + Self { program, - program_constants_ptr: program_constants_ptr as usize, - program_constants_len, - native_helper_fn: native::helper_entry_address(), - native_interrupt_helper_fn: native::interrupt_helper_entry_address(), - program_cache_key: 0, - program_cache_key_ready: false, - ip: 0, - stack: Vec::new(), - locals: vec![Value::Null; local_count], - capture_cells: HashMap::new(), - shared_capture_slots: HashSet::new(), - operand_type_hints, - decoded_instruction_data, - host_functions: Vec::new(), - host_function_symbols: HashMap::new(), - builtin_overrides: HashMap::new(), - resolved_calls: Vec::new(), - resolved_calls_dirty: true, - call_depth: 0, - max_script_call_depth: DEFAULT_MAX_SCRIPT_CALL_DEPTH, - execution_frames: vec![ExecutionFrame::root(local_count)], - active_local_base_cache: 0, - active_operand_stack_base_cache: 0, - host_return: None, - queued_callables: VecDeque::new(), - completed_callable_results: VecDeque::new(), - owned_callables: Vec::new(), - callback_registry_flags: Vec::new(), - draining_queued_callables: false, - shutdown: false, - aot_program: None, - aot_exec_count: 0, - aot_interpreter_boundary_hit: false, - jit: jit::TraceJitEngine::new(jit_config), - native_traces: Vec::new(), - native_trace_exec_count: 0, - jit_native_region_entry_count: 0, - jit_native_region_edge_count: 0, - jit_native_direct_link_count: 0, - jit_native_direct_links_enabled: true, - jit_native_direct_cross_frame_enabled: false, - jit_native_active_direct_trace_id: usize::MAX, - jit_native_direct_escape_streak: 0, - jit_native_direct_region_fallback: false, - jit_native_compile_time_ns: 0, - jit_native_region_compile_time_ns: 0, - jit_trace_exit_count: 0, - jit_native_loop_back_count: 0, - jit_native_link_handoff_count: 0, - jit_native_link_dispatch_depth: 0, - jit_helper_fallback_count: 0, - jit_native_bridge_stats_enabled: false, - jit_native_bridge_counts: HashMap::new(), - async_bridge: None, - runtime_print_sink: None, - waiting_host_op: None, - next_host_op_id: 1, - io_state: crate::builtins::runtime::IoState::default(), - regex_cache: crate::builtins::runtime::regex::RegexCache::default(), - map_iterators: Vec::new(), - epoch_handle, - epoch_counter_ptr, - interrupt_mode: InterruptMode::None, - fuel_remaining: 0, - fuel_check_interval: 1, - fuel_ops_until_check: 1, - epoch_deadline: 0, - epoch_deadline_delta: 0, - epoch_rearm_pending: false, - last_yield_reason: None, - drop_contract_events_enabled: false, - drop_contract_events: 0, - operand_hint_hit_count: 0, - operand_hint_miss_count: 0, - typed_builtin_fast_path_count: 0, - projection_fast_path_count: 0, - generic_builtin_call_count: 0, - scalar_superinstruction_count: 0, - local_type_hint_hit_count: 0, - }; - vm.initialize_root_callable_bindings(); - vm - } - - fn initialize_root_callable_bindings(&mut self) { - let bindings = self.program.root_callable_bindings.clone(); - for binding in bindings { - let Some(kind) = self - .program - .callable_prototypes - .get(binding.prototype_id as usize) - .map(|prototype| prototype.kind) - else { - continue; - }; - if binding.local_slot as usize >= self.locals.len() { - continue; - } - let callable = Arc::new(CallableValue { - prototype_id: binding.prototype_id, - kind, - env: None, - }); - self.owned_callables.push(Arc::downgrade(&callable)); - self.locals[binding.local_slot as usize] = Value::Callable(callable); + engine, + instance, + run_ctx: RunContext::default(), + host: HostRuntime::default(), } } /// Returns the maximum number of simultaneously active script call frames. pub fn max_script_call_depth(&self) -> usize { - self.max_script_call_depth + self.instance.max_script_call_depth } /// Sets the maximum number of simultaneously active script call frames. @@ -790,38 +602,34 @@ impl Vm { if limit == 0 { return Err(VmError::InvalidCallStackLimit(limit)); } - self.max_script_call_depth = limit; + self.instance.max_script_call_depth = limit; Ok(()) } fn ensure_program_cache_key(&mut self) -> u64 { - if !self.program_cache_key_ready { - self.program_cache_key = compute_program_cache_key(&self.program); - self.program_cache_key_ready = true; - } - self.program_cache_key + self.engine.ensure_program_cache_key(&self.program) } #[inline(always)] fn fuel_metering_enabled(&self) -> bool { - self.interrupt_mode == InterruptMode::Fuel + self.run_ctx.interrupt_mode == InterruptMode::Fuel } #[inline(always)] fn epoch_interruption_enabled(&self) -> bool { - self.interrupt_mode == InterruptMode::Epoch + self.run_ctx.interrupt_mode == InterruptMode::Epoch } #[inline(always)] fn interruption_enabled(&self) -> bool { - self.interrupt_mode != InterruptMode::None + self.run_ctx.interrupt_mode != InterruptMode::None } /// Returns the maximum number of compiled regular expressions retained by this VM. /// /// New VMs default to 512 entries. A capacity of zero disables caching. pub fn regex_cache_capacity(&self) -> usize { - self.regex_cache.capacity() + self.engine.regex_cache.capacity() } /// Changes this VM's compiled regular-expression cache capacity. @@ -829,67 +637,68 @@ impl Vm { /// Shrinking evicts least-recently-used entries immediately. Setting zero clears /// all entries and disables caching until a positive capacity is configured. pub fn set_regex_cache_capacity(&mut self, capacity: usize) { - self.regex_cache.set_capacity(capacity); + self.engine.regex_cache.set_capacity(capacity); } pub fn regex_cache_entry_count(&self) -> usize { - self.regex_cache.len() + self.engine.regex_cache.len() } pub fn regex_cache_compile_count(&self) -> u64 { - self.regex_cache.compile_count() + self.engine.regex_cache.compile_count() } pub fn regex_cache_hit_count(&self) -> u64 { - self.regex_cache.hit_count() + self.engine.regex_cache.hit_count() } pub(crate) fn cached_regex( &mut self, pattern: &str, ) -> Result, regex::Error> { - self.regex_cache.get_or_compile(pattern) + self.engine.regex_cache.get_or_compile(pattern) } pub fn set_jit_native_bridge_stats_enabled(&mut self, enabled: bool) { - self.jit_native_bridge_stats_enabled = enabled; + self.engine.jit_native_bridge_stats_enabled = enabled; if !enabled { - self.jit_native_bridge_counts.clear(); + self.engine.jit_native_bridge_counts.clear(); } } pub fn jit_native_bridge_stats_enabled(&self) -> bool { - self.jit_native_bridge_stats_enabled + self.engine.jit_native_bridge_stats_enabled } pub fn clear_jit_native_bridge_stats(&mut self) { - self.jit_native_bridge_counts.clear(); + self.engine.jit_native_bridge_counts.clear(); } pub fn interpreter_metrics_snapshot(&self) -> InterpreterMetrics { InterpreterMetrics { - operand_hint_hit_count: self.operand_hint_hit_count, - operand_hint_miss_count: self.operand_hint_miss_count, - typed_builtin_fast_path_count: self.typed_builtin_fast_path_count, - projection_fast_path_count: self.projection_fast_path_count, - generic_builtin_call_count: self.generic_builtin_call_count, - scalar_superinstruction_count: self.scalar_superinstruction_count, - local_type_hint_hit_count: self.local_type_hint_hit_count, + operand_hint_hit_count: self.instance.operand_hint_hit_count, + operand_hint_miss_count: self.instance.operand_hint_miss_count, + typed_builtin_fast_path_count: self.instance.typed_builtin_fast_path_count, + projection_fast_path_count: self.instance.projection_fast_path_count, + generic_builtin_call_count: self.instance.generic_builtin_call_count, + scalar_superinstruction_count: self.instance.scalar_superinstruction_count, + local_type_hint_hit_count: self.instance.local_type_hint_hit_count, } } pub fn clear_interpreter_metrics(&mut self) { - self.operand_hint_hit_count = 0; - self.operand_hint_miss_count = 0; - self.typed_builtin_fast_path_count = 0; - self.projection_fast_path_count = 0; - self.generic_builtin_call_count = 0; - self.scalar_superinstruction_count = 0; - self.local_type_hint_hit_count = 0; + self.instance.operand_hint_hit_count = 0; + self.instance.operand_hint_miss_count = 0; + self.instance.typed_builtin_fast_path_count = 0; + self.instance.projection_fast_path_count = 0; + self.instance.generic_builtin_call_count = 0; + self.instance.scalar_superinstruction_count = 0; + self.instance.local_type_hint_hit_count = 0; } pub fn jit_native_bridge_stats_snapshot(&self) -> Vec<(&'static str, u64)> { let mut entries: Vec<(&'static str, u64)> = self + .engine .jit_native_bridge_counts .iter() .map(|(name, count)| (*name, *count)) @@ -900,10 +709,11 @@ impl Vm { #[allow(dead_code)] pub(in crate::vm) fn record_native_bridge_hit(&mut self, bridge_name: &'static str) { - if !self.jit_native_bridge_stats_enabled { + if !self.engine.jit_native_bridge_stats_enabled { return; } let entry = self + .engine .jit_native_bridge_counts .entry(bridge_name) .or_insert(0); @@ -916,44 +726,13 @@ impl Vm { /// Locals are reset to `Null`, stack is cleared, and instruction pointer is /// rewound to the program entry. pub fn reset_for_reuse(&mut self) { - self.invalidate_callback_registries(); - self.cancel_waiting_host_op(); - self.ip = 0; - self.drop_contract_events = 0; - self.last_yield_reason = None; - self.epoch_rearm_pending = false; - self.clear_fuel(); - self.clear_epoch_deadline(); - self.clear_stack_with_drop_contract(); - self.capture_cells.clear(); - self.shared_capture_slots.clear(); - self.clear_locals_with_drop_contract(); - self.owned_callables.clear(); - self.locals.resize(self.program.local_count, Value::Null); - self.initialize_root_callable_bindings(); - crate::builtins::runtime::close_all_handles(self); - self.call_depth = 0; - self.execution_frames.clear(); - self.execution_frames - .push(ExecutionFrame::root(self.program.local_count)); - self.active_local_base_cache = 0; - self.active_operand_stack_base_cache = 0; - self.host_return = None; - self.queued_callables.clear(); - self.completed_callable_results.clear(); - self.owned_callables.clear(); - self.draining_queued_callables = false; - self.shutdown = false; - self.aot_interpreter_boundary_hit = self - .aot_program - .as_ref() - .is_some_and(|program| program.interpreter_boundary_only); - self.waiting_host_op = None; - self.io_state = crate::builtins::runtime::IoState::default(); - self.map_iterators.clear(); - self.jit.reset_runtime_backoff(); - self.jit.clear_call_site_profiles(); - self.clear_interpreter_metrics(); + self.cancel_waiting_host_op_with_reason( + crate::builtins::runtime::cancellation::CancellationReason::VmReset, + ); + self.host.reset_for_reuse(); + self.run_ctx.reset_for_reuse(); + self.instance.reset(&self.program); + self.engine.reset_runtime_state(&self.program); } fn validate_map_iterator_slot(&self, slot: usize) -> VmResult<()> { @@ -972,11 +751,11 @@ impl Vm { map: crate::bytecode::SharedMap, ) -> VmResult<()> { self.validate_map_iterator_slot(slot)?; - let depth = self.call_depth; - if self.map_iterators.len() <= depth { - self.map_iterators.resize_with(depth + 1, Vec::new); + let depth = self.instance.call_depth; + if self.instance.map_iterators.len() <= depth { + self.instance.map_iterators.resize_with(depth + 1, Vec::new); } - let frame = &mut self.map_iterators[depth]; + let frame = &mut self.instance.map_iterators[depth]; if frame.len() <= slot { frame.resize_with(slot + 1, || None); } @@ -986,9 +765,13 @@ impl Vm { pub(crate) fn advance_map_iterator(&mut self, slot: usize) -> VmResult { self.validate_map_iterator_slot(slot)?; - let frame = self.map_iterators.get_mut(self.call_depth).ok_or_else(|| { - VmError::HostError("map iterator frame is not initialized".to_string()) - })?; + let frame = self + .instance + .map_iterators + .get_mut(self.instance.call_depth) + .ok_or_else(|| { + VmError::HostError("map iterator frame is not initialized".to_string()) + })?; let state = frame .get_mut(slot) .and_then(Option::as_mut) @@ -1002,8 +785,9 @@ impl Vm { pub(crate) fn take_map_iterator_key(&mut self, slot: usize) -> VmResult { self.validate_map_iterator_slot(slot)?; - self.map_iterators - .get_mut(self.call_depth) + self.instance + .map_iterators + .get_mut(self.instance.call_depth) .and_then(|frame| frame.get_mut(slot)) .and_then(Option::as_mut) .and_then(map_iter::MapIteratorState::take_key) @@ -1012,8 +796,9 @@ impl Vm { pub(crate) fn take_map_iterator_value(&mut self, slot: usize) -> VmResult { self.validate_map_iterator_slot(slot)?; - self.map_iterators - .get_mut(self.call_depth) + self.instance + .map_iterators + .get_mut(self.instance.call_depth) .and_then(|frame| frame.get_mut(slot)) .and_then(Option::as_mut) .and_then(map_iter::MapIteratorState::take_value) @@ -1023,8 +808,9 @@ impl Vm { pub(crate) fn close_map_iterator(&mut self, slot: usize) -> VmResult<()> { self.validate_map_iterator_slot(slot)?; if let Some(state) = self + .instance .map_iterators - .get_mut(self.call_depth) + .get_mut(self.instance.call_depth) .and_then(|frame| frame.get_mut(slot)) { *state = None; @@ -1033,7 +819,7 @@ impl Vm { } fn close_all_map_iterators(&mut self) { - for frame in &mut self.map_iterators { + for frame in &mut self.instance.map_iterators { for state in frame { state.take(); } @@ -1042,19 +828,21 @@ impl Vm { #[inline(always)] pub(super) fn active_operand_stack_base(&self) -> usize { - self.active_operand_stack_base_cache + self.instance.active_operand_stack_base_cache } #[inline(always)] pub(super) fn active_operand_stack_len(&self) -> usize { - self.stack + self.instance + .stack .len() .saturating_sub(self.active_operand_stack_base()) } #[inline(always)] pub(super) fn active_frame_key(&self) -> u64 { - self.execution_frames + self.instance + .execution_frames .last() .and_then(|frame| frame.prototype_id) .map(u64::from) @@ -1063,11 +851,11 @@ impl Vm { #[inline(always)] pub(super) fn active_local_base(&self) -> usize { - self.active_local_base_cache + self.instance.active_local_base_cache } pub(super) fn active_local_types(&self) -> Vec { - self.locals[self.active_local_base()..] + self.instance.locals[self.active_local_base()..] .iter() .map(|value| match value { Value::Null => ValueType::Null, @@ -1085,9 +873,10 @@ impl Vm { pub(super) fn active_local_callable_prototypes(&self) -> Option>> { let base = self.active_local_base(); - let mut prototypes = Vec::with_capacity(self.locals.len().saturating_sub(base)); - for (offset, value) in self.locals[base..].iter().enumerate() { - let prototype_id = if let Some(cell) = self.capture_cells.get(&(base + offset)) { + let mut prototypes = Vec::with_capacity(self.instance.locals.len().saturating_sub(base)); + for (offset, value) in self.instance.locals[base..].iter().enumerate() { + let prototype_id = if let Some(cell) = self.instance.capture_cells.get(&(base + offset)) + { let value = cell.lock().ok()?; inline_compatible_callable_prototype(&value) } else { @@ -1099,21 +888,23 @@ impl Vm { } pub(super) fn active_frame_has_shared_capture_cells(&self) -> bool { - if self.shared_capture_slots.is_empty() { + if self.instance.shared_capture_slots.is_empty() { return false; } - let Some(frame) = self.execution_frames.last() else { + let Some(frame) = self.instance.execution_frames.last() else { return false; }; let base = frame.local_base; let end = base.saturating_add(frame.local_count); - self.shared_capture_slots + self.instance + .shared_capture_slots .iter() .any(|absolute| base <= *absolute && *absolute < end) } fn script_frame_depth(&self) -> usize { - self.execution_frames + self.instance + .execution_frames .iter() .filter(|frame| frame.prototype_id.is_some()) .count() @@ -1125,7 +916,8 @@ impl Vm { .active_local_base() .checked_add(index as usize) .ok_or(VmError::InvalidLocal(index))?; - self.locals + self.instance + .locals .get(absolute) .map(|_| absolute) .ok_or(VmError::InvalidLocal(index)) @@ -1134,8 +926,8 @@ impl Vm { #[inline(always)] fn load_local_value(&self, index: u8) -> VmResult { let absolute = self.absolute_local_index(index)?; - if self.capture_cells.is_empty() { - return Ok(self.locals[absolute].clone()); + if self.instance.capture_cells.is_empty() { + return Ok(self.instance.locals[absolute].clone()); } self.load_local_value_with_captures(absolute, index) } @@ -1143,13 +935,14 @@ impl Vm { #[cold] #[inline(never)] fn load_local_value_with_captures(&self, absolute: usize, index: u8) -> VmResult { - if let Some(cell) = self.capture_cells.get(&absolute) { + if let Some(cell) = self.instance.capture_cells.get(&absolute) { return cell .lock() .map(|value| value.clone()) .map_err(|_| VmError::InvalidFrameState("capture cell lock is poisoned")); } - self.locals + self.instance + .locals .get(absolute) .cloned() .ok_or(VmError::InvalidLocal(index)) @@ -1158,8 +951,8 @@ impl Vm { #[inline(always)] pub(super) fn local_numeric_value(&self, index: u8) -> Option { let absolute = self.absolute_local_index(index).ok()?; - if self.capture_cells.is_empty() { - return match self.locals.get(absolute)? { + if self.instance.capture_cells.is_empty() { + return match self.instance.locals.get(absolute)? { Value::Int(value) => Some(NumericValue::Int(*value)), Value::Float(value) => Some(NumericValue::Float(*value)), _ => None, @@ -1172,10 +965,14 @@ impl Vm { #[inline(never)] fn local_numeric_value_with_captures(&self, absolute: usize) -> Option { let captured = self + .instance .capture_cells .get(&absolute) .and_then(|cell| cell.lock().ok().map(|value| value.clone())); - match captured.as_ref().or_else(|| self.locals.get(absolute))? { + match captured + .as_ref() + .or_else(|| self.instance.locals.get(absolute))? + { Value::Int(value) => Some(NumericValue::Int(*value)), Value::Float(value) => Some(NumericValue::Float(*value)), _ => None, @@ -1183,33 +980,33 @@ impl Vm { } pub fn drop_contract_event_count(&self) -> u64 { - self.drop_contract_events + self.instance.drop_contract_events } pub fn set_drop_contract_events_enabled(&mut self, enabled: bool) { - if self.drop_contract_events_enabled != enabled { + if self.instance.drop_contract_events_enabled != enabled { self.disconnect_native_regions(); - self.native_traces.clear(); + self.engine.invalidate_codegen_caches(); } - self.drop_contract_events_enabled = enabled; + self.instance.drop_contract_events_enabled = enabled; if !enabled { - self.drop_contract_events = 0; + self.instance.drop_contract_events = 0; } } pub fn drop_contract_events_enabled(&self) -> bool { - self.drop_contract_events_enabled + self.instance.drop_contract_events_enabled } fn interruption_mode_conflict(&self, requested: InterruptMode) -> VmError { VmError::InterruptionModeConflict { - active: self.interrupt_mode.label(), + active: self.run_ctx.interrupt_mode.label(), requested: requested.label(), } } fn reset_interrupt_countdown(&mut self) { - self.fuel_ops_until_check = self.fuel_check_interval.max(1); + self.run_ctx.fuel_ops_until_check = self.run_ctx.fuel_check_interval.max(1); } pub fn run(&mut self) -> VmResult { @@ -1226,18 +1023,17 @@ impl Vm { impl Drop for Vm { fn drop(&mut self) { - self.cancel_waiting_host_op(); - self.clear_stack_with_drop_contract(); - self.capture_cells.clear(); - self.shared_capture_slots.clear(); - self.clear_locals_with_drop_contract(); - crate::builtins::runtime::close_all_handles(self); + self.cancel_waiting_host_op_with_reason( + crate::builtins::runtime::cancellation::CancellationReason::VmReset, + ); + self.host.reset_for_reuse(); + self.instance.drop_cleanup(); } } impl Vm { pub(super) fn pop_value(&mut self) -> VmResult { - self.stack.pop().ok_or(VmError::StackUnderflow) + self.instance.stack.pop().ok_or(VmError::StackUnderflow) } pub(crate) fn bind_callable_value( @@ -1276,18 +1072,19 @@ impl Vm { let absolute = active_base .checked_add(usize::from(*source)) .ok_or(VmError::InvalidFrameState("capture source slot overflow"))?; - if absolute >= self.locals.len() { + if absolute >= self.instance.locals.len() { return Err(VmError::InvalidFrameState( "capture source exceeds active frame locals", )); } let cell = self + .instance .capture_cells .entry(absolute) .or_insert_with(|| Arc::new(Mutex::new(value))) .clone(); - self.shared_capture_slots.insert(absolute); - self.locals[absolute] = cell + self.instance.shared_capture_slots.insert(absolute); + self.instance.locals[absolute] = cell .lock() .map_err(|_| VmError::InvalidFrameState("capture cell lock is poisoned"))? .clone(); @@ -1309,7 +1106,9 @@ impl Vm { kind: prototype.kind, env, }); - self.owned_callables.push(Arc::downgrade(&callable)); + self.instance + .owned_callables + .push(Arc::downgrade(&callable)); Ok(Value::Callable(callable)) } @@ -1319,11 +1118,11 @@ impl Vm { call_site_ip: Option, ) -> VmResult { let operand_count = argc as usize + 1; - if self.stack.len() < operand_count { + if self.instance.stack.len() < operand_count { return Err(VmError::StackUnderflow); } - let operand_stack_base = self.stack.len() - operand_count; - let mut operands = self.stack.split_off(operand_stack_base); + let operand_stack_base = self.instance.stack.len() - operand_count; + let mut operands = self.instance.stack.split_off(operand_stack_base); let callee = operands.remove(0); let Value::Callable(callable) = callee else { return Err(VmError::InvalidCallable); @@ -1354,15 +1153,15 @@ impl Vm { match prototype.target { CallableTarget::ScriptFunction(function_id) => { if let Some(call_ip) = call_site_ip { - self.jit.observe_script_call_target( + self.engine.jit.observe_script_call_target( self.active_frame_key(), call_ip, callable.prototype_id, ); } - if self.call_depth >= self.max_script_call_depth { + if self.instance.call_depth >= self.instance.max_script_call_depth { return Err(VmError::CallStackOverflow { - limit: self.max_script_call_depth, + limit: self.instance.max_script_call_depth, }); } let function = self @@ -1379,10 +1178,11 @@ impl Vm { }); } let inherited_callables = self + .instance .execution_frames .last() .map(|frame| { - self.locals[frame.local_base..frame.local_base + frame.local_count] + self.instance.locals[frame.local_base..frame.local_base + frame.local_count] .iter() .enumerate() .filter(|(_, value)| matches!(value, Value::Callable(_))) @@ -1390,9 +1190,10 @@ impl Vm { .collect::>() }) .unwrap_or_default(); - let local_base = self.locals.len(); + let local_base = self.instance.locals.len(); let local_count = prototype.frame_local_count; - self.locals + self.instance + .locals .resize(local_base.saturating_add(local_count), Value::Null); for binding in &self.program.root_callable_bindings { let relative = binding.local_slot as usize; @@ -1412,12 +1213,14 @@ impl Vm { kind, env: None, }); - self.owned_callables.push(Arc::downgrade(&callable)); - self.locals[local_base + relative] = Value::Callable(callable); + self.instance + .owned_callables + .push(Arc::downgrade(&callable)); + self.instance.locals[local_base + relative] = Value::Callable(callable); } for (slot, value) in inherited_callables { if slot < local_count { - self.locals[local_base + slot] = value; + self.instance.locals[local_base + slot] = value; } } for (slot, argument) in prototype.parameter_slots.iter().zip(operands) { @@ -1427,7 +1230,7 @@ impl Vm { "parameter slot is outside the script frame", )); } - self.locals[local_base + relative] = argument; + self.instance.locals[local_base + relative] = argument; } if let Some(environment) = &callable.env { let cells = environment @@ -1452,20 +1255,20 @@ impl Vm { )); } let absolute = local_base + relative; - self.locals[absolute] = cell + self.instance.locals[absolute] = cell .lock() .map_err(|_| { VmError::InvalidFrameState("capture cell lock is poisoned") })? .clone(); if prototype.self_slot != Some(*slot) { - self.capture_cells.insert(absolute, cell.clone()); + self.instance.capture_cells.insert(absolute, cell.clone()); if matches!( mode, crate::CaptureBindingMode::Borrow | crate::CaptureBindingMode::BorrowMut ) { - self.shared_capture_slots.insert(absolute); + self.instance.shared_capture_slots.insert(absolute); } } } @@ -1477,31 +1280,32 @@ impl Vm { "self slot is outside the script frame", )); } - self.locals[local_base + relative] = Value::Callable(callable.clone()); + self.instance.locals[local_base + relative] = Value::Callable(callable.clone()); } - let return_ip = self.ip; - self.execution_frames.push(ExecutionFrame { + let return_ip = self.instance.ip; + self.instance.execution_frames.push(ExecutionFrame { continuation: FrameContinuation::ResumeBytecode { return_ip }, operand_stack_base, local_base, local_count, prototype_id: Some(callable.prototype_id), }); - self.active_local_base_cache = local_base; - self.active_operand_stack_base_cache = operand_stack_base; - self.call_depth = self.script_frame_depth(); - self.ip = function.entry_ip as usize; + self.instance.active_local_base_cache = local_base; + self.instance.active_operand_stack_base_cache = operand_stack_base; + self.instance.call_depth = self.script_frame_depth(); + self.instance.ip = function.entry_ip as usize; self.charge_interrupt_tick()?; Ok(ExecOutcome::Continue) } CallableTarget::HostImport(import_index) => { - self.stack.extend(operands); - let call_ip = self.ip.saturating_sub(2); + self.instance.stack.extend(operands); + let call_ip = self.instance.ip.saturating_sub(2); match self.execute_host_call(import_index, argc, call_ip)? { HostCallExecOutcome::Returned => Ok(ExecOutcome::Continue), HostCallExecOutcome::Halted => Ok(ExecOutcome::Halted), HostCallExecOutcome::Yielded => { - self.stack + self.instance + .stack .insert(operand_stack_base, Value::Callable(callable)); Ok(ExecOutcome::Yielded) } @@ -1513,45 +1317,57 @@ impl Vm { fn complete_active_frame(&mut self) -> VmResult { let frame = self + .instance .execution_frames .pop() .ok_or(VmError::InvalidFrameState("missing active frame"))?; - self.active_local_base_cache = self + self.instance.active_local_base_cache = self + .instance .execution_frames .last() .map(|frame| frame.local_base) .unwrap_or(0); - self.active_operand_stack_base_cache = self + self.instance.active_operand_stack_base_cache = self + .instance .execution_frames .last() .map(|frame| frame.operand_stack_base) .unwrap_or(0); - if self.stack.len() < frame.operand_stack_base { + if self.instance.stack.len() < frame.operand_stack_base { return Err(VmError::InvalidFrameState( "operand stack is below the active frame base", )); } if matches!(frame.continuation, FrameContinuation::Halt) { - self.call_depth = self.script_frame_depth(); + self.instance.call_depth = self.script_frame_depth(); return Ok(ExecOutcome::Halted); } - let result = if self.stack.len() > frame.operand_stack_base { - self.stack.pop().expect("stack length checked above") + let result = if self.instance.stack.len() > frame.operand_stack_base { + self.instance + .stack + .pop() + .expect("stack length checked above") } else { Value::Null }; - while self.stack.len() > frame.operand_stack_base { - let value = self.stack.pop().expect("stack length checked above"); + while self.instance.stack.len() > frame.operand_stack_base { + let value = self + .instance + .stack + .pop() + .expect("stack length checked above"); self.drop_value_with_contract(value); } - self.call_depth = self.script_frame_depth(); + self.instance.call_depth = self.script_frame_depth(); if frame.prototype_id.is_some() { let frame_end = frame.local_base.saturating_add(frame.local_count); - self.capture_cells + self.instance + .capture_cells .retain(|absolute, _| *absolute < frame.local_base || *absolute >= frame_end); - self.shared_capture_slots + self.instance + .shared_capture_slots .retain(|absolute| *absolute < frame.local_base || *absolute >= frame_end); } @@ -1560,12 +1376,16 @@ impl Vm { .local_base .checked_add(frame.local_count) .ok_or(VmError::InvalidFrameState("local frame range overflow"))?; - if frame_end != self.locals.len() { + if frame_end != self.instance.locals.len() { return Err(VmError::InvalidFrameState( "active local frame does not end at the local stack tail", )); } - let drained = self.locals.drain(frame.local_base..).collect::>(); + let drained = self + .instance + .locals + .drain(frame.local_base..) + .collect::>(); for value in drained { self.drop_value_with_contract(value); } @@ -1585,16 +1405,16 @@ impl Vm { match frame.continuation { FrameContinuation::Halt => { - self.stack.push(result); + self.instance.stack.push(result); Ok(ExecOutcome::Halted) } FrameContinuation::ResumeBytecode { return_ip } => { - self.ip = return_ip; - self.stack.push(result); + self.instance.ip = return_ip; + self.instance.stack.push(result); Ok(ExecOutcome::Continue) } FrameContinuation::ReturnToHost => { - self.host_return = Some(result); + self.instance.host_return = Some(result); Ok(ExecOutcome::Halted) } } @@ -1602,25 +1422,25 @@ impl Vm { pub(super) fn can_fuse_call_ret_pattern(&self) -> bool { let code = &self.program.code; - self.ip < code.len() && code[self.ip] == OpCode::Ret as u8 + self.instance.ip < code.len() && code[self.instance.ip] == OpCode::Ret as u8 } pub(super) fn clear_stack_with_drop_contract(&mut self) { - let drained = self.stack.drain(..).collect::>(); + let drained = self.instance.stack.drain(..).collect::>(); for value in drained { self.drop_value_with_contract(value); } } pub(super) fn clear_locals_with_drop_contract(&mut self) { - for slot in 0..self.locals.len() { - let previous = std::mem::replace(&mut self.locals[slot], Value::Null); + for slot in 0..self.instance.locals.len() { + let previous = std::mem::replace(&mut self.instance.locals[slot], Value::Null); self.drop_value_with_contract(previous); } } pub(super) fn drop_value_with_contract(&mut self, value: Value) { - if self.drop_contract_events_enabled { + if self.instance.drop_contract_events_enabled { self.count_value_drop_contract(&value); } } @@ -1629,13 +1449,15 @@ impl Vm { match value { Value::Null => {} Value::Array(values) => { - self.drop_contract_events = self.drop_contract_events.saturating_add(1); + self.instance.drop_contract_events = + self.instance.drop_contract_events.saturating_add(1); for item in values.iter() { self.count_value_drop_contract(item); } } Value::Map(entries) => { - self.drop_contract_events = self.drop_contract_events.saturating_add(1); + self.instance.drop_contract_events = + self.instance.drop_contract_events.saturating_add(1); for (key, value) in entries.iter() { self.count_value_drop_contract(key); self.count_value_drop_contract(value); @@ -1647,14 +1469,15 @@ impl Vm { | Value::String(_) | Value::Bytes(_) | Value::Callable(_) => { - self.drop_contract_events = self.drop_contract_events.saturating_add(1); + self.instance.drop_contract_events = + self.instance.drop_contract_events.saturating_add(1); } } } #[inline(always)] pub(in crate::vm) fn charge_interrupt_tick(&mut self) -> VmResult<()> { - match self.interrupt_mode { + match self.run_ctx.interrupt_mode { InterruptMode::None => Ok(()), InterruptMode::Fuel => self.charge_fuel_tick(), InterruptMode::Epoch => self.charge_epoch_tick(), @@ -1664,15 +1487,15 @@ impl Vm { #[inline(always)] #[allow(dead_code)] pub(in crate::vm) fn charge_aot_call_boundary_interrupt(&mut self) -> VmResult<()> { - match self.interrupt_mode { + match self.run_ctx.interrupt_mode { InterruptMode::None => Ok(()), InterruptMode::Fuel => self.charge_fuel(1), InterruptMode::Epoch => { let current = self.current_epoch(); - if current >= self.epoch_deadline { + if current >= self.run_ctx.epoch_deadline { return Err(VmError::EpochDeadlineReached { current, - deadline: self.epoch_deadline, + deadline: self.run_ctx.epoch_deadline, }); } Ok(()) @@ -1681,7 +1504,7 @@ impl Vm { } pub(super) fn peek_value(&self) -> VmResult<&Value> { - self.stack.last().ok_or(VmError::StackUnderflow) + self.instance.stack.last().ok_or(VmError::StackUnderflow) } pub(super) fn pop_int(&mut self) -> VmResult { @@ -1705,7 +1528,8 @@ impl Vm { #[inline(always)] pub(super) fn operand_type_hint(&self, ip: usize) -> PackedOperandTypes { - self.operand_type_hints + self.engine + .operand_type_hints .as_deref() .map_or(NO_OPERAND_TYPE_HINT, |hints| hints[ip]) } @@ -1727,57 +1551,68 @@ impl Vm { #[inline(always)] pub(super) fn record_local_type_hint_hit(&mut self) { - self.local_type_hint_hit_count = self.local_type_hint_hit_count.saturating_add(1); + self.instance.local_type_hint_hit_count = + self.instance.local_type_hint_hit_count.saturating_add(1); } #[inline(always)] pub(super) fn record_scalar_superinstruction(&mut self) { - self.scalar_superinstruction_count = self.scalar_superinstruction_count.saturating_add(1); + self.instance.scalar_superinstruction_count = self + .instance + .scalar_superinstruction_count + .saturating_add(1); } #[inline(always)] pub(super) fn record_typed_builtin_fast_path(&mut self) { - self.typed_builtin_fast_path_count = self.typed_builtin_fast_path_count.saturating_add(1); + self.instance.typed_builtin_fast_path_count = self + .instance + .typed_builtin_fast_path_count + .saturating_add(1); } #[inline(always)] pub(super) fn record_projection_fast_path(&mut self) { - self.projection_fast_path_count = self.projection_fast_path_count.saturating_add(1); + self.instance.projection_fast_path_count = + self.instance.projection_fast_path_count.saturating_add(1); } #[inline(always)] pub(super) fn record_generic_builtin_call(&mut self) { - self.generic_builtin_call_count = self.generic_builtin_call_count.saturating_add(1); + self.instance.generic_builtin_call_count = + self.instance.generic_builtin_call_count.saturating_add(1); } #[inline(always)] fn record_operand_hint_hit(&mut self) { - self.operand_hint_hit_count = self.operand_hint_hit_count.saturating_add(1); + self.instance.operand_hint_hit_count = + self.instance.operand_hint_hit_count.saturating_add(1); } #[inline(always)] fn record_operand_hint_miss(&mut self) { - self.operand_hint_miss_count = self.operand_hint_miss_count.saturating_add(1); + self.instance.operand_hint_miss_count = + self.instance.operand_hint_miss_count.saturating_add(1); } #[inline(always)] pub(super) fn unary_not_op(&mut self) -> VmResult<()> { let value = self.pop_bool()?; - self.stack.push(Value::Bool(!value)); + self.instance.stack.push(Value::Bool(!value)); Ok(()) } pub(super) fn int_add_op(&mut self) -> VmResult<()> { let rhs = self.pop_int()?; let lhs = self.pop_int()?; - self.stack.push(Value::Int(lhs.wrapping_add(rhs))); + self.instance.stack.push(Value::Int(lhs.wrapping_add(rhs))); Ok(()) } pub(super) fn float_add_op(&mut self) -> VmResult<()> { let rhs = self.pop_float_exact()?; let lhs = self.pop_float_exact()?; - self.stack.push(Value::Float(lhs + rhs)); + self.instance.stack.push(Value::Float(lhs + rhs)); Ok(()) } @@ -1793,7 +1628,7 @@ impl Vm { let mut out = String::with_capacity(lhs.len() + rhs.len()); out.push_str(lhs.as_str()); out.push_str(rhs.as_str()); - self.stack.push(Value::string(out)); + self.instance.stack.push(Value::string(out)); Ok(()) } @@ -1808,7 +1643,7 @@ impl Vm { }; let mut out = crate::bytecode::unwrap_or_clone_shared(lhs); out.extend(crate::bytecode::unwrap_or_clone_shared(rhs)); - self.stack.push(Value::bytes(out)); + self.instance.stack.push(Value::bytes(out)); Ok(()) } @@ -1818,7 +1653,7 @@ impl Vm { ) -> VmResult<()> { let rhs = self.pop_int()?; let lhs = self.pop_int()?; - self.stack.push(Value::Int(op(lhs, rhs)?)); + self.instance.stack.push(Value::Int(op(lhs, rhs)?)); Ok(()) } @@ -1828,40 +1663,40 @@ impl Vm { ) -> VmResult<()> { let rhs = self.pop_float_exact()?; let lhs = self.pop_float_exact()?; - self.stack.push(Value::Float(op(lhs, rhs)?)); + self.instance.stack.push(Value::Float(op(lhs, rhs)?)); Ok(()) } pub(super) fn int_neg_op(&mut self) -> VmResult<()> { let value = self.pop_int()?; - self.stack.push(Value::Int(value.wrapping_neg())); + self.instance.stack.push(Value::Int(value.wrapping_neg())); Ok(()) } pub(super) fn float_neg_op(&mut self) -> VmResult<()> { let value = self.pop_float_exact()?; - self.stack.push(Value::Float(-value)); + self.instance.stack.push(Value::Float(-value)); Ok(()) } pub(super) fn int_eq_op(&mut self) -> VmResult<()> { let rhs = self.pop_int()?; let lhs = self.pop_int()?; - self.stack.push(Value::Bool(lhs == rhs)); + self.instance.stack.push(Value::Bool(lhs == rhs)); Ok(()) } pub(super) fn float_eq_op(&mut self) -> VmResult<()> { let rhs = self.pop_float_exact()?; let lhs = self.pop_float_exact()?; - self.stack.push(Value::Bool(lhs == rhs)); + self.instance.stack.push(Value::Bool(lhs == rhs)); Ok(()) } pub(super) fn bool_eq_op(&mut self) -> VmResult<()> { let rhs = self.pop_bool()?; let lhs = self.pop_bool()?; - self.stack.push(Value::Bool(lhs == rhs)); + self.instance.stack.push(Value::Bool(lhs == rhs)); Ok(()) } @@ -1874,7 +1709,7 @@ impl Vm { Value::String(value) => value, _ => return Err(VmError::TypeMismatch("string")), }; - self.stack.push(Value::Bool(lhs == rhs)); + self.instance.stack.push(Value::Bool(lhs == rhs)); Ok(()) } @@ -1883,7 +1718,7 @@ impl Vm { let lhs = self.pop_value()?; match (lhs, rhs) { (Value::Null, Value::Null) => { - self.stack.push(Value::Bool(true)); + self.instance.stack.push(Value::Bool(true)); Ok(()) } _ => Err(VmError::TypeMismatch("null")), @@ -1893,14 +1728,14 @@ impl Vm { pub(super) fn int_compare_op(&mut self, op: impl FnOnce(i64, i64) -> bool) -> VmResult<()> { let rhs = self.pop_int()?; let lhs = self.pop_int()?; - self.stack.push(Value::Bool(op(lhs, rhs))); + self.instance.stack.push(Value::Bool(op(lhs, rhs))); Ok(()) } pub(super) fn float_compare_op(&mut self, op: impl FnOnce(f64, f64) -> bool) -> VmResult<()> { let rhs = self.pop_float_exact()?; let lhs = self.pop_float_exact()?; - self.stack.push(Value::Bool(op(lhs, rhs))); + self.instance.stack.push(Value::Bool(op(lhs, rhs))); Ok(()) } @@ -1909,26 +1744,32 @@ impl Vm { let lhs = self.pop_value()?; match (lhs, rhs) { (Value::Int(lhs), Value::Int(rhs)) => { - self.stack.push(Value::Int(lhs.wrapping_add(rhs))) + self.instance.stack.push(Value::Int(lhs.wrapping_add(rhs))) + } + (Value::Int(lhs), Value::Float(rhs)) => { + self.instance.stack.push(Value::Float(lhs as f64 + rhs)) + } + (Value::Float(lhs), Value::Int(rhs)) => { + self.instance.stack.push(Value::Float(lhs + rhs as f64)) + } + (Value::Float(lhs), Value::Float(rhs)) => { + self.instance.stack.push(Value::Float(lhs + rhs)) } - (Value::Int(lhs), Value::Float(rhs)) => self.stack.push(Value::Float(lhs as f64 + rhs)), - (Value::Float(lhs), Value::Int(rhs)) => self.stack.push(Value::Float(lhs + rhs as f64)), - (Value::Float(lhs), Value::Float(rhs)) => self.stack.push(Value::Float(lhs + rhs)), (Value::String(lhs), Value::String(rhs)) => { let mut out = String::with_capacity(lhs.len() + rhs.len()); out.push_str(lhs.as_str()); out.push_str(rhs.as_str()); - self.stack.push(Value::string(out)); + self.instance.stack.push(Value::string(out)); } (Value::Bytes(lhs), Value::Bytes(rhs)) => { let mut out = crate::bytecode::unwrap_or_clone_shared(lhs); out.extend(crate::bytecode::unwrap_or_clone_shared(rhs)); - self.stack.push(Value::bytes(out)); + self.instance.stack.push(Value::bytes(out)); } (Value::Array(lhs), Value::Array(rhs)) => { let mut out = crate::bytecode::unwrap_or_clone_shared(lhs); out.extend(crate::bytecode::unwrap_or_clone_shared(rhs)); - self.stack.push(Value::array(out)); + self.instance.stack.push(Value::array(out)); } _ => { return Err(VmError::TypeMismatch( @@ -1948,7 +1789,7 @@ impl Vm { let lhs = self.pop_numeric()?; match (lhs, rhs) { (NumericValue::Int(lhs), NumericValue::Int(rhs)) => { - self.stack.push(Value::Int(int_op(lhs, rhs)?)); + self.instance.stack.push(Value::Int(int_op(lhs, rhs)?)); } (lhs, rhs) => { let lhs = match lhs { @@ -1959,7 +1800,7 @@ impl Vm { NumericValue::Int(v) => v as f64, NumericValue::Float(v) => v, }; - self.stack.push(Value::Float(float_op(lhs, rhs)?)); + self.instance.stack.push(Value::Float(float_op(lhs, rhs)?)); } } Ok(()) @@ -1986,7 +1827,7 @@ impl Vm { float_op(lhs, rhs) } }; - self.stack.push(Value::Bool(result)); + self.instance.stack.push(Value::Bool(result)); Ok(()) } @@ -2015,8 +1856,9 @@ impl Vm { index: u8, value: Value, ) -> VmResult<()> { - if self.capture_cells.is_empty() { + if self.instance.capture_cells.is_empty() { let slot = self + .instance .locals .get_mut(absolute) .ok_or(VmError::InvalidLocal(index))?; @@ -2035,7 +1877,7 @@ impl Vm { index: u8, value: Value, ) -> VmResult<()> { - if let Some(cell) = self.capture_cells.get(&absolute).cloned() { + if let Some(cell) = self.instance.capture_cells.get(&absolute).cloned() { if Self::value_references_capture_cell(&value, &cell, &mut HashSet::new())? { return Err(VmError::InvalidFrameState( "callable capture ownership cycle is unsupported", @@ -2047,11 +1889,12 @@ impl Vm { .map_err(|_| VmError::InvalidFrameState("capture cell lock is poisoned"))?; std::mem::replace(&mut *captured, value.clone()) }; - self.locals[absolute] = value; + self.instance.locals[absolute] = value; self.drop_value_with_contract(previous); return Ok(()); } let slot = self + .instance .locals .get_mut(absolute) .ok_or(VmError::InvalidLocal(index))?; @@ -2113,8 +1956,9 @@ impl Vm { pub(crate) fn detach_local_with_drop_contract(&mut self, index: u8) -> VmResult<()> { let absolute = self.absolute_local_index(index)?; - self.capture_cells.remove(&absolute); + self.instance.capture_cells.remove(&absolute); let slot = self + .instance .locals .get_mut(absolute) .ok_or(VmError::InvalidLocal(index))?; @@ -2124,11 +1968,11 @@ impl Vm { } pub(super) fn read_u8(&mut self) -> VmResult { - if self.ip >= self.program.code.len() { + if self.instance.ip >= self.program.code.len() { return Err(VmError::BytecodeBounds); } - let value = self.program.code[self.ip]; - self.ip += 1; + let value = self.program.code[self.instance.ip]; + self.instance.ip += 1; Ok(value) } @@ -2143,12 +1987,13 @@ impl Vm { } pub(super) fn read_bytes(&mut self, count: usize) -> VmResult<[u8; 4]> { - if self.ip + count > self.program.code.len() { + if self.instance.ip + count > self.program.code.len() { return Err(VmError::BytecodeBounds); } let mut buf = [0u8; 4]; - buf[..count].copy_from_slice(&self.program.code[self.ip..self.ip + count]); - self.ip += count; + buf[..count] + .copy_from_slice(&self.program.code[self.instance.ip..self.instance.ip + count]); + self.instance.ip += count; Ok(buf) } @@ -2158,6 +2003,7 @@ impl Vm { } if !self.program.function_regions.is_empty() { let active_prototype = self + .instance .execution_frames .last() .and_then(|frame| frame.prototype_id); @@ -2191,7 +2037,7 @@ impl Vm { return Err(VmError::InvalidBranchTarget { target }); } } - self.ip = target; + self.instance.ip = target; Ok(()) } } @@ -2248,10 +2094,10 @@ impl Vm { ) -> Option { match outcome { ExecOutcome::Continue => {} - ExecOutcome::Halted | ExecOutcome::Waiting(_) => self.last_yield_reason = None, + ExecOutcome::Halted | ExecOutcome::Waiting(_) => self.instance.last_yield_reason = None, ExecOutcome::Yielded => { - if self.last_yield_reason.is_none() { - self.last_yield_reason = Some(VmYieldReason::Host); + if self.instance.last_yield_reason.is_none() { + self.instance.last_yield_reason = Some(VmYieldReason::Host); } } } @@ -2274,7 +2120,7 @@ impl Vm { fn run_fast_interpreter(&mut self, allow_jit: bool) -> VmResult> { loop { - if self.ip >= self.program.code.len() { + if self.instance.ip >= self.program.code.len() { return Err(VmError::BytecodeBounds); } let opcode = self.read_u8()?; @@ -2282,17 +2128,17 @@ impl Vm { match outcome { ExecOutcome::Continue => {} ExecOutcome::Halted => { - self.last_yield_reason = None; + self.instance.last_yield_reason = None; return Ok(Some(VmStatus::Halted)); } ExecOutcome::Yielded => { - if self.last_yield_reason.is_none() { - self.last_yield_reason = Some(VmYieldReason::Host); + if self.instance.last_yield_reason.is_none() { + self.instance.last_yield_reason = Some(VmYieldReason::Host); } return Ok(Some(VmStatus::Yielded)); } ExecOutcome::Waiting(op_id) => { - self.last_yield_reason = None; + self.instance.last_yield_reason = None; return Ok(Some(VmStatus::Waiting(op_id))); } } @@ -2312,28 +2158,28 @@ impl Vm { ) -> VmResult { self.ensure_call_bindings()?; self.sync_jit_non_yielding_host_imports(); - if let Some(waiting) = self.waiting_host_op { - self.last_yield_reason = None; + if let Some(waiting) = self.instance.waiting_host_op { + self.instance.last_yield_reason = None; let status = VmStatus::Waiting(waiting.op_id); self.notify_debugger_status(&mut debugger, status); return Ok(status); } - self.last_yield_reason = None; - if self.epoch_rearm_pending { + self.instance.last_yield_reason = None; + if self.run_ctx.epoch_rearm_pending { self.rearm_epoch_after_yield_if_needed(); } if debugger.is_none() && !self.interruption_enabled() && (!allow_jit || (!self.jit_config().enabled - && (!self.has_aot_program() || self.aot_interpreter_boundary_hit))) + && (!self.has_aot_program() || self.engine.aot_interpreter_boundary_hit))) && let Some(status) = self.run_fast_interpreter(allow_jit)? { return Ok(status); } loop { - if self.epoch_rearm_pending { + if self.run_ctx.epoch_rearm_pending { self.rearm_epoch_after_yield_if_needed(); } if let Some(active_debugger) = debugger.as_deref_mut() { @@ -2342,7 +2188,7 @@ impl Vm { if allow_jit && self.has_aot_program() - && !self.aot_interpreter_boundary_hit + && !self.engine.aot_interpreter_boundary_hit && !self.drop_contract_events_enabled() { let outcome = match self.execute_aot_entry() { @@ -2369,7 +2215,7 @@ impl Vm { continue; } - if self.aot_interpreter_boundary_hit + if self.engine.aot_interpreter_boundary_hit && debugger.is_none() && !self.interruption_enabled() && !self.jit_config().enabled @@ -2380,12 +2226,14 @@ impl Vm { if allow_jit && self.jit_config().enabled - && self.builtin_overrides.is_empty() + && self.host.allow_default_builtin_capabilities + && self.host.allow_default_host_capabilities + && self.host.builtin_overrides.is_empty() && !self.drop_contract_events_enabled() && !self.active_frame_has_shared_capture_cells() { let frame_key = self.active_frame_key(); - let trace_id = if self.jit.callable_frame_is_blocked(frame_key) { + let trace_id = if self.engine.jit.callable_frame_is_blocked(frame_key) { None } else { let stack_depth = self.active_operand_stack_len(); @@ -2393,9 +2241,9 @@ impl Vm { .then(|| self.active_local_types()); let entry_callable_prototypes = self.active_local_callable_prototypes(); let program = &self.program; - self.jit.observe_hot_entry_with_local_types( + self.engine.jit.observe_hot_entry_with_local_types( frame_key, - self.ip, + self.instance.ip, stack_depth, entry_local_types.as_deref(), entry_callable_prototypes.as_deref(), @@ -2428,7 +2276,7 @@ impl Vm { } } - if self.ip >= self.program.code.len() { + if self.instance.ip >= self.program.code.len() { return Err(VmError::BytecodeBounds); } @@ -2487,9 +2335,9 @@ impl Vm { x if x == OpCode::Nop as u8 => {} x if x == OpCode::Ret as u8 => return self.complete_active_frame(), x if x == OpCode::Ldc as u8 => { - let opcode_ip = self.ip - 1; + let opcode_ip = self.instance.ip - 1; let value = if let Some(value) = self.decoded_ldc_value_at(opcode_ip).cloned() { - self.ip += 4; + self.instance.ip += 4; value } else { let index = self.read_u32()?; @@ -2499,10 +2347,10 @@ impl Vm { .cloned() .ok_or(VmError::InvalidConstant(index))? }; - self.stack.push(value); + self.instance.stack.push(value); } x if x == OpCode::Add as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2527,7 +2375,7 @@ impl Vm { } } x if x == OpCode::Sub as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2547,7 +2395,7 @@ impl Vm { } } x if x == OpCode::Mul as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2567,7 +2415,7 @@ impl Vm { } } x if x == OpCode::Div as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2586,20 +2434,22 @@ impl Vm { x if x == OpCode::Shl as u8 => { let rhs = self.pop_shift_amount()?; let lhs = self.pop_int()?; - self.stack.push(Value::Int(lhs.wrapping_shl(rhs))); + self.instance.stack.push(Value::Int(lhs.wrapping_shl(rhs))); } x if x == OpCode::Shr as u8 => { let rhs = self.pop_shift_amount()?; let lhs = self.pop_int()?; - self.stack.push(Value::Int(lhs.wrapping_shr(rhs))); + self.instance.stack.push(Value::Int(lhs.wrapping_shr(rhs))); } x if x == OpCode::Lshr as u8 => { let rhs = self.pop_shift_amount()?; let lhs = self.pop_int()?; - self.stack.push(Value::Int(logical_shr_i64(lhs, rhs))); + self.instance + .stack + .push(Value::Int(logical_shr_i64(lhs, rhs))); } x if x == OpCode::Mod as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2618,16 +2468,16 @@ impl Vm { x if x == OpCode::And as u8 => { let rhs = self.pop_bool()?; let lhs = self.pop_bool()?; - self.stack.push(Value::Bool(lhs && rhs)); + self.instance.stack.push(Value::Bool(lhs && rhs)); } x if x == OpCode::Or as u8 => { let rhs = self.pop_bool()?; let lhs = self.pop_bool()?; - self.stack.push(Value::Bool(lhs || rhs)); + self.instance.stack.push(Value::Bool(lhs || rhs)); } x if x == OpCode::Not as u8 => self.unary_not_op()?, x if x == OpCode::Neg as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_UNARY_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2641,15 +2491,17 @@ impl Vm { self.record_operand_hint_miss(); match self.pop_numeric()? { NumericValue::Int(value) => { - self.stack.push(Value::Int(value.wrapping_neg())) + self.instance.stack.push(Value::Int(value.wrapping_neg())) + } + NumericValue::Float(value) => { + self.instance.stack.push(Value::Float(-value)) } - NumericValue::Float(value) => self.stack.push(Value::Float(-value)), } } } } x if x == OpCode::Ceq as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2675,12 +2527,12 @@ impl Vm { self.record_operand_hint_miss(); let rhs = self.pop_value()?; let lhs = self.pop_value()?; - self.stack.push(Value::Bool(lhs == rhs)); + self.instance.stack.push(Value::Bool(lhs == rhs)); } } } x if x == OpCode::Clt as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2697,7 +2549,7 @@ impl Vm { } } x if x == OpCode::Cgt as u8 => { - let ip = self.ip - 1; + let ip = self.instance.ip - 1; match self.operand_type_hint(ip) { INT_INT_OPERAND_TYPE_HINT => { self.record_operand_hint_hit(); @@ -2714,23 +2566,23 @@ impl Vm { } } x if x == OpCode::Br as u8 => { - let opcode_ip = self.ip - 1; + let opcode_ip = self.instance.ip - 1; let target = if let Some(target) = self.decoded_jump_target_at(opcode_ip) { - self.ip += 4; + self.instance.ip += 4; target } else { self.read_u32()? as usize }; if self.decoded_jump_target_is_valid_at(opcode_ip) { - self.ip = target; + self.instance.ip = target; } else { self.jump_to(target)?; } } x if x == OpCode::Brfalse as u8 => { - let opcode_ip = self.ip - 1; + let opcode_ip = self.instance.ip - 1; let target = if let Some(target) = self.decoded_jump_target_at(opcode_ip) { - self.ip += 4; + self.instance.ip += 4; target } else { self.read_u32()? as usize @@ -2738,7 +2590,7 @@ impl Vm { let condition = self.pop_bool()?; if !condition { if self.decoded_jump_target_is_valid_at(opcode_ip) { - self.ip = target; + self.instance.ip = target; } else { self.jump_to(target)?; } @@ -2749,12 +2601,12 @@ impl Vm { } x if x == OpCode::Dup as u8 => { let value = self.peek_value()?.clone(); - self.stack.push(value); + self.instance.stack.push(value); } x if x == OpCode::Ldloc as u8 => { - let opcode_ip = self.ip - 1; + let opcode_ip = self.instance.ip - 1; let index = if let Some(index) = self.decoded_local_index_at(opcode_ip) { - self.ip += 1; + self.instance.ip += 1; index } else { self.read_u8()? @@ -2763,12 +2615,12 @@ impl Vm { return Ok(ExecOutcome::Continue); } let value = self.load_local_value(index)?; - self.stack.push(value); + self.instance.stack.push(value); } x if x == OpCode::Stloc as u8 => { - let opcode_ip = self.ip - 1; + let opcode_ip = self.instance.ip - 1; let index = if let Some(index) = self.decoded_local_index_at(opcode_ip) { - self.ip += 1; + self.instance.ip += 1; index } else { self.read_u8()? @@ -2777,7 +2629,7 @@ impl Vm { self.store_local_with_drop_contract(index, value)?; } x if x == OpCode::Call as u8 => { - let call_ip = self.ip - 1; + let call_ip = self.instance.ip - 1; let index = self.read_u16()?; let argc_u8 = self.read_u8()?; let can_fuse_tail_halt = self.can_fuse_call_ret_pattern(); @@ -2787,13 +2639,13 @@ impl Vm { if self.interruption_enabled() { self.charge_interrupt_tick()?; } - self.ip = self.ip.saturating_add(1); + self.instance.ip = self.instance.ip.saturating_add(1); return self.complete_active_frame(); } } HostCallExecOutcome::Halted => return Ok(ExecOutcome::Halted), HostCallExecOutcome::Yielded => { - self.last_yield_reason = Some(VmYieldReason::Host); + self.instance.last_yield_reason = Some(VmYieldReason::Host); return Ok(ExecOutcome::Yielded); } HostCallExecOutcome::Pending(op_id) => return Ok(ExecOutcome::Waiting(op_id)), @@ -2801,7 +2653,7 @@ impl Vm { } x if x == OpCode::CallValue as u8 => { - let call_ip = self.ip.saturating_sub(1); + let call_ip = self.instance.ip.saturating_sub(1); let argc = self.read_u8()?; return self.execute_call_value(argc, Some(call_ip)); } @@ -2812,7 +2664,8 @@ impl Vm { pub fn resume(&mut self) -> VmResult { let allow_jit = !matches!( - self.execution_frames + self.instance + .execution_frames .last() .map(|frame| &frame.continuation), Some(FrameContinuation::ReturnToHost) @@ -2821,16 +2674,16 @@ impl Vm { } pub fn stack(&self) -> &[Value] { - &self.stack + &self.instance.stack } pub fn locals(&self) -> &[Value] { - &self.locals + &self.instance.locals } pub fn set_local(&mut self, index: u8, value: Value) -> VmResult<()> { self.store_local_with_drop_contract(index, value)?; - let config = *self.jit.config(); + let config = *self.engine.jit.config(); self.set_jit_config(config); Ok(()) } @@ -2840,22 +2693,22 @@ impl Vm { } pub fn bound_function_count(&self) -> usize { - self.host_functions.len() + self.host.host_functions.len() } pub fn has_bound_function(&self, name: &str) -> bool { - self.host_function_symbols.contains_key(name) + self.host.host_function_symbols.contains_key(name) } pub fn ip(&self) -> usize { - self.ip + self.instance.ip } pub(super) fn owns_callable(&self, value: &Value) -> bool { let Value::Callable(target) = value else { return false; }; - self.owned_callables.iter().any(|owned| { + self.instance.owned_callables.iter().any(|owned| { owned .upgrade() .is_some_and(|owned| Arc::ptr_eq(&owned, target)) @@ -2872,6 +2725,7 @@ impl Vm { VmError::HostError(format!("unknown exported script function '{name}'")) })?; let value = self + .instance .locals .get(exported.local_slot as usize) .cloned() @@ -2892,7 +2746,7 @@ impl Vm { } pub fn call_depth(&self) -> usize { - self.call_depth + self.instance.call_depth } pub fn queue_callable(&mut self, callable: Value, args: Vec) -> VmResult<()> { @@ -2905,13 +2759,13 @@ impl Vm { args: Vec, subscription: Option>, ) -> VmResult<()> { - if self.shutdown { + if self.instance.shutdown { return Err(VmError::InvalidFrameState("vm is shut down")); } if !matches!(&callable, Value::Callable(_)) { return Err(VmError::InvalidCallable); } - self.queued_callables.push_back(QueuedCallable { + self.instance.queued_callables.push_back(QueuedCallable { callable, args, subscription, @@ -2920,23 +2774,23 @@ impl Vm { } pub fn queued_callable_count(&self) -> usize { - self.queued_callables.len() + self.instance.queued_callables.len() } pub fn drain_callable_queue(&mut self) -> VmResult> { - if self.draining_queued_callables { + if self.instance.draining_queued_callables { return Err(VmError::InvalidFrameState( "callable queue is already being drained", )); } - if !self.execution_frames.is_empty() { + if !self.instance.execution_frames.is_empty() { return Err(VmError::InvalidFrameState( "queued callables can only run after the root frame halts", )); } - self.draining_queued_callables = true; - let mut results = Vec::with_capacity(self.queued_callables.len()); - while let Some(queued) = self.queued_callables.pop_front() { + self.instance.draining_queued_callables = true; + let mut results = Vec::with_capacity(self.instance.queued_callables.len()); + while let Some(queued) = self.instance.queued_callables.pop_front() { if queued .subscription .as_ref() @@ -2946,9 +2800,9 @@ impl Vm { } match self.start_callable(queued.callable, &queued.args) { Ok(VmStatus::Halted) => { - let Some(result) = self.host_return.take() else { - self.completed_callable_results.extend(results); - self.draining_queued_callables = false; + let Some(result) = self.instance.host_return.take() else { + self.instance.completed_callable_results.extend(results); + self.instance.draining_queued_callables = false; return Err(VmError::InvalidFrameState( "queued invocation completed without a result", )); @@ -2956,84 +2810,78 @@ impl Vm { results.push(result); } Ok(VmStatus::Yielded) => { - self.completed_callable_results.extend(results); - self.draining_queued_callables = false; + self.instance.completed_callable_results.extend(results); + self.instance.draining_queued_callables = false; return Err(VmError::InvalidFrameState( "queued invocation yielded; resume it before draining again", )); } Ok(VmStatus::Waiting(_)) => { - self.completed_callable_results.extend(results); - self.draining_queued_callables = false; + self.instance.completed_callable_results.extend(results); + self.instance.draining_queued_callables = false; return Err(VmError::InvalidFrameState( "queued invocation is waiting; resume it before draining again", )); } Err(err) => { - self.completed_callable_results.extend(results); - self.draining_queued_callables = false; + self.instance.completed_callable_results.extend(results); + self.instance.draining_queued_callables = false; return Err(err); } } } - self.draining_queued_callables = false; + self.instance.draining_queued_callables = false; Ok(results) } pub fn shutdown(&mut self) { self.invalidate_callback_registries(); self.cancel_waiting_host_op(); - self.queued_callables.clear(); - self.completed_callable_results.clear(); - self.owned_callables.clear(); - self.draining_queued_callables = false; + self.instance.queued_callables.clear(); + self.instance.completed_callable_results.clear(); + self.instance.owned_callables.clear(); + self.instance.draining_queued_callables = false; self.clear_stack_with_drop_contract(); - self.capture_cells.clear(); - self.shared_capture_slots.clear(); + self.instance.capture_cells.clear(); + self.instance.shared_capture_slots.clear(); self.clear_locals_with_drop_contract(); - self.execution_frames.clear(); - self.active_local_base_cache = 0; - self.active_operand_stack_base_cache = 0; - self.call_depth = 0; - self.host_return = None; - self.waiting_host_op = None; + self.instance.execution_frames.clear(); + self.instance.active_local_base_cache = 0; + self.instance.active_operand_stack_base_cache = 0; + self.instance.call_depth = 0; + self.instance.host_return = None; + self.instance.waiting_host_op = None; crate::builtins::runtime::close_all_handles(self); - self.shutdown = true; + self.instance.shutdown = true; } pub(super) fn register_callback_registry(&mut self, active: &Arc) { - self.callback_registry_flags.push(Arc::downgrade(active)); + self.instance.register_callback_registry(active); } fn invalidate_callback_registries(&mut self) { - for active in self - .callback_registry_flags - .drain(..) - .filter_map(|flag| flag.upgrade()) - { - active.store(false, Ordering::Release); - } + self.instance.invalidate_callback_registries(); } pub fn start_callable(&mut self, callable: Value, args: &[Value]) -> VmResult { - if self.shutdown { + if self.instance.shutdown { return Err(VmError::InvalidFrameState("vm is shut down")); } if !matches!(&callable, Value::Callable(_)) { return Err(VmError::InvalidCallable); } - if !self.execution_frames.is_empty() { + if !self.instance.execution_frames.is_empty() { return Err(VmError::InvalidFrameState( "host invocation requires a halted VM", )); } let argc = u8::try_from(args.len()) .map_err(|_| VmError::InvalidFrameState("too many arguments"))?; - let stack_base = self.stack.len(); - let frame_count = self.execution_frames.len(); - self.stack.push(callable); - self.stack.extend_from_slice(args); - self.host_return = None; + let stack_base = self.instance.stack.len(); + let frame_count = self.instance.execution_frames.len(); + self.instance.stack.push(callable); + self.instance.stack.extend_from_slice(args); + self.instance.host_return = None; let outcome = match self.execute_call_value(argc, None) { Ok(outcome) => outcome, Err(error) => { @@ -3041,10 +2889,10 @@ impl Vm { return Err(error); } }; - if self.execution_frames.len() == frame_count { + if self.instance.execution_frames.len() == frame_count { let result = match outcome { ExecOutcome::Continue | ExecOutcome::Halted => { - self.stack.pop().unwrap_or(Value::Null) + self.instance.stack.pop().unwrap_or(Value::Null) } ExecOutcome::Yielded => { self.abort_host_invocation(stack_base, frame_count); @@ -3059,11 +2907,11 @@ impl Vm { )); } }; - self.stack.truncate(stack_base); - self.host_return = Some(result); + self.instance.stack.truncate(stack_base); + self.instance.host_return = Some(result); return Ok(VmStatus::Halted); } - if let Some(frame) = self.execution_frames.last_mut() { + if let Some(frame) = self.instance.execution_frames.last_mut() { frame.continuation = FrameContinuation::ReturnToHost; } match self.run_internal(None, false) { @@ -3076,12 +2924,16 @@ impl Vm { } pub fn invoke_callable(&mut self, callable: Value, args: &[Value]) -> VmResult { - let stack_base = self.stack.len(); - let frame_count = self.execution_frames.len(); + let stack_base = self.instance.stack.len(); + let frame_count = self.instance.execution_frames.len(); match self.start_callable(callable, args)? { - VmStatus::Halted => self.host_return.take().ok_or(VmError::InvalidFrameState( - "host invocation completed without a result", - )), + VmStatus::Halted => self + .instance + .host_return + .take() + .ok_or(VmError::InvalidFrameState( + "host invocation completed without a result", + )), VmStatus::Yielded => { self.abort_host_invocation(stack_base, frame_count); Err(VmError::InvalidFrameState("host invocation yielded")) @@ -3094,53 +2946,64 @@ impl Vm { } fn abort_host_invocation(&mut self, stack_base: usize, frame_count: usize) { - while self.execution_frames.len() > frame_count { - let Some(frame) = self.execution_frames.pop() else { + while self.instance.execution_frames.len() > frame_count { + let Some(frame) = self.instance.execution_frames.pop() else { break; }; let frame_end = frame.local_base.saturating_add(frame.local_count); - self.capture_cells + self.instance + .capture_cells .retain(|absolute, _| *absolute < frame.local_base || *absolute >= frame_end); - self.shared_capture_slots + self.instance + .shared_capture_slots .retain(|absolute| *absolute < frame.local_base || *absolute >= frame_end); - if frame.local_base <= self.locals.len() { - let drained = self.locals.drain(frame.local_base..).collect::>(); + if frame.local_base <= self.instance.locals.len() { + let drained = self + .instance + .locals + .drain(frame.local_base..) + .collect::>(); for value in drained { self.drop_value_with_contract(value); } } } - self.active_local_base_cache = self + self.instance.active_local_base_cache = self + .instance .execution_frames .last() .map(|frame| frame.local_base) .unwrap_or(0); - self.active_operand_stack_base_cache = self + self.instance.active_operand_stack_base_cache = self + .instance .execution_frames .last() .map(|frame| frame.operand_stack_base) .unwrap_or(0); - while self.stack.len() > stack_base { - if let Some(value) = self.stack.pop() { + while self.instance.stack.len() > stack_base { + if let Some(value) = self.instance.stack.pop() { self.drop_value_with_contract(value); } } - self.call_depth = self.script_frame_depth(); - self.host_return = None; + self.instance.call_depth = self.script_frame_depth(); + self.instance.host_return = None; self.cancel_waiting_host_op(); - self.last_yield_reason = None; - self.map_iterators - .truncate(self.call_depth.saturating_add(1)); + self.instance.last_yield_reason = None; + self.instance + .map_iterators + .truncate(self.instance.call_depth.saturating_add(1)); } pub fn take_callable_result(&mut self) -> Option { - self.completed_callable_results + self.instance + .completed_callable_results .pop_front() - .or_else(|| self.host_return.take()) + .or_else(|| self.instance.host_return.take()) } pub fn execution_frames(&self) -> Vec { - self.execution_frames + self.instance + .execution_frames .iter() .map(|frame| VmExecutionFrameSnapshot { continuation: match frame.continuation { diff --git a/src/vm/native/bridge.rs b/src/vm/native/bridge.rs index ff692ba3..357f6b21 100644 --- a/src/vm/native/bridge.rs +++ b/src/vm/native/bridge.rs @@ -393,12 +393,12 @@ pub(crate) fn non_yielding_i64_host_call_entry_address() -> usize { } pub(crate) fn helper_entry_offset() -> i32 { - i32::try_from(std::mem::offset_of!(Vm, native_helper_fn)) + i32::try_from(std::mem::offset_of!(Vm, engine.native_helper_fn)) .expect("Vm::native_helper_fn offset must fit i32") } pub(crate) fn interrupt_helper_entry_offset() -> i32 { - i32::try_from(std::mem::offset_of!(Vm, native_interrupt_helper_fn)) + i32::try_from(std::mem::offset_of!(Vm, engine.native_interrupt_helper_fn)) .expect("Vm::native_interrupt_helper_fn offset must fit i32") } @@ -780,10 +780,10 @@ pub(crate) extern "C" fn pd_vm_native_restore_exit_state( ip: usize, ) -> i32 { run_step(vm, "restore_exit_state", |vm| { - if locals_len != vm.locals.len() { + if locals_len != vm.instance.locals.len() { return Err(VmError::JitNative(format!( "native exit restore locals length mismatch: expected {}, got {}", - vm.locals.len(), + vm.instance.locals.len(), locals_len ))); } @@ -799,10 +799,10 @@ pub(crate) extern "C" fn pd_vm_native_restore_exit_state( } vm.clear_stack_with_drop_contract(); - vm.stack.reserve(stack_len); + vm.instance.stack.reserve(stack_len); for index in 0..stack_len { let value = unsafe { std::ptr::read(stack_src.add(index)) }; - vm.stack.push(value); + vm.instance.stack.push(value); } for index in 0..locals_len { @@ -819,13 +819,14 @@ pub(crate) extern "C" fn pd_vm_native_restore_exit_state( } fn native_frame_state(vm: &Vm) -> VmResult { - let frame = vm.execution_frames.last(); + let frame = vm.instance.execution_frames.last(); let operand_stack_base = frame.map(|frame| frame.operand_stack_base).unwrap_or(0); let local_base = frame.map(|frame| frame.local_base).unwrap_or(0); let local_count = frame .map(|frame| frame.local_count) - .unwrap_or(vm.locals.len()); + .unwrap_or(vm.instance.locals.len()); let active_stack_len = vm + .instance .stack .len() .checked_sub(operand_stack_base) @@ -847,7 +848,7 @@ fn native_frame_state(vm: &Vm) -> VmResult { active_stack_len, local_base, local_count, - frame_depth: vm.call_depth, + frame_depth: vm.instance.call_depth, continuation_kind, }) } @@ -900,7 +901,7 @@ fn write_inherited_state_packet(vm: &Vm, packet: *mut u8) -> VmResult<()> { packet .add(INHERITED_STATE_TARGET_IP_OFFSET as usize) .cast::() - .write(vm.ip); + .write(vm.instance.ip); packet .add(INHERITED_STATE_VALUE_COUNT_OFFSET as usize) .cast::() @@ -908,11 +909,11 @@ fn write_inherited_state_packet(vm: &Vm, packet: *mut u8) -> VmResult<()> { let values = packet .add(INHERITED_STATE_VALUES_OFFSET as usize) .cast::<*const Value>(); - let stack = vm.stack.as_ptr().add(state.operand_stack_base); + let stack = vm.instance.stack.as_ptr().add(state.operand_stack_base); for index in 0..state.active_stack_len { values.add(index).write(stack.add(index)); } - let locals = vm.locals.as_ptr().add(state.local_base); + let locals = vm.instance.locals.as_ptr().add(state.local_base); for index in 0..state.local_count { values .add(state.active_stack_len + index) @@ -957,13 +958,13 @@ fn native_enter_call_value( .map_err(|_| VmError::InvalidFrameState("native call ip out of range"))?; let resume_ip = usize::try_from(resume_ip) .map_err(|_| VmError::InvalidFrameState("native resume ip out of range"))?; - if vm.ip != call_ip { + if vm.instance.ip != call_ip { vm.jump_to(call_ip)?; } if resume_ip > vm.program.code.len() { return Err(VmError::BytecodeBounds); } - vm.ip = resume_ip; + vm.instance.ip = resume_ip; let status = match vm.execute_call_value(argc, Some(call_ip))? { ExecOutcome::Continue => STATUS_LINKED_CONTINUE, ExecOutcome::Halted => STATUS_HALTED, @@ -1067,17 +1068,17 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_exit_state( let expected_locals_len = local_base .checked_add(locals_len) .ok_or_else(|| VmError::JitNative("native active local length overflow".to_string()))?; - if expected_locals_len != vm.locals.len() { + if expected_locals_len != vm.instance.locals.len() { return Err(VmError::JitNative(format!( "native active exit restore locals length mismatch: expected {}, got {}", - vm.locals.len(), + vm.instance.locals.len(), expected_locals_len ))); } - if stack_base > vm.stack.len() { + if stack_base > vm.instance.stack.len() { return Err(VmError::JitNative(format!( "native active stack base {stack_base} exceeds stack length {}", - vm.stack.len() + vm.instance.stack.len() ))); } if stack_len != 0 && stack_src.is_null() { @@ -1091,11 +1092,11 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_exit_state( )); } - vm.stack.truncate(stack_base); - vm.stack.reserve(stack_len); + vm.instance.stack.truncate(stack_base); + vm.instance.stack.reserve(stack_len); for index in 0..stack_len { let value = unsafe { std::ptr::read(stack_src.add(index)) }; - vm.stack.push(value); + vm.instance.stack.push(value); } for index in 0..locals_len { @@ -1145,10 +1146,10 @@ pub(crate) extern "C" fn pd_vm_native_restore_sparse_exit_state( "native sparse exit restore local index out of range".to_string(), ) })?; - if local_index_usize >= vm.locals.len() { + if local_index_usize >= vm.instance.locals.len() { return Err(VmError::JitNative(format!( "native sparse exit restore local index {local_index} out of range for {} locals", - vm.locals.len() + vm.instance.locals.len() ))); } let local_index = u8::try_from(local_index).map_err(|_| { @@ -1165,10 +1166,10 @@ pub(crate) extern "C" fn pd_vm_native_restore_sparse_exit_state( } vm.clear_stack_with_drop_contract(); - vm.stack.reserve(stack_len); + vm.instance.stack.reserve(stack_len); for index in 0..stack_len { let value = unsafe { std::ptr::read(stack_src.add(index)) }; - vm.stack.push(value); + vm.instance.stack.push(value); } for (compact_index, local_index) in validated_indices.into_iter().enumerate() { @@ -1212,29 +1213,29 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( // while the sparse exit metadata is built. let stack_base = vm.active_operand_stack_base(); - if stack_base > vm.stack.len() { + if stack_base > vm.instance.stack.len() { return Err(VmError::JitNative(format!( "native active sparse stack base {stack_base} exceeds stack length {}", - vm.stack.len() + vm.instance.stack.len() ))); } - vm.stack.truncate(stack_base); - vm.stack.reserve(stack_len); + vm.instance.stack.truncate(stack_base); + vm.instance.stack.reserve(stack_len); for index in 0..stack_len { let value = unsafe { std::ptr::read(stack_src.add(index)) }; - vm.stack.push(value); + vm.instance.stack.push(value); } - if vm.capture_cells.is_empty() { + if vm.instance.capture_cells.is_empty() { let local_base = vm.active_local_base(); - let count_drop_events = vm.drop_contract_events_enabled; + let count_drop_events = vm.instance.drop_contract_events_enabled; for compact_index in 0..dirty_local_count { let local_index = unsafe { *dirty_local_indices.add(compact_index) } as usize; debug_assert!(local_index < 256); let absolute = local_base + local_index; - debug_assert!(absolute < vm.locals.len()); + debug_assert!(absolute < vm.instance.locals.len()); let value = unsafe { std::ptr::read(dirty_local_values.add(compact_index)) }; - let slot = unsafe { vm.locals.get_unchecked_mut(absolute) }; + let slot = unsafe { vm.instance.locals.get_unchecked_mut(absolute) }; let previous = std::mem::replace(slot, value); if count_drop_events { vm.count_value_drop_contract(&previous); @@ -1253,7 +1254,7 @@ pub(crate) extern "C" fn pd_vm_native_restore_active_sparse_exit_state( if ip >= vm.program.code.len() { return Err(VmError::InvalidBranchTarget { target: ip }); } - vm.ip = ip; + vm.instance.ip = ip; Ok(STATUS_CONTINUE) }) } @@ -1281,9 +1282,9 @@ pub(crate) extern "C" fn pd_vm_native_restore_virtual_frame( "virtual frame restore received null locals buffer".to_string(), )); } - if vm.call_depth >= vm.max_script_call_depth { + if vm.instance.call_depth >= vm.instance.max_script_call_depth { return Err(VmError::CallStackOverflow { - limit: vm.max_script_call_depth, + limit: vm.instance.max_script_call_depth, }); } let prototype = vm @@ -1326,29 +1327,31 @@ pub(crate) extern "C" fn pd_vm_native_restore_virtual_frame( )); } - let operand_stack_base = vm.stack.len(); - let local_base = vm.locals.len(); - vm.stack.reserve(stack_len); - vm.locals.reserve(locals_len); + let operand_stack_base = vm.instance.stack.len(); + let local_base = vm.instance.locals.len(); + vm.instance.stack.reserve(stack_len); + vm.instance.locals.reserve(locals_len); for index in 0..stack_len { - vm.stack + vm.instance + .stack .push(unsafe { std::ptr::read(stack_src.add(index)) }); } for index in 0..locals_len { - vm.locals + vm.instance + .locals .push(unsafe { std::ptr::read(locals_src.add(index)) }); } - vm.execution_frames.push(ExecutionFrame { + vm.instance.execution_frames.push(ExecutionFrame { continuation: FrameContinuation::ResumeBytecode { return_ip }, operand_stack_base, local_base, local_count: locals_len, prototype_id: Some(prototype_id), }); - vm.active_local_base_cache = local_base; - vm.active_operand_stack_base_cache = operand_stack_base; - vm.call_depth = vm.script_frame_depth(); - vm.ip = resume_ip; + vm.instance.active_local_base_cache = local_base; + vm.instance.active_operand_stack_base_cache = operand_stack_base; + vm.instance.call_depth = vm.script_frame_depth(); + vm.instance.ip = resume_ip; Ok(STATUS_CONTINUE) }) } @@ -1662,10 +1665,11 @@ fn call_non_yielding_host_value( expected_return_type: Option, ) -> VmResult { let resolved = *vm + .host .resolved_calls .get(import) .ok_or(VmError::InvalidCall(import as u16))?; - let function = match vm.host_functions.get(usize::from(resolved)) { + let function = match vm.host.host_functions.get(usize::from(resolved)) { Some(VmHostFunction::ArgsStaticNonYielding(function)) => *function, _ => { return Err(VmError::JitNative( @@ -1673,9 +1677,9 @@ fn call_non_yielding_host_value( )); } }; - vm.call_depth = vm.call_depth.saturating_add(1); + vm.instance.call_depth = vm.instance.call_depth.saturating_add(1); let outcome = function(args); - vm.call_depth = vm.call_depth.saturating_sub(1); + vm.instance.call_depth = vm.instance.call_depth.saturating_sub(1); outcome .and_then(crate::vm::host::require_non_yielding_host_value) .and_then(|value| { @@ -1809,7 +1813,7 @@ pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, .get(index as usize) .cloned() .ok_or(VmError::InvalidConstant(index))?; - vm.stack.push(value); + vm.instance.stack.push(value); Ok(STATUS_CONTINUE) } OP_ADD => { @@ -1841,34 +1845,41 @@ pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, OP_SHL => { let rhs = vm.pop_shift_amount()?; let lhs = vm.pop_int()?; - vm.stack + vm.instance + .stack .push(crate::bytecode::Value::Int(lhs.wrapping_shl(rhs))); Ok(STATUS_CONTINUE) } OP_SHR => { let rhs = vm.pop_shift_amount()?; let lhs = vm.pop_int()?; - vm.stack + vm.instance + .stack .push(crate::bytecode::Value::Int(lhs.wrapping_shr(rhs))); Ok(STATUS_CONTINUE) } OP_LSHR => { let rhs = vm.pop_shift_amount()?; let lhs = vm.pop_int()?; - vm.stack + vm.instance + .stack .push(crate::bytecode::Value::Int(logical_shr_i64(lhs, rhs))); Ok(STATUS_CONTINUE) } OP_AND => { let rhs = vm.pop_bool()?; let lhs = vm.pop_bool()?; - vm.stack.push(crate::bytecode::Value::Bool(lhs && rhs)); + vm.instance + .stack + .push(crate::bytecode::Value::Bool(lhs && rhs)); Ok(STATUS_CONTINUE) } OP_OR => { let rhs = vm.pop_bool()?; let lhs = vm.pop_bool()?; - vm.stack.push(crate::bytecode::Value::Bool(lhs || rhs)); + vm.instance + .stack + .push(crate::bytecode::Value::Bool(lhs || rhs)); Ok(STATUS_CONTINUE) } OP_NOT => { @@ -1879,18 +1890,22 @@ pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, let value = vm.pop_numeric()?; match value { NumericValue::Int(value) => vm + .instance .stack .push(crate::bytecode::Value::Int(value.wrapping_neg())), - NumericValue::Float(value) => { - vm.stack.push(crate::bytecode::Value::Float(-value)) - } + NumericValue::Float(value) => vm + .instance + .stack + .push(crate::bytecode::Value::Float(-value)), } Ok(STATUS_CONTINUE) } OP_CEQ => { let rhs = vm.pop_value()?; let lhs = vm.pop_value()?; - vm.stack.push(crate::bytecode::Value::Bool(lhs == rhs)); + vm.instance + .stack + .push(crate::bytecode::Value::Bool(lhs == rhs)); Ok(STATUS_CONTINUE) } OP_CLT => { @@ -1907,18 +1922,19 @@ pub(crate) extern "C" fn pd_vm_native_step(vm: *mut Vm, op: i64, a: i64, b: i64, } OP_DUP => { let value = vm.peek_value()?.clone(); - vm.stack.push(value); + vm.instance.stack.push(value); Ok(STATUS_CONTINUE) } OP_LDLOC => { let index = u8::try_from(a) .map_err(|_| VmError::JitNative("ldloc index out of range".to_string()))?; let value = vm + .instance .locals .get(index as usize) .cloned() .ok_or(VmError::InvalidLocal(index))?; - vm.stack.push(value); + vm.instance.stack.push(value); Ok(STATUS_CONTINUE) } OP_STLOC => { @@ -2046,11 +2062,11 @@ mod tests { let mut vm = Vm::new(virtual_frame_program()); let locals = [Value::Int(7)]; let before = ( - vm.ip, - vm.stack.len(), - vm.locals.len(), - vm.execution_frames.len(), - vm.call_depth, + vm.instance.ip, + vm.instance.stack.len(), + vm.instance.locals.len(), + vm.instance.execution_frames.len(), + vm.instance.call_depth, ); let status = pd_vm_native_restore_virtual_frame( &mut vm, @@ -2067,11 +2083,11 @@ mod tests { assert_eq!( before, ( - vm.ip, - vm.stack.len(), - vm.locals.len(), - vm.execution_frames.len(), - vm.call_depth, + vm.instance.ip, + vm.instance.stack.len(), + vm.instance.locals.len(), + vm.instance.execution_frames.len(), + vm.instance.call_depth, ) ); let _ = take_bridge_error(); @@ -2093,11 +2109,11 @@ mod tests { locals.len(), ); assert_eq!(status, STATUS_CONTINUE); - assert_eq!(vm.ip, 2); - assert_eq!(vm.call_depth, 1); - assert_eq!(vm.execution_frames.len(), 2); - assert_eq!(vm.locals.last(), Some(&Value::Int(7))); - let frame = vm.execution_frames.last().unwrap(); + assert_eq!(vm.instance.ip, 2); + assert_eq!(vm.instance.call_depth, 1); + assert_eq!(vm.instance.execution_frames.len(), 2); + assert_eq!(vm.instance.locals.last(), Some(&Value::Int(7))); + let frame = vm.instance.execution_frames.last().unwrap(); assert_eq!(frame.prototype_id, Some(0)); assert_eq!(frame.local_count, 1); assert_eq!( @@ -2135,24 +2151,26 @@ mod tests { let program = crate::Program::new(Vec::new(), vec![crate::OpCode::Ret as u8]).with_local_count(2); let mut vm = Vm::new(program); - vm.stack = vec![Value::Int(10), Value::Int(20)]; - vm.locals = vec![ + vm.instance.stack = vec![Value::Int(10), Value::Int(20)]; + vm.instance.locals = vec![ Value::Int(1), Value::Int(2), Value::Int(3), Value::Int(4), Value::Int(5), ]; - vm.execution_frames.push(crate::vm::ExecutionFrame { - continuation: FrameContinuation::ResumeBytecode { return_ip: 0 }, - operand_stack_base: 1, - local_base: 2, - local_count: 3, - prototype_id: Some(7), - }); - vm.active_local_base_cache = 2; - vm.active_operand_stack_base_cache = 1; - vm.call_depth = 1; + vm.instance + .execution_frames + .push(crate::vm::ExecutionFrame { + continuation: FrameContinuation::ResumeBytecode { return_ip: 0 }, + operand_stack_base: 1, + local_base: 2, + local_count: 3, + prototype_id: Some(7), + }); + vm.instance.active_local_base_cache = 2; + vm.instance.active_operand_stack_base_cache = 1; + vm.instance.call_depth = 1; let mut state = MaybeUninit::::uninit(); assert_eq!( @@ -2188,9 +2206,9 @@ mod tests { ); std::mem::forget(stack); std::mem::forget(locals); - assert_eq!(vm.stack, vec![Value::Int(10), Value::Int(99)]); + assert_eq!(vm.instance.stack, vec![Value::Int(10), Value::Int(99)]); assert_eq!( - vm.locals, + vm.instance.locals, vec![ Value::Int(1), Value::Int(2), @@ -2218,11 +2236,11 @@ mod tests { std::mem::forget(sparse_stack); std::mem::forget(dirty_values); assert_eq!( - vm.stack, + vm.instance.stack, vec![Value::Int(10), Value::Int(77), Value::Int(88)] ); assert_eq!( - vm.locals, + vm.instance.locals, vec![ Value::Int(1), Value::Int(2), @@ -2265,22 +2283,22 @@ mod tests { .expect("bind callable"); assert!(matches!(callable, Value::Callable(_))); - vm.stack.extend([callable, Value::Int(41)]); + vm.instance.stack.extend([callable, Value::Int(41)]); assert_eq!( pd_vm_native_enter_call_value(&mut vm, 1, call_ip as i64, resume_ip as i64,), STATUS_LINKED_CONTINUE ); - assert_eq!(vm.call_depth, 1); - assert_eq!(vm.ip, function.entry_ip as usize); + assert_eq!(vm.instance.call_depth, 1); + assert_eq!(vm.instance.ip, function.entry_ip as usize); - vm.stack.push(Value::Int(42)); + vm.instance.stack.push(Value::Int(42)); assert_eq!( pd_vm_native_leave_frame(&mut vm, ret_ip as i64), STATUS_LINKED_CONTINUE ); - assert_eq!(vm.call_depth, 0); - assert_eq!(vm.ip, resume_ip); - assert_eq!(vm.stack, vec![Value::Int(42)]); + assert_eq!(vm.instance.call_depth, 0); + assert_eq!(vm.instance.ip, resume_ip); + assert_eq!(vm.instance.stack, vec![Value::Int(42)]); } #[test] @@ -2314,7 +2332,7 @@ mod tests { vm.set_local(0, Value::Int(17)).expect("scalar local"); vm.set_local(1, Value::String(preserved.clone())) .expect("heap local"); - vm.stack.push(Value::Int(99)); + vm.instance.stack.push(Value::Int(99)); let status = pd_vm_native_restore_sparse_exit_state( &mut vm, @@ -2342,7 +2360,7 @@ mod tests { crate::Program::new(Vec::new(), vec![crate::OpCode::Ret as u8]).with_local_count(1); let mut vm = Vm::new(program); vm.set_local(0, Value::Int(17)).expect("initial local"); - vm.stack.push(Value::Int(23)); + vm.instance.stack.push(Value::Int(23)); let local_value = Value::Int(99); let null_indices = pd_vm_native_restore_sparse_exit_state( diff --git a/src/vm/native/layout.rs b/src/vm/native/layout.rs index c41c95a5..ed357e12 100644 --- a/src/vm/native/layout.rs +++ b/src/vm/native/layout.rs @@ -56,6 +56,101 @@ pub(crate) struct NativeStackLayout { pub(crate) value: ValueLayout, } +pub(crate) fn native_stack_layout_fingerprint(layout: NativeStackLayout) -> u64 { + const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; + const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + + let NativeStackLayout { + vm_stack_offset, + vm_locals_offset, + vm_program_constants_ptr_offset, + vm_ip_offset, + vm_fuel_remaining_offset, + vm_fuel_ops_until_check_offset, + vm_epoch_deadline_offset, + vm_epoch_counter_ptr_offset, + vm_jit_native_region_edge_count_offset, + vm_jit_native_direct_link_count_offset, + vm_jit_native_active_direct_trace_id_offset, + stack_vec, + map, + value, + } = layout; + let VecLayout { + ptr_offset: stack_vec_ptr_offset, + len_offset: stack_vec_len_offset, + } = stack_vec; + let MapLayout { + len_offset: map_len_offset, + } = map; + let ValueLayout { + size: value_size, + tag_offset: value_tag_offset, + tag_size: value_tag_size, + null_tag, + int_tag, + float_tag, + bool_tag, + string_tag, + bytes_tag, + array_tag, + map_tag, + int_payload_offset, + float_payload_offset, + bool_payload_offset, + heap_payload_offset, + arc_data_offset, + } = value; + + let mut fingerprint = FNV_OFFSET_BASIS; + for byte in b"pd-vm-native-stack-layout-v1\0" { + fingerprint ^= u64::from(*byte); + fingerprint = fingerprint.wrapping_mul(FNV_PRIME); + } + + macro_rules! hash_field { + ($value:expr) => { + for byte in $value.to_le_bytes() { + fingerprint ^= u64::from(byte); + fingerprint = fingerprint.wrapping_mul(FNV_PRIME); + } + }; + } + + hash_field!(vm_stack_offset); + hash_field!(vm_locals_offset); + hash_field!(vm_program_constants_ptr_offset); + hash_field!(vm_ip_offset); + hash_field!(vm_fuel_remaining_offset); + hash_field!(vm_fuel_ops_until_check_offset); + hash_field!(vm_epoch_deadline_offset); + hash_field!(vm_epoch_counter_ptr_offset); + hash_field!(vm_jit_native_region_edge_count_offset); + hash_field!(vm_jit_native_direct_link_count_offset); + hash_field!(vm_jit_native_active_direct_trace_id_offset); + hash_field!(stack_vec_ptr_offset); + hash_field!(stack_vec_len_offset); + hash_field!(map_len_offset); + hash_field!(value_size); + hash_field!(value_tag_offset); + hash_field!(value_tag_size); + hash_field!(null_tag); + hash_field!(int_tag); + hash_field!(float_tag); + hash_field!(bool_tag); + hash_field!(string_tag); + hash_field!(bytes_tag); + hash_field!(array_tag); + hash_field!(map_tag); + hash_field!(int_payload_offset); + hash_field!(float_payload_offset); + hash_field!(bool_payload_offset); + hash_field!(heap_payload_offset); + hash_field!(arc_data_offset); + + fingerprint +} + pub(crate) fn detect_native_stack_layout() -> VmResult { let cached = NATIVE_STACK_LAYOUT .get_or_init(|| detect_native_stack_layout_uncached().map_err(layout_probe_error_message)); @@ -66,39 +161,43 @@ pub(crate) fn detect_native_stack_layout() -> VmResult { } fn detect_native_stack_layout_uncached() -> VmResult { - let vm_stack_offset = usize_to_i32(std::mem::offset_of!(Vm, stack), "Vm::stack offset")?; - let vm_locals_offset = usize_to_i32(std::mem::offset_of!(Vm, locals), "Vm::locals offset")?; + let vm_stack_offset = + usize_to_i32(std::mem::offset_of!(Vm, instance.stack), "Vm::stack offset")?; + let vm_locals_offset = usize_to_i32( + std::mem::offset_of!(Vm, instance.locals), + "Vm::locals offset", + )?; let vm_program_constants_ptr_offset = usize_to_i32( - std::mem::offset_of!(Vm, program_constants_ptr), + std::mem::offset_of!(Vm, engine.program_constants_ptr), "Vm::program_constants_ptr offset", )?; - let vm_ip_offset = usize_to_i32(std::mem::offset_of!(Vm, ip), "Vm::ip offset")?; + let vm_ip_offset = usize_to_i32(std::mem::offset_of!(Vm, instance.ip), "Vm::ip offset")?; let vm_fuel_remaining_offset = usize_to_i32( - std::mem::offset_of!(Vm, fuel_remaining), + std::mem::offset_of!(Vm, run_ctx.fuel_remaining), "Vm::fuel_remaining offset", )?; let vm_fuel_ops_until_check_offset = usize_to_i32( - std::mem::offset_of!(Vm, fuel_ops_until_check), + std::mem::offset_of!(Vm, run_ctx.fuel_ops_until_check), "Vm::fuel_ops_until_check offset", )?; let vm_epoch_deadline_offset = usize_to_i32( - std::mem::offset_of!(Vm, epoch_deadline), + std::mem::offset_of!(Vm, run_ctx.epoch_deadline), "Vm::epoch_deadline offset", )?; let vm_epoch_counter_ptr_offset = usize_to_i32( - std::mem::offset_of!(Vm, epoch_counter_ptr), + std::mem::offset_of!(Vm, run_ctx.epoch_counter_ptr), "Vm::epoch_counter_ptr offset", )?; let vm_jit_native_region_edge_count_offset = usize_to_i32( - std::mem::offset_of!(Vm, jit_native_region_edge_count), + std::mem::offset_of!(Vm, engine.jit_native_region_edge_count), "Vm::jit_native_region_edge_count offset", )?; let vm_jit_native_direct_link_count_offset = usize_to_i32( - std::mem::offset_of!(Vm, jit_native_direct_link_count), + std::mem::offset_of!(Vm, engine.jit_native_direct_link_count), "Vm::jit_native_direct_link_count offset", )?; let vm_jit_native_active_direct_trace_id_offset = usize_to_i32( - std::mem::offset_of!(Vm, jit_native_active_direct_trace_id), + std::mem::offset_of!(Vm, engine.jit_native_active_direct_trace_id), "Vm::jit_native_active_direct_trace_id offset", )?; let stack_vec = detect_vec_layout()?; @@ -508,3 +607,105 @@ fn usize_to_i32(value: usize, context: &str) -> VmResult { i32::try_from(value) .map_err(|_| VmError::JitNative(format!("{} exceeds 32-bit displacement range", context))) } + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture_layout() -> NativeStackLayout { + NativeStackLayout { + vm_stack_offset: 1, + vm_locals_offset: 2, + vm_program_constants_ptr_offset: 3, + vm_ip_offset: 4, + vm_fuel_remaining_offset: 5, + vm_fuel_ops_until_check_offset: 6, + vm_epoch_deadline_offset: 7, + vm_epoch_counter_ptr_offset: 8, + vm_jit_native_region_edge_count_offset: 9, + vm_jit_native_direct_link_count_offset: 10, + vm_jit_native_active_direct_trace_id_offset: 11, + stack_vec: VecLayout { + ptr_offset: 12, + len_offset: 13, + }, + map: MapLayout { len_offset: 14 }, + value: ValueLayout { + size: 15, + tag_offset: 16, + tag_size: 17, + null_tag: 18, + int_tag: 19, + float_tag: 20, + bool_tag: 21, + string_tag: 22, + bytes_tag: 23, + array_tag: 24, + map_tag: 25, + int_payload_offset: 26, + float_payload_offset: 27, + bool_payload_offset: 28, + heap_payload_offset: 29, + arc_data_offset: 30, + }, + } + } + + #[test] + fn native_stack_layout_fingerprint_has_stable_encoding() { + assert_eq!( + native_stack_layout_fingerprint(fixture_layout()), + 0x5730_e57f_2190_5082 + ); + } + + #[test] + fn native_stack_layout_fingerprint_covers_every_embedded_field() { + let layout = fixture_layout(); + let original = native_stack_layout_fingerprint(layout); + + macro_rules! assert_field_is_covered { + ($($field:ident).+) => {{ + let mut changed = layout; + changed.$($field).+ = changed.$($field).+.wrapping_add(1); + assert_ne!( + native_stack_layout_fingerprint(changed), + original, + "{} must affect the native layout fingerprint", + stringify!($($field).+) + ); + }}; + } + + assert_field_is_covered!(vm_stack_offset); + assert_field_is_covered!(vm_locals_offset); + assert_field_is_covered!(vm_program_constants_ptr_offset); + assert_field_is_covered!(vm_ip_offset); + assert_field_is_covered!(vm_fuel_remaining_offset); + assert_field_is_covered!(vm_fuel_ops_until_check_offset); + assert_field_is_covered!(vm_epoch_deadline_offset); + assert_field_is_covered!(vm_epoch_counter_ptr_offset); + assert_field_is_covered!(vm_jit_native_region_edge_count_offset); + assert_field_is_covered!(vm_jit_native_direct_link_count_offset); + assert_field_is_covered!(vm_jit_native_active_direct_trace_id_offset); + assert_field_is_covered!(stack_vec.ptr_offset); + assert_field_is_covered!(stack_vec.len_offset); + assert_field_is_covered!(map.len_offset); + assert_field_is_covered!(value.size); + assert_field_is_covered!(value.tag_offset); + assert_field_is_covered!(value.tag_size); + assert_field_is_covered!(value.null_tag); + assert_field_is_covered!(value.int_tag); + assert_field_is_covered!(value.float_tag); + assert_field_is_covered!(value.bool_tag); + assert_field_is_covered!(value.string_tag); + assert_field_is_covered!(value.bytes_tag); + assert_field_is_covered!(value.array_tag); + assert_field_is_covered!(value.map_tag); + assert_field_is_covered!(value.int_payload_offset); + assert_field_is_covered!(value.float_payload_offset); + assert_field_is_covered!(value.bool_payload_offset); + assert_field_is_covered!(value.heap_payload_offset); + assert_field_is_covered!(value.arc_data_offset); + } +} diff --git a/src/vm/native/mod.rs b/src/vm/native/mod.rs index 86b536cb..41b12ae1 100644 --- a/src/vm/native/mod.rs +++ b/src/vm/native/mod.rs @@ -50,6 +50,7 @@ pub(crate) use codegen::{ pub(crate) use exec::{ExecutableBuffer, prepare_for_execution}; pub(crate) use layout::{ NativeStackLayout, ValueLayout, checked_add_i32, detect_native_stack_layout, + native_stack_layout_fingerprint, }; #[cfg(feature = "cranelift-jit")] pub(crate) use offsets::{HeapIntrinsicAddrs, HeapIntrinsicRefs, ResolvedOffsets, resolve_offsets}; diff --git a/src/vm/program.rs b/src/vm/program.rs new file mode 100644 index 00000000..10d34236 --- /dev/null +++ b/src/vm/program.rs @@ -0,0 +1,22 @@ +//! Immutable program artifact. +//! +//! [`Program`] is the compiled, immutable unit of +//! execution: bytecode, constants, metadata, import requirements, and +//! binding tables. This module documents its ownership contract for the VM +//! runtime decomposition: +//! +//! - A `Program` is immutable after compilation and binding metadata +//! construction; sharing one `Program` (e.g. through `Arc`) is the +//! only supported way to share code between VMs or instances. +//! - Per-run state (stacks, locals, frames, wait state) never lives in the +//! program; it lives in the VM's private `Instance` state. +//! - Backend caches derived from the program (decoded instruction data, +//! operand type hints, AOT/JIT artifacts) live in +//! the VM's private `Engine` state and are keyed by the program's cache +//! identity, never owned by a run. +//! +//! Thread safety: `Program` is `Send + Sync` and `Clone`-cheap only through +//! `Arc`; cloning the struct itself duplicates metadata, which is allowed but +//! wasteful. Prefer `Arc` for sharing. + +pub use crate::bytecode::Program; diff --git a/src/vm/run_context.rs b/src/vm/run_context.rs new file mode 100644 index 00000000..235a1c5a --- /dev/null +++ b/src/vm/run_context.rs @@ -0,0 +1,189 @@ +//! Run-scoped execution context. +//! +//! [`RunContext`] owns everything that belongs to one execution of a program: +//! the generic runtime input/event context, fuel and epoch budgets, the +//! interrupt mode, and the epoch counter handle. A fresh logical run starts +//! from a reset context; nothing here survives a reset except the epoch handle +//! identity (which is intentionally process-lifetime). +//! +//! The embedder-facing fuel/epoch APIs live on the VM facade (see +//! `crate::vm::fuel` and `crate::vm::epoch`) and delegate here; cancellation +//! of pending host operations lives in the facade because it crosses into +//! [`HostRuntime`](super::host_runtime::HostRuntime) state. + +use crate::builtins::runtime::cancellation::{CancellationReason, CancellationToken}; +use crate::builtins::runtime::context::RuntimeContext; +use crate::vm::VmResult; +use crate::vm::epoch::EpochHandle; + +/// Run interruption mode: no budget, fuel metering, or epoch deadlines. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub(crate) enum InterruptMode { + None = 0, + Fuel = 1, + Epoch = 2, +} + +impl InterruptMode { + pub(crate) fn label(self) -> &'static str { + match self { + Self::None => "none", + Self::Fuel => "fuel", + Self::Epoch => "epoch", + } + } +} + +/// Run-scoped input, events, budgets, deadlines, and interruption state. +/// +/// Thread safety: `RunContext` is `!Sync` (event sink and counters are +/// mutable) and not shared; one facade owns one context. Clone semantics: +/// not `Clone` — a clone would duplicate event/input state across runs. +pub(crate) struct RunContext { + pub(crate) runtime_context: RuntimeContext, + pub(crate) cancellation: CancellationToken, + pub(crate) interrupt_mode: InterruptMode, + pub(crate) fuel_remaining: u64, + pub(crate) fuel_check_interval: u32, + pub(crate) fuel_ops_until_check: u32, + pub(crate) epoch_deadline: u64, + pub(crate) epoch_deadline_delta: u64, + pub(crate) epoch_rearm_pending: bool, + pub(crate) epoch_handle: EpochHandle, + // Native ABI mirror: the epoch counter address read by generated code. + // Load-bearing for `crate::vm::native`; see `crate::vm::engine`. + #[allow(dead_code)] + pub(crate) epoch_counter_ptr: usize, +} + +impl RunContext { + /// Creates a fresh run context with no input, no event sink, and no + /// budgets (interrupts disabled). + pub(crate) fn new() -> Self { + let epoch_handle = EpochHandle::default(); + let epoch_counter_ptr = epoch_handle.as_ptr() as usize; + Self { + runtime_context: RuntimeContext::default(), + cancellation: CancellationToken::root(), + interrupt_mode: InterruptMode::None, + fuel_remaining: 0, + fuel_check_interval: 1, + fuel_ops_until_check: 1, + epoch_deadline: 0, + epoch_deadline_delta: 0, + epoch_rearm_pending: false, + epoch_handle, + epoch_counter_ptr, + } + } + + /// Closes run-scoped state for reuse: input and events are cleared and + /// fuel/epoch budgets are dropped (metering disabled, no leftovers). + pub(crate) fn reset_for_reuse(&mut self) { + self.cancellation.cancel(CancellationReason::VmReset); + self.cancellation = CancellationToken::root(); + self.epoch_rearm_pending = false; + self.clear_fuel_internal(); + self.clear_epoch_deadline_internal(); + self.runtime_context.reset_for_reuse(); + } + + pub(crate) fn cancel(&self, reason: CancellationReason) -> VmResult<()> { + self.cancellation.cancel(reason); + match self.cancellation.take_propagation_error() { + Some(error) => Err(crate::vm::VmError::HostError(error.to_string())), + None => Ok(()), + } + } + + pub(crate) fn reset_interrupt_countdown(&mut self) { + self.fuel_ops_until_check = self.fuel_check_interval.max(1); + } + + pub(crate) fn clear_fuel_internal(&mut self) { + if self.interrupt_mode == InterruptMode::Fuel { + self.interrupt_mode = InterruptMode::None; + } + self.fuel_remaining = 0; + self.reset_interrupt_countdown(); + } + + pub(crate) fn clear_epoch_deadline_internal(&mut self) { + if self.interrupt_mode == InterruptMode::Epoch { + self.interrupt_mode = InterruptMode::None; + } + self.epoch_deadline = 0; + self.epoch_deadline_delta = 0; + self.epoch_rearm_pending = false; + self.reset_interrupt_countdown(); + } + + pub(crate) fn pending_fuel_debt(&self) -> u64 { + if self.interrupt_mode != InterruptMode::Fuel { + return 0; + } + let executed_since_last_check = self + .fuel_check_interval + .saturating_sub(self.fuel_ops_until_check); + u64::from(executed_since_last_check) + } + + /// Charges a fixed amount of fuel; errors when the budget is exhausted. + pub(crate) fn charge_fuel(&mut self, amount: u64) -> VmResult<()> { + if amount == 0 || self.interrupt_mode != InterruptMode::Fuel { + return Ok(()); + } + let remaining = self.fuel_remaining; + if remaining < amount { + return Err(crate::vm::VmError::OutOfFuel { + needed: amount, + remaining, + }); + } + self.fuel_remaining = remaining - amount; + Ok(()) + } + + /// Charges one fuel interval according to the countdown; errors when the + /// budget is exhausted. + pub(crate) fn charge_fuel_tick(&mut self) -> VmResult<()> { + if self.interrupt_mode != InterruptMode::Fuel { + return Ok(()); + } + if self.fuel_ops_until_check > 1 { + self.fuel_ops_until_check -= 1; + return Ok(()); + } + let amount = u64::from(self.fuel_check_interval); + self.charge_fuel(amount)?; + self.fuel_ops_until_check = self.fuel_check_interval; + Ok(()) + } + + /// Charges one epoch countdown tick; errors when the deadline passed. + pub(crate) fn charge_epoch_tick(&mut self) -> VmResult<()> { + if self.interrupt_mode != InterruptMode::Epoch { + return Ok(()); + } + if self.fuel_ops_until_check > 1 { + self.fuel_ops_until_check -= 1; + return Ok(()); + } + let current = self.epoch_handle.current(); + if current >= self.epoch_deadline { + return Err(crate::vm::VmError::EpochDeadlineReached { + current, + deadline: self.epoch_deadline, + }); + } + self.fuel_ops_until_check = self.fuel_check_interval; + Ok(()) + } +} + +impl Default for RunContext { + fn default() -> Self { + Self::new() + } +} diff --git a/src/vm/superinstructions.rs b/src/vm/superinstructions.rs index 47c8e581..e6e87f1f 100644 --- a/src/vm/superinstructions.rs +++ b/src/vm/superinstructions.rs @@ -46,7 +46,8 @@ impl Vm { #[inline(always)] pub(super) fn decoded_ldc_value_at(&self, opcode_ip: usize) -> Option<&Value> { - self.decoded_instruction_data + self.engine + .decoded_instruction_data .ldc_values .get(opcode_ip) .and_then(|value| value.as_ref()) @@ -54,7 +55,8 @@ impl Vm { #[inline(always)] pub(super) fn decoded_jump_target_at(&self, opcode_ip: usize) -> Option { - self.decoded_instruction_data + self.engine + .decoded_instruction_data .jump_targets .get(opcode_ip) .and_then(|target| *target) @@ -62,7 +64,8 @@ impl Vm { #[inline(always)] pub(super) fn decoded_jump_target_is_valid_at(&self, opcode_ip: usize) -> bool { - self.decoded_instruction_data + self.engine + .decoded_instruction_data .valid_jump_targets .get(opcode_ip) .copied() @@ -71,7 +74,8 @@ impl Vm { #[inline(always)] pub(super) fn decoded_local_index_at(&self, opcode_ip: usize) -> Option { - self.decoded_instruction_data + self.engine + .decoded_instruction_data .local_indices .get(opcode_ip) .and_then(|index| *index) @@ -90,7 +94,7 @@ impl Vm { let Some(initial) = self.local_scalar_value_with_hint(src) else { return Ok(false); }; - let mut cursor = self.ip; + let mut cursor = self.instance.ip; let mut stack = [None; 8]; let mut stack_len = 1usize; stack[0] = Some(initial); @@ -221,7 +225,7 @@ impl Vm { ))?; self.store_local_absolute_with_drop_contract(absolute, dst, value)?; self.record_scalar_superinstruction(); - self.ip = cursor + 2; + self.instance.ip = cursor + 2; return Ok(true); } OpCode::Clt | OpCode::Cgt => { @@ -261,10 +265,10 @@ impl Vm { }, _ => unreachable!(), }; - self.ip = cursor + 6; + self.instance.ip = cursor + 6; if !condition { if self.decoded_jump_target_is_valid_at(jump_opcode_ip) { - self.ip = target; + self.instance.ip = target; } else { self.jump_to(target)?; } diff --git a/src/vm/tests.rs b/src/vm/tests.rs index b7f0e0d1..b90e2750 100644 --- a/src/vm/tests.rs +++ b/src/vm/tests.rs @@ -1,7 +1,9 @@ +use super::host::WaitingHostOp; use super::*; use crate::builtins::BuiltinFunction; use crate::bytecode::TypeMap; use std::collections::HashMap; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; fn native_cache_test_lock() -> &'static Mutex<()> { @@ -9,18 +11,65 @@ fn native_cache_test_lock() -> &'static Mutex<()> { LOCK.get_or_init(|| Mutex::new(())) } +#[test] +fn failed_dynamic_builtin_override_preserves_runtime_owned_pending_binding() { + struct Dummy; + + impl HostFunction for Dummy { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> VmResult { + unreachable!("rejected override must never be installed") + } + } + + let compiled = crate::compile_source("use runtime; runtime::sleep(0);") + .expect("runtime sleep program should compile"); + let mut vm = Vm::new(compiled.program); + vm.ensure_call_bindings() + .expect("default fallback should bind runtime sleep"); + let slot = vm.host.host_function_symbols["runtime::sleep"]; + assert!(vm.host.runtime_owned_pending_host_slots.contains(&slot)); + + vm.bind_builtin_override("runtime::sleep", Box::new(Dummy)) + .expect_err("runtime sleep is a host import, not a builtin override"); + + assert!(vm.host.runtime_owned_pending_host_slots.contains(&slot)); +} + +#[test] +fn failed_static_builtin_override_preserves_runtime_owned_pending_binding() { + fn dummy(_vm: &mut Vm, _args: &[Value]) -> VmResult { + unreachable!("rejected override must never be installed") + } + + let compiled = crate::compile_source("use runtime; runtime::sleep(0);") + .expect("runtime sleep program should compile"); + let mut vm = Vm::new(compiled.program); + vm.ensure_call_bindings() + .expect("default fallback should bind runtime sleep"); + let slot = vm.host.host_function_symbols["runtime::sleep"]; + assert!(vm.host.runtime_owned_pending_host_slots.contains(&slot)); + + vm.bind_builtin_static_override("runtime::sleep", dummy) + .expect_err("runtime sleep is a host import, not a builtin override"); + + assert!(vm.host.runtime_owned_pending_host_slots.contains(&slot)); +} + #[test] fn root_ret_completes_explicit_halt_frame() { let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); - assert_eq!(vm.execution_frames.len(), 1); - assert_eq!(vm.execution_frames[0].continuation, FrameContinuation::Halt); + assert_eq!(vm.instance.execution_frames.len(), 1); + assert_eq!( + vm.instance.execution_frames[0].continuation, + FrameContinuation::Halt + ); assert_eq!(vm.run().expect("root ret should run"), VmStatus::Halted); - assert!(vm.execution_frames.is_empty()); + assert!(vm.instance.execution_frames.is_empty()); assert!(vm.stack().is_empty()); vm.reset_for_reuse(); - assert_eq!(vm.execution_frames.len(), 1); + assert_eq!(vm.instance.execution_frames.len(), 1); assert_eq!(vm.stack(), &[]); } @@ -32,11 +81,356 @@ fn reset_for_reuse_keeps_host_operation_ids_monotonic() { assert_eq!(vm.allocate_host_op_id(), 2); } +#[test] +fn unused_host_operation_ids_do_not_consume_registry_capacity() { + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + for _ in 0..128 { + vm.allocate_host_op_id(); + } + assert_eq!(vm.host.runtime_operations.active_count(), 0); +} + +#[test] +fn external_host_operations_join_the_shared_registry_without_id_collisions() { + use crate::builtins::runtime::cancellation::{OperationId, OperationOwner}; + + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let runtime_operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Io, + Some(&vm.run_ctx.cancellation), + None, + None, + ) + .expect("runtime operation should start"); + + let collision = vm + .set_waiting_host_op(runtime_operation.id().raw()) + .expect_err("host operation must not reuse a runtime-owned id"); + assert!(collision.to_string().contains("collides")); + assert!( + vm.host + .runtime_operations + .get(runtime_operation.id()) + .is_ok() + ); + vm.host + .runtime_operations + .complete(runtime_operation.id()) + .expect("runtime operation should complete"); + vm.set_waiting_host_op(runtime_operation.id().raw()) + .expect_err("colliding external operation id must remain retired"); + + vm.set_waiting_host_op(99) + .expect("external host operation should register"); + let external = vm + .host + .runtime_operations + .get(OperationId::from_raw(99).expect("operation id should be valid")) + .expect("external operation should be registered"); + assert_eq!(external.owner(), OperationOwner::HostBridge); +} + +#[test] +fn invalid_host_completion_preserves_the_registered_operation() { + use crate::builtins::runtime::cancellation::{OperationId, OperationOwner}; + + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + vm.set_waiting_host_op(101) + .expect("external host operation should register"); + + vm.complete_host_op(102, CallReturn::none()) + .expect_err("completion for a different operation should fail"); + let operation_id = OperationId::from_raw(101).expect("operation id should be valid"); + assert_eq!( + vm.host + .runtime_operations + .get(operation_id) + .expect("waiting operation should remain registered") + .owner(), + OperationOwner::HostBridge + ); + assert_eq!(vm.waiting_host_op_id(), Some(101)); +} + +#[test] +fn reset_and_drop_cleanup_real_host_resources_exactly_once() { + use crate::builtins::runtime::cancellation::CancellationReason; + use crate::builtins::runtime::resource::ResourceTypeId; + + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let reasons = Arc::new(Mutex::new(Vec::new())); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let cleanup_count_for_resource = Arc::clone(&cleanup_count); + let reasons_for_resource = Arc::clone(&reasons); + vm.host + .runtime_resources + .insert_with_cleanup(ResourceTypeId::IO_FILE, (), move |(), reason| { + cleanup_count_for_resource.fetch_add(1, Ordering::SeqCst); + reasons_for_resource + .lock() + .expect("reason lock") + .push(reason); + Ok(()) + }) + .expect("test resource should be inserted"); + + vm.reset_for_reuse(); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + assert_eq!( + reasons.lock().expect("reason lock").as_slice(), + &[CancellationReason::VmReset] + ); + + drop(vm); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn drop_cleans_real_host_resources_without_prior_reset() { + use crate::builtins::runtime::cancellation::CancellationReason; + use crate::builtins::runtime::resource::ResourceTypeId; + + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let cleanup_reason = Arc::new(Mutex::new(None)); + { + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let cleanup_count_for_resource = Arc::clone(&cleanup_count); + let cleanup_reason_for_resource = Arc::clone(&cleanup_reason); + vm.host + .runtime_resources + .insert_with_cleanup(ResourceTypeId::IO_FILE, (), move |(), reason| { + cleanup_count_for_resource.fetch_add(1, Ordering::SeqCst); + *cleanup_reason_for_resource.lock().expect("reason lock") = Some(reason); + Ok(()) + }) + .expect("test resource should be inserted"); + } + + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + assert_eq!( + *cleanup_reason.lock().expect("reason lock"), + Some(CancellationReason::VmReset) + ); +} + +#[test] +fn reset_propagates_to_real_host_operation_cleanup() { + use crate::builtins::runtime::cancellation::{ + CancellationReason, OperationEnd, OperationOwner, OperationStatus, + }; + + let cleanup_end = Arc::new(Mutex::new(None)); + let cleanup_end_for_operation = Arc::clone(&cleanup_end); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Io, + Some(&vm.run_ctx.cancellation), + None, + Some(Box::new(move |end| { + *cleanup_end_for_operation.lock().expect("cleanup lock") = Some(end); + Ok(()) + })), + ) + .expect("test operation should start"); + vm.instance.waiting_host_op = Some(WaitingHostOp { + op_id: operation.id().raw(), + }); + + vm.reset_for_reuse(); + assert_eq!( + operation.status(), + OperationStatus::Cancelled(CancellationReason::VmReset) + ); + assert_eq!( + *cleanup_end.lock().expect("cleanup lock"), + Some(OperationEnd::Cancelled(CancellationReason::VmReset)) + ); +} + +#[test] +fn deadline_cancellation_closes_operation_payload_before_registry_removal() { + use crate::builtins::runtime::cancellation::{CancellationReason, OperationOwner}; + use crate::builtins::runtime::resource::ResourceTypeId; + use std::task::{Context, Poll}; + use std::time::{Duration, Instant}; + + let cleanup_reason = Arc::new(Mutex::new(None)); + let cleanup_reason_for_payload = Arc::clone(&cleanup_reason); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Http, + Some(&vm.run_ctx.cancellation), + Some(Instant::now() - Duration::from_millis(1)), + None, + ) + .expect("deadline operation should start"); + let payload = vm + .host + .runtime_resources + .insert_with_cleanup(ResourceTypeId::CALLBACK, (), move |(), reason| { + *cleanup_reason_for_payload.lock().expect("cleanup lock") = Some(reason); + Ok(()) + }) + .expect("payload should be inserted"); + operation.set_payload(payload); + + let waker = futures_util::task::noop_waker(); + let mut context = Context::from_waker(&waker); + let result = + crate::builtins::runtime::poll_builtin_io_op(&mut vm, operation.id().raw(), &mut context); + + assert!(matches!(result, Poll::Ready(Err(_)))); + assert_eq!( + *cleanup_reason.lock().expect("cleanup lock"), + Some(CancellationReason::Deadline) + ); + assert!(vm.host.runtime_operations.get(operation.id()).is_err()); + assert!( + vm.host + .runtime_resources + .get::<()>(payload, ResourceTypeId::CALLBACK) + .is_err() + ); +} + +#[test] +fn worker_observed_deadline_retains_payload_until_vm_consumes_operation() { + use crate::builtins::runtime::cancellation::{CancellationReason, OperationOwner}; + use crate::builtins::runtime::error::{RuntimeError, RuntimeErrorCode}; + use crate::builtins::runtime::resource::ResourceTypeId; + use std::task::{Context, Poll}; + use std::time::{Duration, Instant}; + + let cleanup_reason = Arc::new(Mutex::new(None)); + let cleanup_reason_for_payload = Arc::clone(&cleanup_reason); + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Http, + Some(&vm.run_ctx.cancellation), + Some(Instant::now() - Duration::from_millis(1)), + None, + ) + .expect("deadline operation should start"); + let payload = vm + .host + .runtime_resources + .insert_with_cleanup(ResourceTypeId::CALLBACK, (), move |(), reason| { + *cleanup_reason_for_payload.lock().expect("cleanup lock") = Some(reason); + Ok(()) + }) + .expect("payload should be inserted"); + operation.set_payload(payload); + + assert!( + operation + .fail(RuntimeError::new( + RuntimeErrorCode::OperationFailed, + "test::worker", + "worker failure", + )) + .expect("worker terminal transition should succeed") + ); + assert!(vm.host.runtime_operations.get(operation.id()).is_ok()); + + let waker = futures_util::task::noop_waker(); + let mut context = Context::from_waker(&waker); + let result = + crate::builtins::runtime::poll_builtin_io_op(&mut vm, operation.id().raw(), &mut context); + + assert!(matches!(result, Poll::Ready(Err(_)))); + assert_eq!( + *cleanup_reason.lock().expect("cleanup lock"), + Some(CancellationReason::Deadline) + ); + assert!(vm.host.runtime_operations.get(operation.id()).is_err()); + assert!( + vm.host + .runtime_resources + .get::<()>(payload, ResourceTypeId::CALLBACK) + .is_err() + ); +} + +#[cfg(feature = "sqlite")] +#[test] +fn sqlite_reconfiguration_only_closes_sqlite_owned_state() { + use crate::builtins::runtime::cancellation::OperationOwner; + use crate::builtins::runtime::resource::ResourceTypeId; + + let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])); + let io_resource = vm + .host + .runtime_resources + .insert(ResourceTypeId::IO_FILE, 11_i64) + .expect("IO resource should be inserted"); + let sqlite_resource = vm + .host + .runtime_resources + .insert(ResourceTypeId::SQLITE_CONNECTION, 22_i64) + .expect("SQLite resource should be inserted"); + let io_operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Io, + Some(&vm.run_ctx.cancellation), + None, + None, + ) + .expect("IO operation should start"); + io_operation.set_resource(io_resource); + let sqlite_operation = vm + .host + .runtime_operations + .start_owned( + OperationOwner::Sqlite, + Some(&vm.run_ctx.cancellation), + None, + None, + ) + .expect("SQLite operation should start"); + sqlite_operation.set_resource(sqlite_resource); + + vm.configure_sqlite(crate::vm::SqlitePolicy::default()); + + assert!( + vm.host + .runtime_resources + .get::(io_resource, ResourceTypeId::IO_FILE) + .is_ok() + ); + assert!(vm.host.runtime_operations.get(io_operation.id()).is_ok()); + assert!( + vm.host + .runtime_resources + .get::(sqlite_resource, ResourceTypeId::SQLITE_CONNECTION) + .is_err() + ); + assert!( + vm.host + .runtime_operations + .get(sqlite_operation.id()) + .is_err() + ); +} + #[test] fn shared_capture_cell_rejects_callable_ownership_cycle() { let mut vm = Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8]).with_local_count(1)); let cell = Arc::new(Mutex::new(Value::Null)); - vm.capture_cells.insert(0, Arc::clone(&cell)); + vm.instance.capture_cells.insert(0, Arc::clone(&cell)); let environment = Arc::new(crate::CallableEnvironment { cells: Mutex::new(vec![cell]), }); @@ -109,7 +503,7 @@ fn callvalue_decodes_its_arity_before_callable_validation() { Vec::new(), vec![OpCode::CallValue as u8, 0, OpCode::Ret as u8], )); - vm.stack.push(Value::Null); + vm.instance.stack.push(Value::Null); assert!(matches!(vm.run(), Err(VmError::InvalidCallable))); assert_eq!(vm.ip(), 2); } @@ -293,7 +687,7 @@ fn aot_executes_move_detach_without_stack_contract_mismatch() { VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::String(Arc::new("x".to_string()))]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -314,7 +708,7 @@ fn aot_executes_script_callable_frames_without_interpreter_boundary() { ); assert_eq!(vm.stack(), &[Value::Int(42)]); assert!(vm.aot_exec_count() >= 3); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -334,7 +728,7 @@ fn aot_executes_typed_script_callable_parameter_equality_without_interpreter_bou VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Bool(true)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -355,7 +749,7 @@ fn aot_executes_script_callable_bool_return_in_branch_without_interpreter_bounda VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Int(1)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -377,7 +771,7 @@ fn aot_executes_capturing_closure_without_interpreter_boundary() { ); assert_eq!(vm.stack(), &[Value::Int(42)]); assert!(vm.aot_exec_count() >= 3); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -397,7 +791,7 @@ fn aot_executes_builtin_callable_values_without_interpreter_boundary() { VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Int(3)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -424,7 +818,7 @@ fn aot_callable_call_resumes_after_fuel_yield_without_interpreter_boundary() { VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Int(42)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -445,7 +839,7 @@ fn aot_executes_nested_script_callables_without_interpreter_boundary() { VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Int(42)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -464,7 +858,7 @@ fn aot_recursive_script_callable_reports_depth_limit_without_interpreter_boundar vm.run(), Err(VmError::CallStackOverflow { limit: 1024 }) )); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[cfg(feature = "cranelift-jit")] @@ -493,7 +887,7 @@ fn aot_host_callable_value_waits_and_resumes_without_interpreter_boundary() { vm.run().expect("pending host callable should wait"), VmStatus::Waiting(812) ); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); vm.complete_host_op(812, vec![Value::Int(42)]) .expect("host operation should complete"); assert_eq!( @@ -501,7 +895,7 @@ fn aot_host_callable_value_waits_and_resumes_without_interpreter_boundary() { VmStatus::Halted ); assert_eq!(vm.stack(), &[Value::Int(42)]); - assert!(!vm.aot_interpreter_boundary_hit); + assert!(!vm.engine.aot_interpreter_boundary_hit); } #[test] @@ -943,8 +1337,8 @@ fn vm_instances_share_decoded_instruction_metadata_across_program_clones() { assert!( Arc::ptr_eq( - &vm_one.decoded_instruction_data, - &vm_two.decoded_instruction_data + &vm_one.engine.decoded_instruction_data, + &vm_two.engine.decoded_instruction_data ), "program clones should share decoded instruction metadata" ); @@ -969,7 +1363,11 @@ fn borrowed_map_iterator_state_is_released_after_break() { assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); assert!( - vm.map_iterators.iter().flatten().all(Option::is_none), + vm.instance + .map_iterators + .iter() + .flatten() + .all(Option::is_none), "break must release every iterator owned by the exited loop" ); } @@ -991,7 +1389,11 @@ fn borrowed_map_iterator_state_is_released_after_runtime_error() { vm.run().expect_err("program should fail at runtime"); assert!( - vm.map_iterators.iter().flatten().all(Option::is_none), + vm.instance + .map_iterators + .iter() + .flatten() + .all(Option::is_none), "runtime errors must release active map iterators" ); } @@ -1008,7 +1410,7 @@ fn map_iterator_ids_are_isolated_by_call_depth() { }; vm.init_map_iterator(7, outer).expect("outer init"); - vm.call_depth = 1; + vm.instance.call_depth = 1; vm.init_map_iterator(7, inner).expect("inner init"); assert!(vm.advance_map_iterator(7).expect("inner advance")); assert_eq!( @@ -1017,7 +1419,7 @@ fn map_iterator_ids_are_isolated_by_call_depth() { ); vm.close_map_iterator(7).expect("inner close"); - vm.call_depth = 0; + vm.instance.call_depth = 0; assert!(vm.advance_map_iterator(7).expect("outer advance")); assert_eq!( vm.take_map_iterator_key(7).expect("outer key"), @@ -1079,7 +1481,7 @@ fn native_trace_cache_resets_when_program_changes() { jit::runtime::native_trace_cache_snapshot_for_tests(); assert_eq!( cache_program_after_one, - Some(vm_one.program_cache_key), + Some(vm_one.engine.program_cache_key), "cache should be keyed to first program after first run" ); assert_eq!( @@ -1094,7 +1496,7 @@ fn native_trace_cache_resets_when_program_changes() { max_trace_len: 512, }); assert_ne!( - vm_one.program_cache_key, vm_two.program_cache_key, + vm_one.engine.program_cache_key, vm_two.engine.program_cache_key, "test programs should have different cache keys" ); let status_two = vm_two.run().expect("second vm should run"); @@ -1109,7 +1511,7 @@ fn native_trace_cache_resets_when_program_changes() { jit::runtime::native_trace_cache_snapshot_for_tests(); assert_eq!( cache_program_after_two, - Some(vm_two.program_cache_key), + Some(vm_two.engine.program_cache_key), "cache should switch to second program key" ); assert_eq!( @@ -1161,7 +1563,7 @@ fn native_trace_cache_reuses_entries_for_same_program() { jit::runtime::native_trace_cache_snapshot_for_tests(); assert_eq!( cache_program_after_one, - Some(vm_one.program_cache_key), + Some(vm_one.engine.program_cache_key), "cache should be keyed to the first program" ); assert_eq!( @@ -1176,7 +1578,7 @@ fn native_trace_cache_reuses_entries_for_same_program() { max_trace_len: 512, }); assert_eq!( - vm_two.program_cache_key, vm_one.program_cache_key, + vm_two.engine.program_cache_key, vm_one.engine.program_cache_key, "same program should use identical cache key" ); @@ -1192,7 +1594,7 @@ fn native_trace_cache_reuses_entries_for_same_program() { jit::runtime::native_trace_cache_snapshot_for_tests(); assert_eq!( cache_program_after_two, - Some(vm_two.program_cache_key), + Some(vm_two.engine.program_cache_key), "cache key should remain the same for identical program" ); assert_eq!( @@ -1344,7 +1746,7 @@ fn interpreter_superinstructions_use_local_type_hints() { let outcome = step_once(&mut vm).expect("ldloc should fuse scalar sequence"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.locals[0], Value::Int(10)); + assert_eq!(vm.instance.locals[0], Value::Int(10)); let metrics = vm.interpreter_metrics_snapshot(); assert_eq!(metrics.scalar_superinstruction_count, 1); assert!( @@ -1375,7 +1777,8 @@ fn interpreter_ldc_shares_string_constant_backing() { fn interpreter_dup_shares_array_backing() { let program = Program::new(vec![], vec![OpCode::Dup as u8, OpCode::Ret as u8]); let mut vm = Vm::new(program); - vm.stack + vm.instance + .stack .push(Value::array(vec![Value::Int(1), Value::Int(2)])); let outcome = step_once(&mut vm).expect("dup should execute"); @@ -1503,14 +1906,17 @@ fn interpreter_ldloc_preserves_local_slot() { let outcome = step_once(&mut vm).expect("ldloc should execute"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.ip, 2); - assert_eq!(vm.locals[0], map_value, "ldloc should leave local intact"); + assert_eq!(vm.instance.ip, 2); + assert_eq!( + vm.instance.locals[0], map_value, + "ldloc should leave local intact" + ); assert_eq!( vm.stack(), &[map_value], "stack should receive copied value" ); - assert_shared_heap_backing(&vm.locals[0], &vm.stack()[0]); + assert_shared_heap_backing(&vm.instance.locals[0], &vm.stack()[0]); assert_eq!(vm.drop_contract_event_count(), 0); } @@ -1539,9 +1945,9 @@ fn interpreter_explicit_move_sequence_clears_local_slot() { let ldloc = step_once(&mut vm).expect("ldloc should execute"); assert!(matches!(ldloc, ExecOutcome::Continue)); - assert_eq!(vm.locals[0], map_value); + assert_eq!(vm.instance.locals[0], map_value); assert_eq!(vm.stack(), std::slice::from_ref(&map_value)); - assert_shared_heap_backing(&vm.locals[0], &vm.stack()[0]); + assert_shared_heap_backing(&vm.instance.locals[0], &vm.stack()[0]); let ldc = step_once(&mut vm).expect("ldc should execute"); assert!(matches!(ldc, ExecOutcome::Continue)); @@ -1549,8 +1955,8 @@ fn interpreter_explicit_move_sequence_clears_local_slot() { let stloc = step_once(&mut vm).expect("stloc should execute"); assert!(matches!(stloc, ExecOutcome::Continue)); - assert_eq!(vm.ip, 9); - assert_eq!(vm.locals[0], Value::Null); + assert_eq!(vm.instance.ip, 9); + assert_eq!(vm.instance.locals[0], Value::Null); assert_eq!(vm.stack(), &[map_value]); } @@ -1579,9 +1985,9 @@ fn interpreter_fuses_ldloc_ldc_add_stloc_without_touching_stack() { let outcome = step_once(&mut vm).expect("fused sequence should execute"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.ip, 10, "fusion should consume ldc/add/stloc"); - assert_eq!(vm.locals[0], Value::Int(41)); - assert_eq!(vm.locals[1], Value::Int(42)); + assert_eq!(vm.instance.ip, 10, "fusion should consume ldc/add/stloc"); + assert_eq!(vm.instance.locals[0], Value::Int(41)); + assert_eq!(vm.instance.locals[1], Value::Int(42)); assert!( vm.stack().is_empty(), "fusion should avoid transient stack traffic" @@ -1621,7 +2027,10 @@ fn interpreter_fuses_ldloc_ldc_compare_brfalse() { let outcome = step_once(&mut vm).expect("fused compare should execute"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.ip, 15, "fusion should jump directly to branch target"); + assert_eq!( + vm.instance.ip, 15, + "fusion should jump directly to branch target" + ); assert!( vm.stack().is_empty(), "fusion should avoid bool stack traffic" @@ -1664,9 +2073,9 @@ fn interpreter_fuses_generic_scalar_update_chain() { let outcome = step_once(&mut vm).expect("generic chain should fuse"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.ip, 19); - assert_eq!(vm.locals[0], Value::Int(29)); - assert_eq!(vm.locals[1], Value::Int(4)); + assert_eq!(vm.instance.ip, 19); + assert_eq!(vm.instance.locals[0], Value::Int(29)); + assert_eq!(vm.instance.locals[1], Value::Int(4)); assert!(vm.stack().is_empty()); } @@ -1708,13 +2117,13 @@ fn interpreter_fuses_float_scalar_sequences() { let first = step_once(&mut vm).expect("float update should fuse"); assert!(matches!(first, ExecOutcome::Continue)); - assert_eq!(vm.ip, 10); - assert_eq!(vm.locals[0], Value::Float(2.5)); + assert_eq!(vm.instance.ip, 10); + assert_eq!(vm.instance.locals[0], Value::Float(2.5)); assert!(vm.stack().is_empty()); let second = step_once(&mut vm).expect("float compare should fuse"); assert!(matches!(second, ExecOutcome::Continue)); - assert_eq!(vm.ip, 23); + assert_eq!(vm.instance.ip, 23); assert!(vm.stack().is_empty()); } @@ -1747,9 +2156,12 @@ fn interpreter_does_not_fuse_ldloc_sequences_when_fuel_is_enabled() { .execute_interpreter_instruction(opcode, false) .expect("ldloc should execute without fusion"); assert!(matches!(outcome, ExecOutcome::Continue)); - assert_eq!(vm.ip, 2, "ldloc should advance only past its operand"); + assert_eq!( + vm.instance.ip, 2, + "ldloc should advance only past its operand" + ); assert_eq!(vm.stack(), &[Value::Int(41)]); - assert_eq!(vm.locals[0], Value::Int(41)); + assert_eq!(vm.instance.locals[0], Value::Int(41)); } #[test] @@ -1776,7 +2188,7 @@ fn interpreter_copy_like_ldloc_dup_stloc_shares_map_backing_with_fuel() { let _ = step_once(&mut vm).expect("stloc should execute"); assert_eq!(vm.stack().len(), 1); - assert_shared_heap_backing(&vm.locals[0], &vm.stack()[0]); + assert_shared_heap_backing(&vm.instance.locals[0], &vm.stack()[0]); } #[test] @@ -1787,11 +2199,14 @@ fn interpreter_fuses_call_ret_without_fuel() { vec![OpCode::Call as u8, call_lo, call_hi, 1, OpCode::Ret as u8], ); let mut vm = Vm::new(program); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let outcome = step_once(&mut vm).expect("call should execute"); assert!(matches!(outcome, ExecOutcome::Halted)); - assert_eq!(vm.ip, 5, "tail-call fusion should consume trailing ret"); + assert_eq!( + vm.instance.ip, 5, + "tail-call fusion should consume trailing ret" + ); assert_eq!(vm.stack(), &[Value::Int(4)]); } @@ -1804,12 +2219,15 @@ fn interpreter_fuses_call_ret_when_fuel_enabled_if_tail_tick_available() { ); let mut vm = Vm::new(program); vm.set_fuel(1); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); // `step_once` bypasses the outer run-loop pre-tick, so this fuel only covers fused `ret`. let call = step_once(&mut vm).expect("call should execute"); assert!(matches!(call, ExecOutcome::Halted)); - assert_eq!(vm.ip, 5, "tail-call fusion should consume trailing ret"); + assert_eq!( + vm.instance.ip, 5, + "tail-call fusion should consume trailing ret" + ); assert_eq!(vm.stack(), &[Value::Int(4)]); assert_eq!(vm.get_fuel(), Some(0)); } @@ -1823,7 +2241,7 @@ fn interpreter_call_ret_fusion_preserves_ip_when_tail_tick_exhausted() { ); let mut vm = Vm::new(program); vm.set_fuel(0); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let err = match step_once(&mut vm) { Ok(_) => panic!("tail tick should fail with out-of-fuel"), @@ -1831,7 +2249,7 @@ fn interpreter_call_ret_fusion_preserves_ip_when_tail_tick_exhausted() { }; assert!(matches!(err, VmError::OutOfFuel { .. })); assert_eq!( - vm.ip, 4, + vm.instance.ip, 4, "ret must remain pending when tail tick cannot be charged" ); assert_eq!(vm.stack(), &[Value::Int(4)]); @@ -1847,7 +2265,7 @@ fn interpreter_call_ret_fusion_preserves_ip_when_epoch_deadline_is_reached() { let mut vm = Vm::new(program); vm.set_epoch_deadline(0) .expect("setting epoch deadline should succeed"); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let err = match step_once(&mut vm) { Ok(_) => panic!("tail tick should fail with epoch deadline reached"), @@ -1855,7 +2273,7 @@ fn interpreter_call_ret_fusion_preserves_ip_when_epoch_deadline_is_reached() { }; assert!(matches!(err, VmError::EpochDeadlineReached { .. })); assert_eq!( - vm.ip, 4, + vm.instance.ip, 4, "ret must remain pending when the epoch check trips during fused tail execution" ); assert_eq!(vm.stack(), &[Value::Int(4)]); @@ -1870,11 +2288,11 @@ fn run_consumes_two_ticks_for_call_ret_when_fuel_enabled() { ); let mut vm = Vm::new(program); vm.set_fuel(2); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let status = vm.run().expect("run should complete"); assert_eq!(status, VmStatus::Halted); - assert_eq!(vm.ip, 5); + assert_eq!(vm.instance.ip, 5); assert_eq!(vm.stack(), &[Value::Int(4)]); assert_eq!( vm.get_fuel(), @@ -1892,12 +2310,12 @@ fn run_yields_before_ret_in_call_ret_sequence_when_out_of_fuel() { ); let mut vm = Vm::new(program); vm.set_fuel(1); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let status = vm.run().expect("first run should yield"); assert_eq!(status, VmStatus::Yielded); assert_eq!( - vm.ip, 4, + vm.instance.ip, 4, "fuel exhaustion should happen before trailing ret" ); assert_eq!(vm.stack(), &[Value::Int(4)]); @@ -1906,7 +2324,7 @@ fn run_yields_before_ret_in_call_ret_sequence_when_out_of_fuel() { vm.add_fuel(1).expect("recharging fuel should succeed"); let resumed = vm.resume().expect("resume should execute trailing ret"); assert_eq!(resumed, VmStatus::Halted); - assert_eq!(vm.ip, 5); + assert_eq!(vm.instance.ip, 5); assert_eq!(vm.stack(), &[Value::Int(4)]); } @@ -1923,12 +2341,12 @@ fn run_yields_before_ret_in_call_ret_sequence_when_epoch_deadline_is_reached() { vm.set_epoch_deadline(1) .expect("setting epoch deadline should succeed"); assert_eq!(vm.increment_epoch(), 1); - vm.stack.push(Value::string("tail")); + vm.instance.stack.push(Value::string("tail")); let status = vm.run().expect("first run should yield"); assert_eq!(status, VmStatus::Yielded); assert_eq!( - vm.ip, 4, + vm.instance.ip, 4, "epoch interruption should happen before trailing ret" ); assert_eq!(vm.last_yield_reason(), Some(VmYieldReason::Epoch)); @@ -1938,7 +2356,7 @@ fn run_yields_before_ret_in_call_ret_sequence_when_epoch_deadline_is_reached() { .resume() .expect("resume should auto re-arm the epoch deadline and execute trailing ret"); assert_eq!(resumed, VmStatus::Halted); - assert_eq!(vm.ip, 5); + assert_eq!(vm.instance.ip, 5); assert_eq!(vm.stack(), &[Value::Int(4)]); } @@ -1950,7 +2368,7 @@ fn call_ret_fusion_pattern_requires_immediate_ret() { vec![OpCode::Call as u8, call_lo, call_hi, 1, OpCode::Ret as u8], ); let mut vm_with_ret = Vm::new(with_ret); - vm_with_ret.ip = 4; + vm_with_ret.instance.ip = 4; assert!(vm_with_ret.can_fuse_call_ret_pattern()); let wrong_next = Program::new( @@ -1958,11 +2376,11 @@ fn call_ret_fusion_pattern_requires_immediate_ret() { vec![OpCode::Call as u8, call_lo, call_hi, 1, OpCode::Nop as u8], ); let mut vm_wrong_next = Vm::new(wrong_next); - vm_wrong_next.ip = 4; + vm_wrong_next.instance.ip = 4; assert!(!vm_wrong_next.can_fuse_call_ret_pattern()); let no_next = Program::new(vec![], vec![OpCode::Call as u8, call_lo, call_hi, 1]); let mut vm_no_next = Vm::new(no_next); - vm_no_next.ip = 4; + vm_no_next.instance.ip = 4; assert!(!vm_no_next.can_fuse_call_ret_pattern()); } diff --git a/tests/builtins/io_builtin_edge_tests.rs b/tests/builtins/io_builtin_edge_tests.rs index 0e58dc0c..9348ed44 100644 --- a/tests/builtins/io_builtin_edge_tests.rs +++ b/tests/builtins/io_builtin_edge_tests.rs @@ -1,5 +1,10 @@ use vm::{Value, Vm, VmError, VmStatus, compile_source}; +#[cfg(unix)] +use std::path::PathBuf; +#[cfg(unix)] +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + fn run_source(source: &str) -> Result, VmError> { let wrapped = format!("use io;\n{source}"); let compiled = compile_source(&wrapped).expect("source should compile"); @@ -28,6 +33,162 @@ fn run_source_host_error(source: &str) -> String { } } +#[cfg(unix)] +fn unique_temp_path(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should follow the Unix epoch") + .as_nanos(); + std::env::temp_dir().join(format!("pd-vm-{label}-{}-{nonce}", std::process::id())) +} + +#[cfg(unix)] +fn process_exists(process_id: i32) -> bool { + // SAFETY: signal zero performs existence/permission checking without sending a signal. + let result = unsafe { libc::kill(process_id, 0) }; + result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +#[test] +fn io_callback_resource_is_registered_before_worker_spawn() { + let source = include_str!("../../src/builtins/runtime/io.rs"); + let schedule = source + .split_once("fn schedule_io_task(") + .expect("schedule_io_task should exist") + .1 + .split_once("fn runtime_host_error(") + .expect("schedule_io_task should precede runtime_host_error") + .0; + let callback_registration = schedule + .find(".insert(ResourceTypeId::CALLBACK, receiver)") + .expect("schedule_io_task should register its callback receiver"); + let worker_spawn = schedule + .find(".spawn(move ||") + .expect("schedule_io_task should spawn its worker"); + + assert!( + callback_registration < worker_spawn, + "callback receiver must be registered before the worker can run" + ); +} + +#[test] +fn popen_teardown_does_not_invoke_external_kill_programs() { + let source = include_str!("../../src/builtins/runtime/io.rs"); + assert!( + !source.contains("Command::new(\"kill\")"), + "Unix popen teardown must use the platform process API" + ); + assert!( + !source.contains("Command::new(\"taskkill\")"), + "Windows popen teardown must use the platform process API" + ); +} + +#[cfg(unix)] +#[test] +fn reset_terminates_popen_descendants() { + let child_pid_path = unique_temp_path("popen-descendant-pid"); + let command = format!( + "sleep 3600 & child=$!; echo $child > {}; wait", + child_pid_path.display() + ); + let compiled = compile_source(&format!( + r#" + use io; + let handle = io::popen("{command}", "r"); + io::read_all(handle); + "# + )) + .expect("descendant popen source should compile"); + let mut vm = Vm::new(compiled.program); + + let first = vm.run().expect("popen should start"); + assert!(matches!(first, VmStatus::Waiting(_))); + vm.wait_for_host_op_blocking() + .expect("popen should complete"); + let second = vm.resume().expect("read_all should start"); + assert!(matches!(second, VmStatus::Waiting(_))); + + let pid_deadline = Instant::now() + Duration::from_secs(2); + while !child_pid_path.exists() && Instant::now() < pid_deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let child_pid = std::fs::read_to_string(&child_pid_path) + .expect("popen command should publish its descendant pid") + .trim() + .parse::() + .expect("descendant pid should be numeric"); + assert!(process_exists(child_pid), "descendant should be running"); + + vm.reset_for_reuse(); + + let exit_deadline = Instant::now() + Duration::from_secs(2); + while process_exists(child_pid) && Instant::now() < exit_deadline { + std::thread::sleep(Duration::from_millis(5)); + } + let _ = std::fs::remove_file(&child_pid_path); + assert!( + !process_exists(child_pid), + "popen descendant {child_pid} survived VM reset" + ); +} + +#[cfg(unix)] +#[test] +fn reset_interrupts_a_blocked_popen_read_within_a_bounded_time() { + let compiled = compile_source( + r#" + use io; + let handle = io::popen("sleep 3600", "r"); + io::read_all(handle); + "#, + ) + .expect("blocking popen source should compile"); + let mut vm = Vm::new(compiled.program); + + let first = vm.run().expect("popen should start"); + assert!(matches!(first, VmStatus::Waiting(_))); + vm.wait_for_host_op_blocking() + .expect("popen should complete"); + let second = vm.resume().expect("read_all should start"); + assert!(matches!(second, VmStatus::Waiting(_))); + std::thread::sleep(Duration::from_millis(25)); + + let started = Instant::now(); + vm.reset_for_reuse(); + assert!( + started.elapsed() < Duration::from_secs(2), + "reset exceeded bounded I/O teardown window: {:?}", + started.elapsed() + ); +} + +#[cfg(unix)] +#[test] +fn reset_reaps_a_popen_child_before_completion_is_polled() { + let compiled = compile_source( + r#" + use io; + io::popen("sleep 3599", "r"); + "#, + ) + .expect("popen source should compile"); + let mut vm = Vm::new(compiled.program); + + let status = vm.run().expect("popen should enter waiting state"); + assert!(matches!(status, VmStatus::Waiting(_))); + std::thread::sleep(Duration::from_millis(100)); + + let started = Instant::now(); + vm.reset_for_reuse(); + assert!( + started.elapsed() < Duration::from_secs(2), + "reset exceeded queued-completion teardown window: {:?}", + started.elapsed() + ); +} + #[test] fn io_open_rejects_unsupported_mode() { let err = run_source_host_error( @@ -119,3 +280,33 @@ fn io_flush_on_read_handle_is_a_noop_true() { .expect("program should execute"); assert_eq!(stack.last(), Some(&Value::Bool(true))); } + +#[test] +fn io_close_rejects_a_stale_resource_handle() { + let err = run_source_host_error( + r#" + let handle = io::open("Cargo.toml", "r"); + io::close(handle); + io::close(handle); + "#, + ); + assert!( + err.contains("resource_already_closed"), + "unexpected error message: {err}" + ); +} + +#[test] +fn io_handles_cannot_cross_vm_resource_arenas() { + let stack = run_source(r#"io::open("Cargo.toml", "r");"#) + .expect("first VM should open a file resource"); + let Value::Int(handle) = stack.last().expect("open should return a handle") else { + panic!("open should return an integer resource handle"); + }; + + let err = run_source_host_error(&format!("io::close({handle});")); + assert!( + err.contains("resource_handle_wrong_table"), + "unexpected error message: {err}" + ); +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index f6967e09..89fc659d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -141,6 +141,7 @@ pub enum CompileErrorKind { InvalidFieldAccess, FunctionParameterTypeConflict, StrictTypingRequired, + UnresolvedModuleCall, } #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -184,6 +185,7 @@ fn compile_error_kind(err: &vm::CompileError) -> CompileErrorKind { CompileErrorKind::FunctionParameterTypeConflict } vm::CompileError::StrictTypingRequired { .. } => CompileErrorKind::StrictTypingRequired, + vm::CompileError::UnresolvedModuleCall => CompileErrorKind::UnresolvedModuleCall, } } diff --git a/tests/compiler/compiler_rustscript_tests.rs b/tests/compiler/compiler_rustscript_tests.rs index 0ccb66ef..024a2bf1 100644 --- a/tests/compiler/compiler_rustscript_tests.rs +++ b/tests/compiler/compiler_rustscript_tests.rs @@ -2642,8 +2642,8 @@ fn compile_source_file_rustscript_imports_merge_with_scoped_locals() { debug .locals .iter() - .any(|local| local.name == "module::shared"), - "module-scoped local should remain visible in debug metadata" + .any(|local| local.name.ends_with("::shared") && local.name != "shared"), + "module-scoped local should remain visible in debug metadata with a deterministic module-identity scope (milestone 4), not a bare file stem" ); assert!( debug.locals.iter().any(|local| local.name == "shared"), @@ -2858,7 +2858,10 @@ fn compile_source_file_rustscript_imported_direct_capture_multiple_move_is_rejec Err(err) => err, }; match err { - vm::SourcePathError::Source(vm::SourceError::Parse(parse)) => { + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(parse), + .. + } => { assert!( parse.message.contains("lut") && parse.message.contains("moved"), "unexpected parse error: {parse:?}" diff --git a/tests/compiler/diagnostics_tests.rs b/tests/compiler/diagnostics_tests.rs index 6e68eba3..57651c3f 100644 --- a/tests/compiler/diagnostics_tests.rs +++ b/tests/compiler/diagnostics_tests.rs @@ -95,16 +95,19 @@ pub fn ok() { let _ = fs::remove_dir(&root); match result { - Err(SourcePathError::Source(SourceError::Compile(compile))) => { + Err(SourcePathError::SourceWithMap { + error: SourceError::Compile(compile), + sources, + }) => { assert_eq!( compile.source_name(), Some(module_path.to_string_lossy().as_ref()) ); assert_eq!(compile.line(), Some(2)); - let mut source_map = SourceMap::new(); - source_map.add_source(module_path.display().to_string(), module_source); - let rendered = render_compile_error(&source_map, &compile, false); + // Milestone 5: the compilation-wide map travels with the error, + // so the rendered diagnostic reads the owning module source. + let rendered = render_compile_error(&sources, &compile, false); assert!(rendered.contains(&format!("{}:2:1", module_path.display()))); assert!(rendered.contains("let broken = if cond => {")); assert!(rendered.contains("int vs string")); diff --git a/tests/compiler/frontend_plugin_tests.rs b/tests/compiler/frontend_plugin_tests.rs index e6dcdfd8..9bc9ec6f 100644 --- a/tests/compiler/frontend_plugin_tests.rs +++ b/tests/compiler/frontend_plugin_tests.rs @@ -36,6 +36,8 @@ impl SourcePlugin for ConstantPlugin { function_impls: HashMap::new(), stmt_sources: Vec::new(), function_sources: HashMap::new(), + use_declarations: Vec::new(), + implicit_extern_names: Vec::new(), }) } } diff --git a/tests/compiler/module_import_tests.rs b/tests/compiler/module_import_tests.rs index 1973cb0b..10229fcd 100644 --- a/tests/compiler/module_import_tests.rs +++ b/tests/compiler/module_import_tests.rs @@ -16,7 +16,9 @@ fn temp_module_root(prefix: &str) -> PathBuf { ); let root = std::env::temp_dir().join(unique); std::fs::create_dir_all(&root).expect("temp module root should be created"); - root + // Module identities are canonical for existing files; keep expected paths + // canonical too so assertions match under symlinked temp directories. + root.canonicalize().unwrap_or(root) } fn write_source(path: &Path, source: &str, description: &str) { @@ -69,6 +71,128 @@ fn compile_source_file_module_override_path_redirects_import_spec() { remove_module_root(&root); } +#[test] +fn nested_module_override_parse_error_preserves_source_text_and_path() { + let root = temp_module_root("vm_rustscript_nested_override_error_test"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::virtual::nested as nested; + nested::run(); + "#, + "main source", + ); + let override_source = "pub fn run( {"; + let options = CompileSourceFileOptions::new() + .with_module_override_source("virtual/nested.rss", override_source); + + let error = match compile_source_file_with_options(&main_path, options) { + Ok(_) => panic!("invalid override module should fail"), + Err(error) => error, + }; + match error { + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(parse), + .. + } => { + assert!(parse.message.contains("virtual/nested.rss")); + } + error => panic!("expected source-aware nested error, got {error:?}"), + } + + remove_module_root(&root); +} + +#[test] +fn nested_module_strict_unknown_diagnostic_keeps_module_source() { + let root = temp_module_root("vm_rustscript_nested_strict_diag_test"); + let main_path = root.join("main.rss"); + let nested_path = root.join("nested.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + write_source( + &nested_path, + "pub fn run() -> unknown { 1 }", + "nested source", + ); + + let error = match compile_source_file(&main_path) { + Ok(_) => panic!("unknown nested annotation should fail in strict RustScript"), + Err(error) => error, + }; + match error { + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(parse), + .. + } => { + assert!(parse.message.contains(&nested_path.display().to_string())); + } + error => panic!("expected nested strict diagnostic, got {error:?}"), + } + + remove_module_root(&root); +} + +#[test] +fn strict_nested_diagnostic_path_is_consistent_across_option_entry_points() { + let root_source = "use self::nested as nested;\nnested::run();\n"; + let nested_source = "pub fn run() -> unknown { 1 }"; + let options = + CompileSourceFileOptions::new().with_module_override_source("nested.rss", nested_source); + + let in_memory_error = match vm::compile_source_with_flavor_and_options( + root_source, + SourceFlavor::RustScript, + options.clone(), + ) { + Ok(_) => panic!("strict nested annotation should fail"), + Err(error) => error, + }; + match in_memory_error { + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(parse), + .. + } => { + assert!(parse.message.contains("__pd_vm_inmemory__/nested.rss")); + } + error => panic!("expected nested strict diagnostic, got {error:?}"), + } + + let root = temp_module_root("vm_rustscript_nested_strict_entry_test"); + let main_path = root.join("main.rss"); + let at_path_error = match vm::compile_source_at_path_with_flavor_and_options( + &main_path, + root_source, + SourceFlavor::RustScript, + options, + ) { + Ok(_) => panic!("strict nested annotation should fail"), + Err(error) => error, + }; + match at_path_error { + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(parse), + .. + } => { + assert!( + parse + .message + .contains(&root.join("nested.rss").display().to_string()) + ); + } + error => panic!("expected nested strict diagnostic, got {error:?}"), + } + + remove_module_root(&root); +} + #[test] fn compile_source_file_rustscript_named_import_is_selective() { let root = temp_module_root("vm_rustscript_selective_import_test"); @@ -104,7 +228,10 @@ fn compile_source_file_rustscript_named_import_is_selective() { assert!( matches!( err, - vm::SourcePathError::Source(vm::SourceError::Parse(vm::ParseError { ref message, .. })) + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(vm::ParseError { ref message, .. }), + .. + } if message.contains("unknown function 'add_two'") ), "expected unknown function error, got {err:?}" @@ -211,7 +338,10 @@ fn compile_source_file_rustscript_module_exports_only_pub_functions() { assert!( matches!( err, - vm::SourcePathError::Source(vm::SourceError::Parse(vm::ParseError { ref message, .. })) + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(vm::ParseError { ref message, .. }), + .. + } if message.contains("unknown function 'private_add'") ), "expected unknown function error, got {err:?}" @@ -318,3 +448,466 @@ fn compile_source_file_imported_module_dynamic_slice_end_bindings_work() { remove_module_root(&root); } + +#[test] +fn nested_module_namespace_import_rewrites_sibling_calls() { + let root = temp_module_root("vm_rustscript_nested_namespace_import_test"); + write_source( + &root.join("sibling.rss"), + r#" + pub fn value() -> int { 7 } + "#, + "sibling source", + ); + write_source( + &root.join("nested.rss"), + r#" + use self::sibling as sibling; + pub fn run() -> int { sibling::value() } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("nested namespace import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(7)]); + + remove_module_root(&root); +} + +#[test] +fn nested_module_named_import_rewrites_sibling_calls() { + let root = temp_module_root("vm_rustscript_nested_named_import_test"); + write_source( + &root.join("sibling.rss"), + r#" + pub fn value() -> int { 11 } + "#, + "sibling source", + ); + write_source( + &root.join("nested.rss"), + r#" + use self::sibling::{value}; + pub fn run() -> int { value() } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("nested named import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(11)]); + remove_module_root(&root); +} + +#[test] +fn nested_module_super_import_resolves_parent_directory_sibling() { + let root = temp_module_root("vm_rustscript_nested_super_import_test"); + write_source( + &root.join("shared.rss"), + r#" + pub fn value() -> int { 13 } + "#, + "parent sibling source", + ); + let package = root.join("pkg"); + std::fs::create_dir_all(&package).expect("package directory should be created"); + write_source( + &package.join("nested.rss"), + r#" + use super::shared as shared; + pub fn run() -> int { shared::value() } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::pkg::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("nested super import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(13)]); + + remove_module_root(&root); +} + +#[test] +fn nested_module_missing_sibling_reports_nested_source() { + let root = temp_module_root("vm_rustscript_nested_missing_import_test"); + write_source( + &root.join("nested.rss"), + r#" + use self::missing as missing; + pub fn run() -> int { missing::value() } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + + let error = match compile_source_file(&main_path) { + Ok(_) => panic!("missing nested sibling should fail"), + Err(error) => error, + }; + assert!( + matches!( + error, + vm::SourcePathError::Io(ref io_error) + if io_error.kind() == std::io::ErrorKind::NotFound + ), + "missing nested sibling should remain a filesystem error: {error:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn explicit_self_import_cycle_is_detected_after_path_normalization() { + let root = temp_module_root("vm_rustscript_self_cycle_import_test"); + write_source( + &root.join("a.rss"), + r#" + use self::b as b; + pub fn run() -> int { b::run() } + "#, + "a source", + ); + write_source( + &root.join("b.rss"), + r#" + use self::a as a; + pub fn run() -> int { a::run() } + "#, + "b source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::a as a; + a::run(); + "#, + "main source", + ); + + let error = match compile_source_file(&main_path) { + Ok(_) => panic!("explicit self import cycle should fail"), + Err(error) => error, + }; + assert!( + matches!(error, vm::SourcePathError::ImportCycle(_)), + "expected import cycle error, got {error:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn nested_module_does_not_reexport_transitive_imports() { + let root = temp_module_root("vm_rustscript_nested_export_boundary_test"); + write_source( + &root.join("sibling.rss"), + r#" + pub fn leaf() -> int { 19 } + "#, + "sibling source", + ); + write_source( + &root.join("nested.rss"), + r#" + use self::sibling as sibling; + pub fn run() -> int { sibling::leaf() } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::leaf(); + "#, + "main source", + ); + + let error = match compile_source_file(&main_path) { + Ok(_) => panic!("transitive import should not be re-exported"), + Err(error) => error, + }; + assert!( + matches!( + error, + vm::SourcePathError::SourceWithMap { + error: vm::SourceError::Parse(vm::ParseError { ref message, .. }), + .. + } + if message.contains("nested::leaf") || message.contains("unknown namespace") + ), + "expected transitive export boundary error, got {error:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn nested_module_rewrite_preserves_utf8_values_byte_for_byte() { + let root = temp_module_root("vm_rustscript_nested_utf8_import_test"); + write_source( + &root.join("sibling.rss"), + r#" + // 猫のコメント: the sibling module is untouched by rewriting. + pub fn echo(value: string) -> string { value } + "#, + "sibling source", + ); + write_source( + &root.join("nested.rss"), + r#" + /* 前置ブロック: 猫 */ + use self::sibling as sibling; + use self::sibling::{echo as echo_named}; + pub fn run() -> string { + let namespace_value = sibling::echo("猫"); + let named_value = echo_named("🐱 にゃん"); + // 行コメント: 猫 + let joined = namespace_value + named_value; + joined + } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("nested utf-8 imports should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::string("猫🐱 にゃん")], + "UTF-8 literals must survive namespace and named import rewriting" + ); + + remove_module_root(&root); +} + +#[test] +fn nested_module_consecutive_super_import_resolves_two_levels_up() { + let root = temp_module_root("vm_rustscript_consecutive_super_import_test"); + write_source( + &root.join("shared.rss"), + r#" + pub fn value() -> int { 17 } + "#, + "root sibling source", + ); + let package = root.join("pkg").join("sub"); + std::fs::create_dir_all(&package).expect("package directory should be created"); + write_source( + &package.join("nested.rss"), + r#" + use super::super::shared as shared; + pub fn run() -> int { shared::value() } + "#, + "two-level nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::pkg::sub::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = + compile_source_file(&main_path).expect("consecutive super import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(17)]); + + remove_module_root(&root); +} + +#[test] +fn path_aliases_resolve_to_single_module_identity() { + let root = temp_module_root("vm_rustscript_path_alias_identity_test"); + write_source( + &root.join("a.rss"), + r#" + pub fn value() -> int { 23 } + "#, + "module a source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::a as a; + use a as a2; + let x = a::value(); + let y = a2::value(); + x + y; + "#, + "main source", + ); + + let compiled = + compile_source_file(&main_path).expect("lexically distinct path aliases should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(46)]); + + remove_module_root(&root); +} + +#[test] +fn import_cycle_detected_across_lexically_distinct_aliases() { + let root = temp_module_root("vm_rustscript_cycle_alias_identity_test"); + write_source( + &root.join("a.rss"), + r#" + use self::b as b; + pub fn run() -> int { b::run() } + "#, + "a source", + ); + write_source( + &root.join("b.rss"), + r#" + use a as a; + pub fn run() -> int { a::run() } + "#, + "b source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::a as a; + a::run(); + "#, + "main source", + ); + + let error = match compile_source_file(&main_path) { + Ok(_) => panic!("lexically distinct cycle aliases should fail"), + Err(error) => error, + }; + assert!( + matches!(error, vm::SourcePathError::ImportCycle(_)), + "expected import cycle error across alias forms, got {error:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn duplicate_import_aliases_are_idempotent() { + let root = temp_module_root("vm_rustscript_duplicate_alias_import_test"); + write_source( + &root.join("sibling.rss"), + r#" + pub fn value() -> int { 29 } + "#, + "sibling source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::sibling as sib; + use self::sibling as sib; + sib::value(); + "#, + "main source", + ); + + let compiled = + compile_source_file(&main_path).expect("duplicate import aliases should be idempotent"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(29)]); + + remove_module_root(&root); +} + +#[test] +fn nested_module_host_namespace_import_stays_host() { + let root = temp_module_root("vm_rustscript_nested_host_namespace_test"); + write_source( + &root.join("nested.rss"), + r#" + use math; + pub fn run() -> float { math::sqrt(81) } + "#, + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::nested as nested; + nested::run(); + "#, + "main source", + ); + + let compiled = + compile_source_file(&main_path).expect("nested host namespace import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Float(9.0)]); + + remove_module_root(&root); +} diff --git a/tests/compiler/semantic_module_m12_tests.rs b/tests/compiler/semantic_module_m12_tests.rs new file mode 100644 index 00000000..55f58492 --- /dev/null +++ b/tests/compiler/semantic_module_m12_tests.rs @@ -0,0 +1,260 @@ +//! Milestones 1-2 of the semantic module system: structured `use` parsing with +//! spans/clauses, deterministic module identities, same-stem uniqueness, and +//! the dedicated host-namespace path. + +#[path = "../common/mod.rs"] +mod common; + +use std::path::{Path, PathBuf}; + +use common::*; +use vm::{ + ImportClause, ParserDialect, SharedParserOptions, UsePathSegment, parse_source_with_dialect, +}; + +/// Minimal dialect for driving the shared frontend parser from tests. +struct TestDialect; + +impl ParserDialect for TestDialect {} + +static TEST_DIALECT: TestDialect = TestDialect; + +fn rustscript_options() -> SharedParserOptions { + SharedParserOptions { + source_id: 0, + allow_implicit_externs: false, + allow_implicit_semicolons: false, + enforce_mutable_bindings: true, + import_scan_mode: false, + } +} + +fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + root.canonicalize().unwrap_or(root) +} + +fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source).unwrap_or_else(|err| panic!("{description} should write: {err}")); +} + +fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); +} + +#[test] +fn parser_records_structured_use_nodes_with_spans_and_clauses() { + let source = "use self::pkg::nested as nested;\n\ + use sibling::{value as v, other};\n\ + use super::shared;\n\ + use io;\n\ + 1;\n"; + let ir = parse_source_with_dialect(source, &TEST_DIALECT, rustscript_options()) + .expect("source should parse"); + + let decls = &ir.use_declarations; + assert_eq!( + decls.len(), + 4, + "every use directive becomes a structured node" + ); + + // self-qualified namespace import. + assert_eq!( + decls[0].path, + vec![ + UsePathSegment::Self_, + UsePathSegment::Ident("pkg".to_string()), + UsePathSegment::Ident("nested".to_string()), + ] + ); + assert!( + matches!(&decls[0].clause, ImportClause::Namespace(alias) if alias == "nested"), + "namespace alias clause expected" + ); + assert_eq!(decls[0].line, 1); + + // Named import list with an alias. + assert_eq!( + decls[1].path, + vec![UsePathSegment::Ident("sibling".to_string())] + ); + match &decls[1].clause { + ImportClause::Named(named) => { + assert_eq!(named.len(), 2); + assert_eq!(named[0].imported, "value"); + assert_eq!(named[0].local, "v"); + assert_eq!(named[1].imported, "other"); + assert_eq!(named[1].local, "other"); + } + other => panic!("expected named clause, got {other:?}"), + } + assert_eq!(decls[1].line, 2); + + // super-qualified and bare builtin imports. + assert_eq!( + decls[2].path, + vec![ + UsePathSegment::Super, + UsePathSegment::Ident("shared".to_string()) + ] + ); + assert!(matches!(decls[2].clause, ImportClause::AllPublic)); + assert!(matches!(decls[3].clause, ImportClause::AllPublic)); + assert_eq!(decls[3].line, 4); + + // Every span covers exactly its directive text in the source. + for decl in decls { + assert!( + decl.span.lo < decl.span.hi, + "span must cover the directive: {decl:?}" + ); + let text = &source[decl.span.lo..decl.span.hi]; + assert!( + text.starts_with("use ") && text.ends_with(';'), + "span must cover the full directive, got {text:?}" + ); + } +} + +#[test] +fn parser_import_scan_mode_tolerates_file_module_calls() { + // The source-loader discovery parse must accept calls to functions that + // only the later prelude/rewrite step resolves: unknown direct calls and + // namespace calls through multi-segment file-module paths. + let source = "use self::nested as nested;\n\ + nested::run();\n\ + imported_helper(1);\n\ + 1;\n"; + let options = SharedParserOptions { + allow_implicit_externs: true, + import_scan_mode: true, + ..rustscript_options() + }; + let ir = parse_source_with_dialect(source, &TEST_DIALECT, options) + .expect("scan mode must tolerate unresolved module calls"); + assert_eq!(ir.use_declarations.len(), 1); + assert_eq!( + ir.use_declarations[0].path, + vec![ + UsePathSegment::Self_, + UsePathSegment::Ident("nested".to_string()) + ] + ); +} + +#[test] +fn same_stem_modules_in_different_directories_compile_and_run() { + let root = temp_module_root("semantic_m12_same_stem"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + + let a_module = a_dir.join("util.rss"); + let b_module = b_dir.join("util.rss"); + write_source(&a_module, "pub fn alpha() { 11; }\n", "a/util source"); + write_source(&b_module, "pub fn beta() { 22; }\n", "b/util source"); + + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::alpha();\nbu::beta();\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("same-stem modules should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(11), Value::Int(22)], + "both same-stem modules must resolve independently" + ); + + remove_module_root(&root); +} + +#[test] +fn host_namespace_imports_keep_dedicated_resolution_path() { + struct ExistsOverride; + + impl HostFunction for ExistsOverride { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> vm::VmResult { + Ok(CallOutcome::Return(vec![Value::Bool(false)].into())) + } + } + + let source = "use io;\nio::exists(\"request_body\");\n"; + let compiled = compile_source(source).expect("host namespace import should compile"); + let mut vm = Vm::new(compiled.program); + vm.bind_function("io::exists", Box::new(ExistsOverride)); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Bool(false)]); +} + +#[test] +fn named_import_with_alias_through_self_resolves_structurally() { + let root = temp_module_root("semantic_m12_named_alias"); + let module_path = root.join("module.rss"); + write_source( + &module_path, + "pub fn echo(value) { value; }\n", + "module source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::module::{echo as e};\ne(42);\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("named alias import should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); + + remove_module_root(&root); +} + +#[test] +fn structured_import_syntax_rejects_crate_paths() { + let err = match compile_source("use crate::x;\n1;\n") { + Ok(_) => panic!("crate:: paths should be rejected"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("crate:: paths are not supported"), + "unexpected error: {message}" + ); +} + +#[test] +fn structured_import_syntax_rejects_import_keyword() { + let root = temp_module_root("semantic_m12_import_keyword"); + let main_path = root.join("main.rss"); + write_source(&main_path, "import \"./module.rss\";\n1;\n", "main source"); + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("legacy import syntax should be rejected"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("uses 'use', not 'import'"), + "unexpected error: {message}" + ); + remove_module_root(&root); +} diff --git a/tests/compiler/semantic_module_m3_tests.rs b/tests/compiler/semantic_module_m3_tests.rs new file mode 100644 index 00000000..80dae118 --- /dev/null +++ b/tests/compiler/semantic_module_m3_tests.rs @@ -0,0 +1,215 @@ +//! Milestone 3 of the semantic module system: declaration symbols owned by +//! modules, public export tables, imported-vs-local separation, duplicate +//! declaration diagnostics, same-named helpers across modules, and no +//! implicit transitive re-export — with bytecode behavior preserved. + +#[path = "../common/mod.rs"] +mod common; + +use std::path::{Path, PathBuf}; + +use common::*; + +fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + root.canonicalize().unwrap_or(root) +} + +fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source).unwrap_or_else(|err| panic!("{description} should write: {err}")); +} + +fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); +} + +/// `a/util` exports `alpha` and keeps `hidden` private; `b/util` exports +/// `beta`. Both modules declare a private helper named `helper`. +fn write_public_private_fixture(root: &Path) -> (PathBuf, PathBuf, PathBuf) { + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + + let a_module = a_dir.join("util.rss"); + write_source( + &a_module, + "pub fn alpha() { helper(); }\nfn helper() { 42; }\nfn hidden() { 7; }\n", + "a/util source", + ); + let b_module = b_dir.join("util.rss"); + write_source( + &b_module, + "pub fn beta() { helper(); }\nfn helper() { 42; }\n", + "b/util source", + ); + + let main_path = root.join("main.rss"); + (main_path, a_module, b_module) +} + +#[test] +fn same_named_helpers_across_modules_coexist() { + // Milestone 4 lifts the flat-merge limitation documented by milestone 3: + // same-named private helpers in independent modules now coexist, each + // resolved by its compiler-owned symbol. `alpha` and `beta` each call + // their own module's `helper`. + let root = temp_module_root("semantic_m3_same_helpers"); + let (main_path, _, _) = write_public_private_fixture(&root); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::alpha();\nbu::beta();\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("same-named helpers should compile"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(42), Value::Int(42)], + "each module's helper must resolve within its own module" + ); + + remove_module_root(&root); +} + +#[test] +fn public_functions_are_importable_private_functions_are_not() { + let root = temp_module_root("semantic_m3_visibility"); + let (main_path, _, _) = write_public_private_fixture(&root); + + // Public export: `alpha` resolves through the namespace import. + write_source( + &main_path, + "use a::util as au;\nau::alpha();\n", + "main source", + ); + let compiled = compile_source_file(&main_path).expect("public export should compile"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); + + // Private declaration: `hidden` is not in a/util's export table, so the + // call cannot resolve through the import. + write_source( + &main_path, + "use a::util as au;\nau::hidden();\n", + "main source", + ); + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("private functions must not be importable"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("hidden"), + "diagnostic should name the private function, got: {message}" + ); + + remove_module_root(&root); +} + +#[test] +fn transitive_imports_are_not_reexported() { + // a imports c and uses c::shared internally; the root imports only a. + // `shared` must stay out of a's export table: calling it from the root + // without a direct import is a diagnostic, not a silent re-export. + let root = temp_module_root("semantic_m3_no_reexport"); + let c_module = root.join("c.rss"); + write_source(&c_module, "pub fn shared() { 100; }\n", "c source"); + let a_module = root.join("a.rss"); + write_source( + &a_module, + "use self::c;\npub fn alpha() { c::shared(); }\n", + "a source", + ); + let main_path = root.join("main.rss"); + + write_source(&main_path, "use a;\nalpha();\nshared();\n", "main source"); + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("transitive imports must not be re-exported implicitly"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("shared"), + "diagnostic should name the non-reexported function, got: {message}" + ); + + // Positive control: importing c directly makes `shared` resolvable. + write_source( + &main_path, + "use a;\nuse c;\nalpha();\nshared();\n", + "main source", + ); + let compiled = compile_source_file(&main_path).expect("direct import should compile"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(100), Value::Int(100)]); + + remove_module_root(&root); +} + +#[test] +fn imported_name_clashing_with_local_declaration_is_a_diagnostic() { + // The import prelude declares the imported name, so declaring the same + // name locally in the importing module is a duplicate diagnostic instead + // of a silent shadow. + let root = temp_module_root("semantic_m3_import_clash"); + let a_dir = root.join("a"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + let a_module = a_dir.join("util.rss"); + write_source(&a_module, "pub fn helper() { 1; }\n", "a/util source"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util;\nfn helper() { 2; }\nhelper();\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("imported name clashing with a local declaration must fail"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("conflicts with a local declaration") + || message.contains("duplicate function 'helper'"), + "unexpected diagnostic: {message}" + ); + + remove_module_root(&root); +} + +#[test] +fn duplicate_local_declaration_in_a_module_is_a_diagnostic() { + let root = temp_module_root("semantic_m3_dup_local"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "fn dup() { 1; }\nfn dup() { 2; }\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("duplicate local declarations must fail"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("duplicate function 'dup'"), + "unexpected diagnostic: {message}" + ); + + remove_module_root(&root); +} diff --git a/tests/compiler/semantic_module_m4_tests.rs b/tests/compiler/semantic_module_m4_tests.rs new file mode 100644 index 00000000..5a8e43d2 --- /dev/null +++ b/tests/compiler/semantic_module_m4_tests.rs @@ -0,0 +1,343 @@ +//! Milestone 4 of the semantic module system: calls resolve by compiler-owned +//! `SymbolId` before unit merge, the flat linker keys module functions by +//! symbol instead of by source name, and names are deterministically mangled +//! only at the flat bytecode boundary. Same-named declarations in independent +//! modules coexist; local bindings are scoped by full module identity instead +//! of a bare file stem. + +#[path = "../common/mod.rs"] +mod common; + +use std::path::{Path, PathBuf}; + +use common::*; + +fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + root.canonicalize().unwrap_or(root) +} + +fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source).unwrap_or_else(|err| panic!("{description} should write: {err}")); +} + +fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); +} + +/// `a/util` and `b/util` both export a public `run` (different bodies) and +/// each keeps a private helper named `helper` that its own `run` calls. +fn write_same_export_fixture(root: &Path) -> PathBuf { + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + + let a_module = a_dir.join("util.rss"); + write_source( + &a_module, + "pub fn run() { helper(); }\nfn helper() { 11; }\n", + "a/util source", + ); + let b_module = b_dir.join("util.rss"); + write_source( + &b_module, + "pub fn run() { helper(); }\nfn helper() { 22; }\n", + "b/util source", + ); + + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::run();\nbu::run();\n", + "main source", + ); + main_path +} + +#[test] +fn same_exported_function_name_in_two_namespaces_calls_separately() { + let root = temp_module_root("semantic_m4_same_export"); + let main_path = write_same_export_fixture(&root); + + let compiled = compile_source_file(&main_path).expect("same-named exports should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(11), Value::Int(22)], + "au::run and bu::run must resolve to their own module's export" + ); + + // The flat exported-callable table keeps both exports addressable: the + // first export keeps its bare name and the collision is deterministically + // mangled with the module identity. + let exported_names = vm + .program() + .exported_callables + .iter() + .map(|exported| exported.name.as_str()) + .collect::>(); + assert!( + exported_names.contains(&"run"), + "one export keeps the bare name: {exported_names:?}" + ); + assert_eq!( + exported_names + .iter() + .filter(|name| name.starts_with("run__m")) + .count(), + 1, + "the colliding export is deterministically mangled: {exported_names:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn same_named_private_helpers_are_resolved_within_their_own_module() { + let root = temp_module_root("semantic_m4_private_helpers"); + let main_path = write_same_export_fixture(&root); + + let compiled = compile_source_file(&main_path).expect("same-named helpers should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(11), Value::Int(22)], + "each module's private helper must be the one its own run calls" + ); + + remove_module_root(&root); +} + +#[test] +fn named_import_aliases_resolve_to_distinct_symbols() { + let root = temp_module_root("semantic_m4_named_aliases"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn emit(value) { value * 2; }\n", + "a/util source", + ); + write_source( + &b_dir.join("util.rss"), + "pub fn emit(value) { value * 3; }\n", + "b/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util::{emit as twice};\nuse b::util::{emit as thrice};\ntwice(4);\nthrice(4);\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("named alias imports should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(8), Value::Int(12)], + "each alias must call its own module's emit" + ); + + remove_module_root(&root); +} + +#[test] +fn local_functions_resolve_within_their_own_module() { + // `run` (pub) calls `local` (private) in both modules; the local calls + // must stay inside their declaring module even though both modules define + // same-named functions. + let root = temp_module_root("semantic_m4_local_functions"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn run() { local(); }\nfn local() { 1; }\n", + "a/util source", + ); + write_source( + &b_dir.join("util.rss"), + "pub fn run() { local(); }\nfn local() { 2; }\n", + "b/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::run();\nbu::run();\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("local functions should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!( + vm.stack(), + &[Value::Int(1), Value::Int(2)], + "each module's local function must resolve within its own module" + ); + + remove_module_root(&root); +} + +#[test] +fn ambiguous_direct_call_to_same_name_from_two_modules_is_a_diagnostic() { + // Both modules export `helper` and the root imports both without aliases: + // a bare `helper()` call cannot name a single symbol. The legacy pipeline + // reported a flat merge error; milestone 4 reports the ambiguity and asks + // for a namespace-qualified or named-import call. + let root = temp_module_root("semantic_m4_ambiguous_direct"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn helper() { 1; }\n", + "a/util source", + ); + write_source( + &b_dir.join("util.rss"), + "pub fn helper() { 2; }\n", + "b/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util;\nuse b::util;\nhelper();\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("ambiguous direct calls must be rejected"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("ambiguous"), + "diagnostic should report the ambiguity, got: {message}" + ); + assert!( + message.contains("helper"), + "diagnostic should name the ambiguous function, got: {message}" + ); + + // The same fixture compiles once the calls are namespace-qualified. + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::helper();\nbu::helper();\n", + "main source", + ); + let compiled = compile_source_file(&main_path).expect("qualified calls should compile"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(1), Value::Int(2)]); + + remove_module_root(&root); +} + +#[test] +fn internal_lowering_is_deterministic_across_repeated_discovery() { + // The same fixture compiled twice must produce byte-identical bytecode: + // module ids, symbol ids, stub names, flat indices, and mangled names are + // all assigned deterministically from discovery order. + let root = temp_module_root("semantic_m4_deterministic"); + let main_path = write_same_export_fixture(&root); + + let compile_bytes = || { + let compiled = compile_source_file(&main_path).expect("compile should succeed"); + vm::encode_program(&compiled.program).expect("program should encode") + }; + + let first = compile_bytes(); + let second = compile_bytes(); + assert_eq!( + first, second, + "internal lowering must be deterministic across discovery passes" + ); + + remove_module_root(&root); +} + +#[test] +fn same_stem_modules_do_not_collide_local_binding_scope_names() { + // Two same-stem modules (`a/util`, `b/util`) both declare a local `x`. + // Milestone 4 scopes non-root locals by full module identity (never a + // bare file stem), so both survive the flat boundary with distinct names. + let root = temp_module_root("semantic_m4_no_basename_scope"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn alpha() { let x = 7; x; }\n", + "a/util source", + ); + write_source( + &b_dir.join("util.rss"), + "pub fn beta() { let x = 8; x; }\n", + "b/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::alpha();\nbu::beta();\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("same-stem modules should compile"); + let debug = compiled + .program + .debug + .as_ref() + .expect("compiled program should include debug info"); + let x_names = debug + .locals + .iter() + .filter(|local| local.name.ends_with("::x")) + .map(|local| local.name.as_str()) + .collect::>(); + assert_eq!( + x_names.len(), + 2, + "both modules' x locals must survive the flat boundary: {x_names:?}" + ); + assert!( + x_names.iter().all(|name| *name != "x"), + "non-root locals must be scoped by module identity, got: {x_names:?}" + ); + assert!( + x_names.iter().all(|name| name.contains("__m")), + "scope identity must encode the compiler-owned module id: {x_names:?}" + ); + assert!( + x_names[0] != x_names[1], + "same-stem modules must not share a scope identity: {x_names:?}" + ); + + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(7), Value::Int(8)]); + + remove_module_root(&root); +} diff --git a/tests/compiler/semantic_module_m5_tests.rs b/tests/compiler/semantic_module_m5_tests.rs new file mode 100644 index 00000000..023f3d7c --- /dev/null +++ b/tests/compiler/semantic_module_m5_tests.rs @@ -0,0 +1,457 @@ +//! Milestone 5 of the semantic module system: source-owned spans and +//! diagnostics through merge. +//! +//! Every span produced during load/parse/typing/merge references the semantic +//! module graph's `SourceId` space, and the compilation-wide `SourceMap` +//! travels with module-compile errors (`SourcePathError::SourceWithMap`), so +//! rendered diagnostics always read from the owning source. Merging units can +//! never reinterpret one module's offsets or lines against another file. + +#[path = "../common/mod.rs"] +mod common; + +use std::path::{Path, PathBuf}; + +use common::*; + +fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + root.canonicalize().unwrap_or(root) +} + +fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source).unwrap_or_else(|err| panic!("{description} should write: {err}")); +} + +fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); +} + +/// Render a module-compile error against the compilation-wide source map it +/// carries (milestone 5): the rendered diagnostic shows the owning file name, +/// line, and code frame. +fn render_path_error(err: &vm::SourcePathError) -> String { + match err { + vm::SourcePathError::SourceWithMap { error, sources } => match error { + vm::SourceError::Parse(parse) => vm::render_source_error(sources, parse, false), + vm::SourceError::Compile(compile) => vm::render_compile_error(sources, compile, false), + }, + other => vm::render_source_path_error(Path::new(""), other, false), + } +} + +#[test] +fn root_parse_error_renders_root_path_and_frame() { + let root = temp_module_root("semantic_m5_root_parse"); + let main_path = root.join("main.rss"); + write_source(&main_path, "fn run() {\nlet x = ;\n}\n", "main source"); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("invalid root source should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains(&main_path.display().to_string()), + "root diagnostic must name the root path, got:\n{rendered}" + ); + assert!( + rendered.contains("let x = ;"), + "root diagnostic must show the root code frame, got:\n{rendered}" + ); + assert!( + rendered.contains("--> "), + "root diagnostic must include a source frame, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn nested_module_parse_error_renders_from_owning_source() { + let root = temp_module_root("semantic_m5_nested_parse"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::nested as nested;\nnested::run();\n", + "main source", + ); + let nested_path = root.join("nested.rss"); + write_source(&nested_path, "pub fn run( {\n", "nested source"); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("malformed nested module should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains(&nested_path.display().to_string()), + "nested parse diagnostic must name the nested path, got:\n{rendered}" + ); + assert!( + rendered.contains("pub fn run( {"), + "nested parse diagnostic must show the nested code frame, got:\n{rendered}" + ); + assert!( + !rendered.contains("nested::run();"), + "nested parse diagnostic must not show the root frame, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn nested_module_typing_error_renders_from_owning_source() { + let root = temp_module_root("semantic_m5_nested_typing"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::broken as broken;\nbroken::run();\n", + "main source", + ); + let broken_path = root.join("broken.rss"); + write_source( + &broken_path, + "pub fn run() {\nlet cond = 1 == 1;\nlet value = if cond => {\n 1\n} else => {\n \"x\"\n};\nvalue\n}\n", + "broken source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("typed mismatched nested module should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains("compile error"), + "typing diagnostic must render as a compile error, got:\n{rendered}" + ); + assert!( + rendered.contains(&broken_path.display().to_string()), + "typing diagnostic must name the nested module, got:\n{rendered}" + ); + assert!( + rendered.contains("let value = if cond => {"), + "typing diagnostic must show the nested module's code frame, got:\n{rendered}" + ); + assert!( + rendered.contains("int vs string") || rendered.contains("incompatible"), + "typing diagnostic must keep its detail message, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn duplicate_function_error_renders_from_owning_source() { + let root = temp_module_root("semantic_m5_duplicate"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::dupmod as d;\nd::run();\n", + "main source", + ); + let dup_path = root.join("dupmod.rss"); + write_source( + &dup_path, + "fn run() { 1; }\nfn run() { 2; }\n", + "duplicate source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("duplicate declarations should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains("duplicate"), + "duplicate diagnostic must say 'duplicate', got:\n{rendered}" + ); + assert!( + rendered.contains(&dup_path.display().to_string()), + "duplicate diagnostic must name the owning module, got:\n{rendered}" + ); + assert!( + rendered.contains("fn run() { 2; }"), + "duplicate diagnostic must point at the redeclaration's frame, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn private_export_visibility_failure_renders_from_importing_source() { + // `hidden` is private in the module; the named import in main must fail + // and the diagnostic must render from main's own `use` line. + let root = temp_module_root("semantic_m5_visibility"); + let module_path = root.join("module.rss"); + write_source(&module_path, "fn hidden() { 1; }\n", "module source"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use module::{hidden};\nhidden();\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("importing a private function should fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("has no public function 'hidden'"), + "visibility diagnostic must name the missing public function, got: {err}" + ); + let rendered = render_path_error(&err); + assert!( + rendered.contains(&main_path.display().to_string()), + "visibility diagnostic must render from the importing module, got:\n{rendered}" + ); + assert!( + rendered.contains("use module::{hidden};"), + "visibility diagnostic must show the importing module's use line, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn unresolved_module_call_renders_from_owning_source() { + // `nested` imports `sibling` privately; main cannot call `leaf` through + // `nested` (no implicit transitive re-export). The unresolved call must + // render from main's own source. + let root = temp_module_root("semantic_m5_unresolved_call"); + let sibling_path = root.join("sibling.rss"); + write_source( + &sibling_path, + "pub fn leaf() -> int { 19 }\n", + "sibling source", + ); + let nested_path = root.join("nested.rss"); + write_source( + &nested_path, + "use self::sibling as sibling;\npub fn run() -> int { sibling::leaf() }\n", + "nested source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::nested as nested;\nnested::leaf();\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("calling a non re-exported function should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains(&main_path.display().to_string()), + "unresolved call diagnostic must name the calling module, got:\n{rendered}" + ); + assert!( + rendered.contains("nested::leaf();"), + "unresolved call diagnostic must show the call site frame, got:\n{rendered}" + ); + assert!( + !rendered.contains("pub fn run() -> int"), + "unresolved call diagnostic must not show the nested module's frame, got:\n{rendered}" + ); + + remove_module_root(&root); +} + +#[test] +fn same_line_number_in_different_modules_renders_each_owning_source() { + // Both modules fail strict typing on line 2, but each compilation must + // render its own file's line 2 text. + let root = temp_module_root("semantic_m5_same_line"); + let a_module = root.join("a.rss"); + write_source( + &a_module, + "pub fn run() {\nlet a: unknown = 1;\na\n}\n", + "a module source", + ); + let b_module = root.join("b.rss"); + write_source( + &b_module, + "pub fn run() {\nlet b: unknown = 2;\nb\n}\n", + "b module source", + ); + + let compile_entry = |entry_name: &str, module: &Path| { + let entry = root.join(entry_name); + let module_name = module + .file_stem() + .and_then(|stem| stem.to_str()) + .expect("stem"); + write_source( + &entry, + &format!("use self::{module_name} as m;\nm::run();\n"), + "entry source", + ); + compile_source_file(&entry) + }; + + let err_a = match compile_entry("main_a.rss", &a_module) { + Ok(_) => panic!("a module should fail strict typing"), + Err(err) => err, + }; + let rendered_a = render_path_error(&err_a); + assert!( + rendered_a.contains(&a_module.display().to_string()), + "a diagnostic must name a.rss, got:\n{rendered_a}" + ); + assert!( + rendered_a.contains("let a: unknown = 1;"), + "a diagnostic must show a.rss line 2, got:\n{rendered_a}" + ); + + let err_b = match compile_entry("main_b.rss", &b_module) { + Ok(_) => panic!("b module should fail strict typing"), + Err(err) => err, + }; + let rendered_b = render_path_error(&err_b); + assert!( + rendered_b.contains(&b_module.display().to_string()), + "b diagnostic must name b.rss, got:\n{rendered_b}" + ); + assert!( + rendered_b.contains("let b: unknown = 2;"), + "b diagnostic must show b.rss line 2, got:\n{rendered_b}" + ); + assert!( + !rendered_b.contains("let a: unknown = 1;"), + "b diagnostic must never show a.rss's line 2 text, got:\n{rendered_b}" + ); + + remove_module_root(&root); +} + +#[test] +fn in_memory_override_and_disk_modules_render_their_own_sources() { + // Disk module a/util and in-memory override b/util both fail strict + // typing on line 2 of their own text. The disk failure must render the + // file that exists on disk; the override failure must render the override + // text even though b/util.rss does not exist on disk. + let root = temp_module_root("semantic_m5_override_ownership"); + let a_dir = root.join("a"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + let a_module = a_dir.join("util.rss"); + write_source( + &a_module, + "pub fn alpha() {\nlet a: unknown = 1;\na\n}\n", + "a/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::alpha();\nbu::beta();\n", + "main source", + ); + + // Phase 1: a/util (disk) is the first failing module; the diagnostic must + // render the disk text of a/util.rss. + let options_phase1 = vm::CompileSourceFileOptions::new() + .with_module_override_source("b/util.rss", "pub fn beta() {\nlet b: unknown = 2;\nb\n}\n"); + let err_phase1 = match compile_source_file_with_options(&main_path, options_phase1) { + Ok(_) => panic!("disk module should fail strict typing"), + Err(err) => err, + }; + let rendered_phase1 = render_path_error(&err_phase1); + assert!( + rendered_phase1.contains(&a_module.display().to_string()), + "disk diagnostic must name a/util.rss, got:\n{rendered_phase1}" + ); + assert!( + rendered_phase1.contains("let a: unknown = 1;"), + "disk diagnostic must show the disk code frame, got:\n{rendered_phase1}" + ); + + // Phase 2: a/util (disk) is valid; the in-memory override for b/util is + // the failing module. The diagnostic must render the override text even + // though no b/util.rss exists on disk. + write_source(&a_module, "pub fn alpha() { 1; }\n", "a/util fixed source"); + let options_phase2 = vm::CompileSourceFileOptions::new() + .with_module_override_source("b/util.rss", "pub fn beta() {\nlet b: unknown = 2;\nb\n}\n"); + let err_phase2 = match compile_source_file_with_options(&main_path, options_phase2) { + Ok(_) => panic!("override module should fail strict typing"), + Err(err) => err, + }; + let rendered_phase2 = render_path_error(&err_phase2); + assert!( + rendered_phase2.contains("__pd_vm_inmemory__/b/util.rss") + || rendered_phase2.contains(&root.join("b/util.rss").display().to_string()), + "override diagnostic must name the virtual b/util identity, got:\n{rendered_phase2}" + ); + assert!( + rendered_phase2.contains("let b: unknown = 2;"), + "override diagnostic must show the override text frame, got:\n{rendered_phase2}" + ); + assert!( + !rendered_phase2.contains("let a: unknown = 1;"), + "override diagnostic must not show a/util's frame, got:\n{rendered_phase2}" + ); + + remove_module_root(&root); +} + +#[test] +fn in_memory_root_error_renders_virtual_path_and_frame() { + // `compile_source_with_flavor_and_options` compiles a virtual root; its + // parse error must render the virtual path and the in-memory frame. + let source = "use self::nested as nested;\nnested::run();\n"; + let options = vm::CompileSourceFileOptions::new() + .with_module_override_source("nested.rss", "pub fn run( {\n"); + + let err = match vm::compile_source_with_flavor_and_options( + source, + vm::SourceFlavor::RustScript, + options, + ) { + Ok(_) => panic!("malformed override module should fail"), + Err(err) => err, + }; + let rendered = render_path_error(&err); + assert!( + rendered.contains("__pd_vm_inmemory__/nested.rss"), + "in-memory diagnostic must name the virtual nested path, got:\n{rendered}" + ); + assert!( + rendered.contains("pub fn run( {"), + "in-memory diagnostic must show the override code frame, got:\n{rendered}" + ); + + // The same fixture through the at-path entry point renders the disk-path + // identity of the override instead. + let root = temp_module_root("semantic_m5_virtual_at_path"); + let main_path = root.join("main.rss"); + let at_path_err = match vm::compile_source_at_path_with_flavor_and_options( + &main_path, + source, + vm::SourceFlavor::RustScript, + vm::CompileSourceFileOptions::new() + .with_module_override_source("nested.rss", "pub fn run( {\n"), + ) { + Ok(_) => panic!("malformed override module should fail"), + Err(err) => err, + }; + let rendered_at_path = render_path_error(&at_path_err); + assert!( + rendered_at_path.contains(&root.join("nested.rss").display().to_string()), + "at-path diagnostic must name the disk-path override identity, got:\n{rendered_at_path}" + ); + assert!( + rendered_at_path.contains("pub fn run( {"), + "at-path diagnostic must show the override code frame, got:\n{rendered_at_path}" + ); + + remove_module_root(&root); +} diff --git a/tests/compiler/semantic_module_m6_tests.rs b/tests/compiler/semantic_module_m6_tests.rs new file mode 100644 index 00000000..df9679c5 --- /dev/null +++ b/tests/compiler/semantic_module_m6_tests.rs @@ -0,0 +1,399 @@ +//! Milestone 6/7 verification: the semantic module pipeline is the sole +//! file-module path. +//! +//! These tests exercise the end-to-end module behavior that the removed +//! textual rewrite/prelude machinery used to provide: wildcard imports, +//! function values of imported functions, generic calls through every import +//! form, single-segment host-form namespace calls, deterministic output, and +//! import-order independence of behavior. + +#[path = "../common/mod.rs"] +mod common; + +use std::path::{Path, PathBuf}; + +use common::*; + +fn temp_module_root(prefix: &str) -> PathBuf { + let unique = format!( + "{prefix}_{}_{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock should be valid") + .as_nanos() + ); + let root = std::env::temp_dir().join(unique); + std::fs::create_dir_all(&root).expect("temp module root should be created"); + root.canonicalize().unwrap_or(root) +} + +fn write_source(path: &Path, source: &str, description: &str) { + std::fs::write(path, source).unwrap_or_else(|err| panic!("{description} should write: {err}")); +} + +fn remove_module_root(root: &Path) { + let _ = std::fs::remove_dir_all(root); +} + +#[test] +fn wildcard_import_exposes_all_public_exports_directly_and_by_namespace() { + let root = temp_module_root("semantic_m6_wildcard"); + write_source( + &root.join("util.rss"), + "pub fn value() -> int { 5 }\npub fn double(x) { x * 2; }\nfn private_helper() { 99; }\n", + "util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::util::*;\nlet direct = value();\nlet ns = util::double(direct);\nns;\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("wildcard import should compile"); + assert!( + compiled.functions.is_empty(), + "wildcard imports must not produce host imports" + ); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(10)]); + + remove_module_root(&root); +} + +#[test] +fn wildcard_import_does_not_expose_private_helpers() { + let root = temp_module_root("semantic_m6_wildcard_private"); + write_source( + &root.join("util.rss"), + "pub fn value() -> int { 5 }\nfn private_helper() { 99; }\n", + "util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::util::*;\nprivate_helper();\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("wildcard import must not expose private functions"), + Err(err) => err, + }; + assert!( + err.to_string() + .contains("unknown function 'private_helper'"), + "unexpected diagnostic: {err}" + ); + + remove_module_root(&root); +} + +#[test] +fn imported_function_values_resolve_to_module_symbols() { + let root = temp_module_root("semantic_m6_function_values"); + write_source( + &root.join("util.rss"), + "pub fn add_one(x) { x + 1; }\n", + "util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::util;\nlet f = add_one;\nf(41);\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("function value import should compile"); + assert!( + compiled.functions.is_empty(), + "imported function values must not produce host imports" + ); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); + + remove_module_root(&root); +} + +#[test] +fn generic_calls_work_through_named_namespace_and_alias_import_forms() { + let root = temp_module_root("semantic_m6_generic_forms"); + let a_dir = root.join("a"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn wrap(value: T) { let copied = value; [copied]; }\n", + "a/util source", + ); + write_source( + &root.join("helpers.rss"), + "pub fn wrap(value: T) { let copied = value; [copied]; }\n", + "helpers source", + ); + + // Named import (direct), all-public namespace call, and aliased + // namespace call all carry explicit type arguments through to the + // exported type parameters. + let main_path = root.join("main.rss"); + write_source( + &main_path, + r#" + use self::helpers::{wrap as direct_wrap}; + use self::helpers; + use a::util as au; + + let named = direct_wrap::(1); + let namespace_value = helpers::wrap::(2); + let aliased = au::wrap::(3); + named.length + namespace_value.length + aliased.length; + "#, + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("generic import forms should compile"); + assert!( + compiled.functions.is_empty(), + "generic imported calls must not produce host imports" + ); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(3)]); + + remove_module_root(&root); +} + +#[test] +fn single_segment_module_import_namespace_calls_stay_module_calls() { + // `use module; module::fn()` parses as a host-form call (the parser + // cannot know `module` is a file module); the loader must fix it up to a + // module call instead of emitting a host import. + let root = temp_module_root("semantic_m6_single_segment_ns"); + write_source( + &root.join("module.rss"), + "pub fn public_add(x) { x + 1; }\n", + "module source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use module;\nmodule::public_add(41);\n", + "main source", + ); + + let compiled = + compile_source_file(&main_path).expect("single-segment namespace call should compile"); + assert!( + compiled.functions.is_empty(), + "file-module namespace calls must not become host imports" + ); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); + + remove_module_root(&root); +} + +#[test] +fn single_segment_named_import_missing_member_stays_unknown_function() { + let root = temp_module_root("semantic_m6_single_segment_named"); + write_source( + &root.join("module.rss"), + "pub fn add_one(x) { x + 1; }\n", + "module source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use module::{add_one};\nadd_two(40);\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("unlisted member must fail"), + Err(err) => err, + }; + assert!( + err.to_string().contains("unknown function 'add_two'"), + "unexpected diagnostic: {err}" + ); + + remove_module_root(&root); +} + +#[test] +fn same_exported_name_from_two_modules_resolves_per_namespace() { + // Two modules exporting the same name, imported through aliases; the + // final flat boundary keeps both addressable with deterministic + // module-identity mangling for the colliding name. + let root = temp_module_root("semantic_m6_same_exports"); + let a_dir = root.join("a"); + let b_dir = root.join("b"); + std::fs::create_dir_all(&a_dir).expect("a dir should be created"); + std::fs::create_dir_all(&b_dir).expect("b dir should be created"); + write_source( + &a_dir.join("util.rss"), + "pub fn helper() { 1; }\n", + "a/util source", + ); + write_source( + &b_dir.join("util.rss"), + "pub fn helper() { 2; }\n", + "b/util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use a::util as au;\nuse b::util as bu;\nau::helper();\nbu::helper();\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("same-name exports should compile"); + let mut vm = Vm::new(compiled.program); + let status = vm.run().expect("vm should run"); + assert_eq!(status, VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(1), Value::Int(2)]); + + remove_module_root(&root); +} + +#[test] +fn compiled_output_is_deterministic_across_repeated_compilations() { + let root = temp_module_root("semantic_m6_deterministic_bytes"); + write_source( + &root.join("util.rss"), + "pub fn helper() { 1; }\npub fn other() { helper() + 1; }\n", + "util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::util;\nhelper();\nother();\n", + "main source", + ); + + let compile = || { + let compiled = compile_source_file(&main_path).expect("compile should succeed"); + let function_names = compiled + .functions + .iter() + .map(|func| func.name.clone()) + .collect::>(); + (compiled.program.code.clone(), function_names) + }; + let (first_instructions, first_names) = compile(); + let (second_instructions, second_names) = compile(); + + assert_eq!( + first_names, second_names, + "function table must be identical across compilations" + ); + assert_eq!( + first_instructions, second_instructions, + "bytecode must be identical across compilations" + ); + + remove_module_root(&root); +} + +#[test] +fn import_order_swap_produces_identical_behavior() { + let root = temp_module_root("semantic_m6_import_order"); + write_source( + &root.join("a.rss"), + "pub fn value() -> int { 3 }\n", + "a source", + ); + write_source( + &root.join("b.rss"), + "pub fn value() -> int { 4 }\n", + "b source", + ); + let main_ab = root.join("main_ab.rss"); + write_source( + &main_ab, + "use self::a as a;\nuse self::b as b;\na::value();\nb::value();\n", + "main ab source", + ); + let main_ba = root.join("main_ba.rss"); + write_source( + &main_ba, + "use self::b as b;\nuse self::a as a;\nb::value();\na::value();\n", + "main ba source", + ); + + let run = |path: &Path| -> Vec { + let compiled = compile_source_file(path).expect("compile should succeed"); + let mut vm = Vm::new(compiled.program); + assert_eq!(vm.run().expect("vm should run"), VmStatus::Halted); + vm.stack().to_vec() + }; + assert_eq!(run(&main_ab), vec![Value::Int(3), Value::Int(4)]); + assert_eq!( + run(&main_ba), + vec![Value::Int(4), Value::Int(3)], + "import order must not change which module each call resolves to" + ); + + remove_module_root(&root); +} + +#[test] +fn host_namespace_imports_stay_on_the_host_path_without_rewriting() { + // A virtual host namespace import must compile to host imports even + // though its single-segment form parses like a file-module candidate. + let root = temp_module_root("semantic_m6_host_path"); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use myhost;\nmyhost::do_thing(81);\n", + "main source", + ); + + let compiled = compile_source_file(&main_path).expect("host namespace import should compile"); + let host_names = compiled + .functions + .iter() + .map(|func| func.name.as_str()) + .collect::>(); + assert!( + host_names.contains(&"myhost::do_thing"), + "host namespace call must remain a host import: {host_names:?}" + ); + + remove_module_root(&root); +} + +#[test] +fn namespace_member_arity_mismatch_is_a_diagnostic() { + let root = temp_module_root("semantic_m6_arity_mismatch"); + write_source( + &root.join("util.rss"), + "pub fn add(x, y) { x + y; }\n", + "util source", + ); + let main_path = root.join("main.rss"); + write_source( + &main_path, + "use self::util as u;\nu::add(1);\n", + "main source", + ); + + let err = match compile_source_file(&main_path) { + Ok(_) => panic!("arity mismatch must fail"), + Err(err) => err, + }; + let message = err.to_string(); + assert!( + message.contains("function 'u::add' expects 2 arguments"), + "unexpected diagnostic: {message}" + ); + + remove_module_root(&root); +} diff --git a/tests/compiler_tests.rs b/tests/compiler_tests.rs index b95d10fa..11328072 100644 --- a/tests/compiler_tests.rs +++ b/tests/compiler_tests.rs @@ -26,3 +26,23 @@ mod type_inference_tests; #[cfg(feature = "runtime")] #[path = "compiler/whitespace_resilience_tests.rs"] mod whitespace_resilience_tests; + +#[cfg(feature = "runtime")] +#[path = "compiler/semantic_module_m12_tests.rs"] +mod semantic_module_m12_tests; + +#[cfg(feature = "runtime")] +#[path = "compiler/semantic_module_m3_tests.rs"] +mod semantic_module_m3_tests; + +#[cfg(feature = "runtime")] +#[path = "compiler/semantic_module_m4_tests.rs"] +mod semantic_module_m4_tests; + +#[cfg(feature = "runtime")] +#[path = "compiler/semantic_module_m5_tests.rs"] +mod semantic_module_m5_tests; + +#[cfg(feature = "runtime")] +#[path = "compiler/semantic_module_m6_tests.rs"] +mod semantic_module_m6_tests; diff --git a/tests/host_binding_generation_tests.rs b/tests/host_binding_generation_tests.rs index e465a9dc..3e44c1af 100644 --- a/tests/host_binding_generation_tests.rs +++ b/tests/host_binding_generation_tests.rs @@ -201,6 +201,44 @@ fn runtime_sleep_default_bindings_remain_inside_jit_loop_traces() { assert_runtime_sleep_loop_uses_native_host_call(true); } +#[test] +fn restricted_capabilities_disable_trace_jit_for_host_imports_and_builtins() { + for source in [ + r#" + use runtime; + let mut i = 0; + while i < 4 { + let _ = runtime::sleep(0); + i = i + 1; + } + i; + "#, + r#" + use re; + let mut i = 0; + while i < 4 { + let _ = re::match("a", "a"); + i = i + 1; + } + i; + "#, + ] { + let compiled = compile_source(source).expect("restricted loop should compile"); + let mut vm = Vm::new(compiled.program); + vm.set_jit_config(JitConfig { + enabled: native_jit_supported(), + hot_loop_threshold: 1, + max_trace_len: 512, + }); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut vm) + .expect("restricted registry should bind program imports"); + + assert!(matches!(vm.run(), Err(vm::VmError::UnboundImport(_)))); + assert_eq!(vm.jit_native_exec_count(), 0); + } +} + #[test] fn runtime_exit_still_halts_for_direct_and_cached_default_bindings() { for bind_cached_registry in [false, true] { diff --git a/tests/runtime_context_tests.rs b/tests/runtime_context_tests.rs new file mode 100644 index 00000000..31dfb06b --- /dev/null +++ b/tests/runtime_context_tests.rs @@ -0,0 +1,401 @@ +mod vm { + pub use ::vm::{Value, VmError, VmResult}; +} + +#[allow(dead_code)] +#[path = "../src/builtins/runtime/cancellation.rs"] +mod cancellation; +#[allow(dead_code)] +#[path = "../src/builtins/runtime/context.rs"] +mod context; +#[allow(dead_code)] +#[path = "../src/builtins/runtime/error.rs"] +mod error; +#[allow(dead_code)] +#[path = "../src/builtins/runtime/event.rs"] +mod event; +#[allow(dead_code)] +#[path = "../src/builtins/runtime/resource.rs"] +mod resource; + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier, Mutex}; +use std::time::{Duration, Instant}; + +use cancellation::{CancellationReason, OperationRegistry, OperationStatus}; +use context::{RuntimeContext, RuntimeContextConfig}; +use error::RuntimeErrorCode; +use event::{EventLimits, EventPayload}; +use resource::{CloseStatus, ResourceArena, ResourceHandle, ResourceTypeId}; +use vm::Value; + +#[test] +fn runtime_input_is_run_scoped_and_missing_input_is_typed() { + let mut context = RuntimeContext::default(); + let missing = context.input().expect_err("unset input should be rejected"); + assert_eq!(missing.code(), RuntimeErrorCode::InputUnavailable); + + let input = Value::map(vec![(Value::string("kind"), Value::string("message"))]); + context + .set_input(input.clone()) + .expect("input should be accepted"); + assert_eq!(context.input().expect("input should be available"), input); +} + +#[test] +fn runtime_emit_validates_payload_before_calling_the_sink() { + let mut context = RuntimeContext::with_config(RuntimeContextConfig::new( + EventLimits::new(8, 4).expect("test limits should be valid"), + )) + .expect("context should be constructible"); + let seen = Arc::new(Mutex::new(Vec::::new())); + let seen_by_sink = Arc::clone(&seen); + context + .set_event_sink(move |payload: EventPayload| { + seen_by_sink + .lock() + .expect("event sink lock should not be poisoned") + .push(payload.into_value()); + Ok(()) + }) + .expect("event sink should be installed"); + + context + .emit(Value::string("ok")) + .expect("bounded event should reach the sink"); + assert_eq!(seen.lock().expect("event sink lock").len(), 1); + + let too_large = context + .emit(Value::string("payload-too-large")) + .expect_err("oversized event should be rejected"); + assert_eq!(too_large.code(), RuntimeErrorCode::EventPayloadTooLarge); + assert_eq!(seen.lock().expect("event sink lock").len(), 1); +} + +#[test] +fn runtime_emit_reports_missing_sink_without_dropping_the_value_contract() { + let mut context = RuntimeContext::default(); + let error = context + .emit(Value::Bool(true)) + .expect_err("emit without a sink should fail"); + assert_eq!(error.code(), RuntimeErrorCode::EventSinkUnavailable); +} + +#[test] +fn resource_handles_are_opaque_bounded_typed_and_cleanup_is_idempotent() { + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let count_for_cleanup = Arc::clone(&cleanup_count); + let mut arena = ResourceArena::with_limit(1).expect("resource limit should be valid"); + let handle = arena + .insert_with_cleanup(ResourceTypeId::IO_FILE, 7_u32, move |resource, reason| { + assert_eq!(resource, 7); + assert_eq!(reason, CancellationReason::ResourceClosed); + count_for_cleanup.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .expect("first resource should be allocated"); + + assert_eq!( + arena + .get::(handle, ResourceTypeId::IO_FILE) + .expect("handle should resolve"), + &7 + ); + assert_eq!( + ResourceHandle::from_value(&handle.as_value()).expect("VM value should decode"), + handle + ); + let Value::Int(encoded) = handle.as_value() else { + unreachable!("resource handle should encode as an integer"); + }; + let forged_generation = ResourceHandle::from_value(&Value::Int(encoded + (1 << 8))) + .expect("the altered token remains structurally valid"); + let forged = arena + .get::(forged_generation, ResourceTypeId::IO_FILE) + .expect_err("an altered generation must not resolve"); + assert_eq!(forged.code(), RuntimeErrorCode::ResourceStale); + let wrong_type = arena + .get::(handle, ResourceTypeId::SQLITE_CONNECTION) + .expect_err("wrong resource type should be rejected"); + assert_eq!(wrong_type.code(), RuntimeErrorCode::ResourceTypeMismatch); + let limit_error = arena + .insert(ResourceTypeId::IO_FILE, 8_u32) + .expect_err("the bounded arena should reject a second resource"); + assert_eq!(limit_error.code(), RuntimeErrorCode::ResourceLimitExceeded); + + assert_eq!( + arena + .close(handle, CancellationReason::ResourceClosed) + .expect("close should succeed"), + CloseStatus::Closed + ); + assert_eq!( + arena + .close(handle, CancellationReason::ResourceClosed) + .expect("repeated close should be harmless"), + CloseStatus::AlreadyClosed + ); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + + let replacement = arena + .insert(ResourceTypeId::IO_FILE, 9_u32) + .expect("capacity should be reusable after close"); + assert_ne!( + replacement, handle, + "reusing a slot must change its generation" + ); + let closed = arena + .get::(handle, ResourceTypeId::IO_FILE) + .expect_err("the prior generation must not resolve after slot reuse"); + assert_eq!(closed.code(), RuntimeErrorCode::ResourceStale); + assert_eq!( + arena + .get::(replacement, ResourceTypeId::IO_FILE) + .expect("the replacement generation should resolve"), + &9 + ); +} + +#[test] +fn resource_handles_cannot_cross_resource_arenas() { + let mut first = ResourceArena::with_limit(1).expect("resource limit should be valid"); + let second = ResourceArena::with_limit(1).expect("resource limit should be valid"); + let handle = first + .insert(ResourceTypeId::IO_FILE, 1_u32) + .expect("resource should be allocated"); + + let error = second + .get::(handle, ResourceTypeId::IO_FILE) + .expect_err("a handle from another arena must be rejected"); + assert_eq!(error.code(), RuntimeErrorCode::ResourceHandleWrongTable); +} + +#[test] +fn cancellation_transitions_once_and_runs_cleanup_once() { + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let count_for_cleanup = Arc::clone(&cleanup_count); + let mut registry = OperationRegistry::with_limit(2).expect("operation limit should be valid"); + let operation = registry + .start_owned( + cancellation::OperationOwner::Io, + None, + None, + Some(Box::new(move |end| { + assert_eq!( + end, + cancellation::OperationEnd::Cancelled(CancellationReason::Requested) + ); + count_for_cleanup.fetch_add(1, Ordering::SeqCst); + Ok(()) + })), + ) + .expect("operation should start"); + let token = operation.token(); + + assert_eq!(operation.status(), OperationStatus::Pending); + assert!( + operation + .cancel(CancellationReason::Requested) + .expect("cancel should succeed") + ); + assert!( + !operation + .cancel(CancellationReason::Requested) + .expect("cancel is idempotent") + ); + assert_eq!( + operation.status(), + OperationStatus::Cancelled(CancellationReason::Requested) + ); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + let cancelled = token + .check() + .expect_err("the cancellation token should stop the operation"); + assert_eq!(cancelled.code(), RuntimeErrorCode::OperationCancelled); + assert!( + !operation + .complete() + .expect("terminal operation should remain terminal") + ); +} + +#[test] +fn cancellation_after_completion_does_not_reopen_or_relabel_operation() { + let mut registry = OperationRegistry::with_limit(2).expect("operation limit should be valid"); + let operation = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect("operation should start"); + assert!(operation.complete().expect("operation should complete")); + assert!( + !operation + .cancel(CancellationReason::Requested) + .expect("cancel is idempotent") + ); + assert_eq!(operation.status(), OperationStatus::Completed); + assert!(!operation.token().is_cancelled()); +} + +#[test] +fn operation_registry_bounds_active_operations_and_releases_cancelled_state() { + let mut registry = OperationRegistry::with_limit(1).expect("operation limit should be valid"); + let operation = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect("first operation should start"); + let limit_error = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect_err("active operation limit should be enforced"); + assert_eq!(limit_error.code(), RuntimeErrorCode::OperationLimitExceeded); + + assert!( + registry + .cancel(operation.id(), CancellationReason::VmReset) + .expect("registry cancellation should succeed") + ); + assert_eq!(registry.active_count(), 0); + assert!(matches!( + operation.status(), + OperationStatus::Cancelled(CancellationReason::VmReset) + )); +} + +#[test] +fn registry_retains_terminal_result_until_it_is_consumed() { + let mut registry = OperationRegistry::with_limit(1).expect("operation limit should be valid"); + let operation = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect("operation should start"); + assert!(operation.complete().expect("completion should succeed")); + + let limit_error = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect_err("unconsumed terminal result should retain its registry slot"); + assert_eq!(limit_error.code(), RuntimeErrorCode::OperationLimitExceeded); + assert!(registry.get(operation.id()).is_ok()); + + assert!( + !registry + .complete(operation.id()) + .expect("consuming an already completed operation should succeed") + ); + assert!(registry.get(operation.id()).is_err()); + registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect("consuming the terminal result should release capacity"); +} + +#[test] +fn concurrent_completion_and_cancellation_choose_one_terminal_state() { + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let cleanup_for_operation = Arc::clone(&cleanup_count); + let mut registry = OperationRegistry::with_limit(2).expect("operation limit should be valid"); + let operation = registry + .start_owned( + cancellation::OperationOwner::Io, + None, + None, + Some(Box::new(move |_| { + cleanup_for_operation.fetch_add(1, Ordering::SeqCst); + Ok(()) + })), + ) + .expect("operation should start"); + let barrier = Arc::new(Barrier::new(3)); + + let complete_operation = operation.clone(); + let complete_barrier = Arc::clone(&barrier); + let complete = std::thread::spawn(move || { + complete_barrier.wait(); + complete_operation + .complete() + .expect("completion should run") + }); + + let cancel_operation = operation.clone(); + let cancel_barrier = Arc::clone(&barrier); + let cancel = std::thread::spawn(move || { + cancel_barrier.wait(); + cancel_operation + .cancel(CancellationReason::Requested) + .expect("cancellation should run") + }); + + barrier.wait(); + let terminal_wins = usize::from(complete.join().expect("completion thread")) + + usize::from(cancel.join().expect("cancellation thread")); + assert_eq!(terminal_wins, 1); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + match operation.status() { + OperationStatus::Completed => assert_eq!(operation.token().reason(), None), + OperationStatus::Cancelled(reason) => { + assert_eq!(reason, CancellationReason::Requested); + assert_eq!(operation.token().reason(), Some(reason)); + } + status => panic!("unexpected terminal state: {status:?}"), + } +} + +#[test] +fn completed_child_ignores_later_parent_cancellation() { + let mut registry = OperationRegistry::with_limit(4).expect("operation limit should be valid"); + let parent = registry + .start_owned(cancellation::OperationOwner::Http, None, None, None) + .expect("parent should start"); + let child = registry + .start_owned( + cancellation::OperationOwner::Io, + Some(&parent.token()), + None, + None, + ) + .expect("child should start"); + + assert!(child.complete().expect("child should complete")); + assert!( + parent + .cancel(CancellationReason::Requested) + .expect("parent should cancel") + ); + assert_eq!(child.status(), OperationStatus::Completed); + assert_eq!(child.token().reason(), None); +} + +#[test] +fn expired_deadline_is_the_status_token_and_cleanup_reason() { + let cleanup_end = Arc::new(Mutex::new(None)); + let cleanup_end_for_operation = Arc::clone(&cleanup_end); + let mut registry = OperationRegistry::with_limit(4).expect("operation limit should be valid"); + let parent = registry + .start_owned(cancellation::OperationOwner::Io, None, None, None) + .expect("parent should start"); + let operation = registry + .start_owned( + cancellation::OperationOwner::Io, + Some(&parent.token()), + Some(Instant::now() - Duration::from_millis(1)), + Some(Box::new(move |end| { + *cleanup_end_for_operation.lock().expect("cleanup lock") = Some(end); + Ok(()) + })), + ) + .expect("deadline child should start"); + + assert!( + operation + .cancel(CancellationReason::Requested) + .expect("deadline cancellation should run") + ); + assert_eq!( + operation.token().reason(), + Some(CancellationReason::Deadline) + ); + assert_eq!( + operation.status(), + OperationStatus::Cancelled(CancellationReason::Deadline) + ); + assert_eq!( + *cleanup_end.lock().expect("cleanup lock"), + Some(cancellation::OperationEnd::Cancelled( + CancellationReason::Deadline + )) + ); +} diff --git a/tests/runtime_host_tests.rs b/tests/runtime_host_tests.rs new file mode 100644 index 00000000..af5d5785 --- /dev/null +++ b/tests/runtime_host_tests.rs @@ -0,0 +1,87 @@ +#![cfg(feature = "runtime")] + +use std::sync::{Arc, Mutex}; + +use vm::{ + EventPayload, EventSink, HostFunctionRegistry, RuntimeResult, Value, Vm, VmStatus, + compile_source, +}; + +struct RecordingEventSink(Arc>>); + +impl EventSink for RecordingEventSink { + fn emit(&mut self, payload: EventPayload) -> RuntimeResult<()> { + self.0 + .lock() + .expect("event capture lock should not be poisoned") + .push(payload.into_value()); + Ok(()) + } +} + +#[test] +fn runtime_input_host_reads_embedding_run_value() { + let program = compile_source( + r#" + use runtime; + runtime::input(); + "#, + ) + .expect("runtime input source should compile") + .program; + let mut vm = Vm::new(program); + vm.set_runtime_input(Value::string("run-input")) + .expect("runtime input should be configurable"); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default runtime host registry should bind"); + + assert_eq!( + vm.run().expect("runtime input should execute"), + VmStatus::Halted + ); + assert_eq!(vm.stack().last(), Some(&Value::string("run-input"))); +} + +#[test] +fn runtime_input_host_reports_missing_embedding_value() { + let program = compile_source( + r#" + use runtime; + runtime::input(); + "#, + ) + .expect("runtime input source should compile") + .program; + let mut vm = Vm::new(program); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default runtime host registry should bind"); + + let error = vm.run().expect_err("missing runtime input should fail"); + assert!(error.to_string().contains("input_unavailable")); +} + +#[test] +fn public_runtime_event_contract_is_implementable_and_configurable() { + let program = compile_source("0;") + .expect("minimal runtime host program should compile") + .program; + let events = Arc::new(Mutex::new(Vec::new())); + let mut vm = Vm::new(program); + vm.set_runtime_event_sink(RecordingEventSink(Arc::clone(&events))) + .expect("public EventSink implementation should be configurable"); + vm.clear_runtime_event_sink(); +} + +#[cfg(feature = "sqlite")] +#[test] +fn public_sqlite_policy_configures_the_production_vm() { + let program = compile_source("0;") + .expect("minimal SQLite host program should compile") + .program; + let mut vm = Vm::new(program); + vm.configure_sqlite(vm::SqlitePolicy::default()); + let _limits = vm::SqliteLimits::default(); + vm.clear_sqlite_configuration(); +} diff --git a/tests/vm/http_host_tests.rs b/tests/vm/http_host_tests.rs new file mode 100644 index 00000000..2e30abdf --- /dev/null +++ b/tests/vm/http_host_tests.rs @@ -0,0 +1,310 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::thread; + +use vm::{ + CallOutcome, CallReturn, HostFunctionRegistry, HttpConfig, Program, Value, Vm, VmStatus, + compile_source, +}; + +fn build_request_program(url: String) -> Program { + compile_source(&format!( + r#" + use http; + http::client::request({{"method": "GET", "url": "{url}"}}); + "# + )) + .expect("HTTP request source should compile") + .program +} + +fn local_http_config(port: u16) -> HttpConfig { + HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![port], + allow_private_ips: true, + ..HttpConfig::default() + } +} + +fn spawn_test_server() -> (u16, thread::JoinHandle<()>) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("test listener should bind"); + let port = listener + .local_addr() + .expect("test listener should have an address") + .port(); + let handle = thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("test request should arrive"); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = stream + .read(&mut buffer) + .expect("request should be readable"); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + assert!(request.starts_with(b"GET / HTTP/1.1")); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nX-Test: yes\r\n\r\nok") + .expect("response should be writable"); + }); + (port, handle) +} + +fn response_field<'a>(value: &'a Value, key: &str) -> &'a Value { + let Value::Map(map) = value else { + panic!("expected response map, got {value:?}"); + }; + map.get(&Value::string(key)) + .unwrap_or_else(|| panic!("response missing field {key}")) +} + +async fn drive_vm_to_halt(vm: &mut Vm) -> Result<(), vm::VmError> { + let mut status = vm.run()?; + loop { + match status { + VmStatus::Halted => return Ok(()), + VmStatus::Yielded => status = vm.resume()?, + VmStatus::Waiting(_) => { + vm.await_waiting_host_op().await?; + status = vm.resume()?; + } + } + } +} + +#[tokio::test(flavor = "current_thread")] +async fn http_host_executes_a_bounded_request_and_returns_a_response_map() { + let (port, server) = spawn_test_server(); + let mut vm = Vm::new(build_request_program(format!("http://127.0.0.1:{port}/"))); + vm.configure_http(local_http_config(port)); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + + drive_vm_to_halt(&mut vm) + .await + .expect("http request should complete"); + server.join().expect("test server should finish"); + + assert_eq!(response_field(&vm.stack()[0], "status"), &Value::Int(200)); + assert_eq!( + response_field(&vm.stack()[0], "body"), + &Value::bytes(b"ok".to_vec()) + ); +} + +#[test] +fn http_host_rejects_targets_until_an_explicit_policy_allows_them() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + let error = vm + .run() + .expect_err("unconfigured HTTP targets must be rejected"); + assert!( + error.to_string().contains("HTTP host is not configured") + || error + .to_string() + .contains("HTTP target host is not allowed"), + "unexpected error: {error}" + ); +} + +#[test] +fn empty_registry_keeps_language_builtins_but_rejects_http_capability() { + let mut language_vm = Vm::new( + vm::compile_source("assert(true);") + .expect("language builtin program should compile") + .program, + ); + HostFunctionRegistry::empty() + .bind_vm_cached(&mut language_vm) + .expect("empty registry should bind a program without host imports"); + assert_eq!( + language_vm.run().expect("language builtin should run"), + VmStatus::Halted + ); + + let mut http_vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut http_vm) + .expect("empty registry should prepare the program"); + let error = http_vm + .run() + .expect_err("unapproved HTTP capability must be rejected"); + assert!(matches!(error, vm::VmError::UnboundImport(name) if name == "http::client::request")); +} + +#[test] +fn restricted_registry_requires_explicit_namespaced_builtin_capability() { + let compiled = compile_source( + r#"use io; +io::open("/tmp/rustscript-capability-test", "r");"#, + ) + .expect("namespaced host builtin should compile"); + let mut vm = Vm::new(compiled.program); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut vm) + .expect("restricted registry should bind namespaced host builtin"); + let error = vm + .run() + .expect_err("restricted registry should reject ungranted namespaced builtin"); + assert!(matches!(error, vm::VmError::UnboundImport(name) if name == "io_open")); +} + +#[test] +fn capability_binding_plan_cannot_cross_registry_profiles() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let unrestricted = HostFunctionRegistry::new(); + let plan = unrestricted + .prepare_plan(&program.imports) + .expect("unrestricted registry should prepare HTTP plan"); + let mut vm = Vm::new(program); + let error = HostFunctionRegistry::restricted() + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("capability plan must not cross registry profiles"); + assert!(error.to_string().contains("different capability profile")); +} + +#[test] +fn capability_binding_plan_cannot_outlive_registry_mutation() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let mut registry = HostFunctionRegistry::new(); + let plan = registry + .prepare_plan(&program.imports) + .expect("registry should prepare HTTP plan"); + registry + .allow_builtin("http::client::request") + .expect("HTTP capability should be a known host callable"); + let mut vm = Vm::new(program); + let error = registry + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("stale capability plan must not bind"); + assert!(error.to_string().contains("different capability state")); +} + +#[test] +fn capability_binding_plan_detects_divergent_registry_clone_mutations() { + let unchanged_program = build_request_program("http://127.0.0.1:1/".to_string()); + let unchanged_registry = HostFunctionRegistry::restricted(); + let unchanged_plan = unchanged_registry + .prepare_plan(&unchanged_program.imports) + .expect("restricted registry should prepare HTTP plan"); + let unchanged_clone = unchanged_registry.clone(); + let mut unchanged_vm = Vm::new(unchanged_program); + unchanged_clone + .bind_vm_with_plan(&mut unchanged_vm, &unchanged_plan) + .expect("an unchanged registry clone should reuse the plan"); + + let branch_program = build_request_program("http://127.0.0.1:1/".to_string()); + let branch_registry = HostFunctionRegistry::restricted(); + let mut first_mutation = branch_registry.clone(); + let mut second_mutation = branch_registry; + first_mutation + .allow_builtin("http::client::request") + .expect("HTTP capability should be known"); + second_mutation + .allow_builtin("io::open") + .expect("io capability should be known"); + let plan = first_mutation + .prepare_plan(&branch_program.imports) + .expect("first capability branch should prepare HTTP plan"); + let mut mutated_vm = Vm::new(branch_program); + let error = second_mutation + .bind_vm_with_plan(&mut mutated_vm, &plan) + .expect_err("divergent capability branches must reject each other's plan"); + assert!(error.to_string().contains("different capability state")); +} + +#[test] +fn registry_state_rejects_structural_sibling_mutations() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let registry = HostFunctionRegistry::new(); + let mut source = registry.clone(); + let destination = registry; + source.register_static_args("test::structural", 0, |_args| { + Ok(CallOutcome::Return(CallReturn::One(Value::Null))) + }); + let plan = source + .prepare_plan(&program.imports) + .expect("mutated source registry should prepare HTTP plan"); + let mut vm = Vm::new(program); + let error = destination + .bind_vm_with_plan(&mut vm, &plan) + .expect_err("structural sibling mutation must reject the plan"); + assert!(error.to_string().contains("different registry state")); +} + +#[test] +fn cached_plan_refreshes_after_a_sibling_registry_mutation() { + let program = build_request_program("http://127.0.0.1:1/".to_string()); + let registry = HostFunctionRegistry::new(); + let mut mutating_sibling = registry.clone(); + let destination = registry; + + let mut priming_vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + destination + .bind_vm_cached(&mut priming_vm) + .expect("destination should prime its plan cache"); + mutating_sibling.register_static_args("test::cache_refresh", 0, |_args| { + Ok(CallOutcome::Return(CallReturn::One(Value::Null))) + }); + + let mut refreshed_vm = Vm::new(program); + destination + .bind_vm_cached(&mut refreshed_vm) + .expect("destination should rebuild a plan after sibling mutation"); +} + +#[tokio::test(flavor = "current_thread")] +async fn explicitly_allowed_http_capability_reaches_http_policy() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + vm.configure_http(HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![1], + allow_private_ips: true, + ..HttpConfig::default() + }); + let mut registry = HostFunctionRegistry::restricted(); + registry + .allow_builtin("http::client::request") + .expect("HTTP builtin should be explicitly allowlisted"); + registry + .bind_vm_cached(&mut vm) + .expect("explicit capability plan should bind"); + let error = drive_vm_to_halt(&mut vm) + .await + .expect_err("connection failure should reach HTTP runtime"); + assert!(!matches!(error, vm::VmError::UnboundImport(_))); +} + +#[test] +fn http_in_flight_limit_rejects_before_starting_a_request() { + let mut vm = Vm::new(build_request_program("http://127.0.0.1:1/".to_string())); + vm.set_http_max_in_flight(0); + vm.configure_http(HttpConfig { + allowed_schemes: vec!["http".to_string()], + allowed_hosts: vec!["127.0.0.1".to_string()], + allowed_ports: vec![1], + allow_private_ips: true, + + ..HttpConfig::default() + }); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("default host registry should bind HTTP"); + let error = vm + .run() + .expect_err("zero in-flight capacity must reject the request"); + assert!(error.to_string().contains("in-flight request limit")); +} diff --git a/tests/vm/ownership_tests.rs b/tests/vm/ownership_tests.rs new file mode 100644 index 00000000..ffad0c5c --- /dev/null +++ b/tests/vm/ownership_tests.rs @@ -0,0 +1,385 @@ +//! Milestone 1 of the VM runtime decomposition plan: ownership tests. +//! +//! These tests pin the ownership contract through the public embedding API: +//! - one immutable program can create multiple isolated instances; +//! - run input/events/budgets never leak between runs; +//! - backend caches may be shared without sharing stacks/resources; +//! - reset closes run-scoped state and retains only documented reusable state. + +#[path = "../common/mod.rs"] +mod common; +use common::*; + +use std::sync::{Arc, Mutex}; + +use vm::{HostFunctionRegistry, Value, VmStatus}; + +fn non_yielding_returns_zero(_: &[Value]) -> Result { + Ok(vm::CallOutcome::Return(vm::CallReturn::one(Value::Int(0)))) +} + +fn non_yielding_returns_seven(_: &[Value]) -> Result { + Ok(vm::CallOutcome::Return(vm::CallReturn::one(Value::Int(7)))) +} + +fn non_yielding_returns_nine(_: &[Value]) -> Result { + Ok(vm::CallOutcome::Return(vm::CallReturn::one(Value::Int(9)))) +} + +fn non_yielding_returns_forty_two(_: &[Value]) -> Result { + Ok(vm::CallOutcome::Return(vm::CallReturn::one(Value::Int(42)))) +} + +struct PendingOneHost; + +impl vm::HostArgsFunction for PendingOneHost { + fn call(&mut self, _args: &[Value]) -> vm::VmResult { + Ok(vm::CallOutcome::Pending(1)) + } +} + +/// One immutable program produces independent instances: each run keeps its own +/// stack, locals, and input, and no instance observes another's execution. +#[test] +fn one_immutable_program_creates_multiple_isolated_instances() { + let program = Arc::new( + compile_source( + r#" + use runtime; + let value: string = runtime::input_json(); + value; + "#, + ) + .expect("source should compile") + .program, + ); + + let mut first = Vm::new_shared(Arc::clone(&program)); + let mut second = Vm::new_shared(Arc::clone(&program)); + HostFunctionRegistry::new() + .bind_vm_cached(&mut first) + .expect("runtime hosts should bind"); + HostFunctionRegistry::new() + .bind_vm_cached(&mut second) + .expect("runtime hosts should bind"); + + first + .set_runtime_input(Value::string("first")) + .expect("input should be accepted"); + second + .set_runtime_input(Value::string("second")) + .expect("input should be accepted"); + + assert_eq!(first.run().expect("first should run"), VmStatus::Halted); + assert_eq!( + first.stack().last(), + Some(&Value::string("\"first\"")), + "first instance must observe its own input" + ); + assert_eq!(second.run().expect("second should run"), VmStatus::Halted); + assert_eq!( + second.stack().last(), + Some(&Value::string("\"second\"")), + "second instance must observe its own input" + ); + assert_eq!( + first.stack().last(), + Some(&Value::string("\"first\"")), + "second's run must not overwrite first's stack" + ); + + // Re-running one instance after reset must not disturb the other. + first.reset_for_reuse(); + first + .set_runtime_input(Value::string("first-again")) + .expect("input should be accepted"); + assert_eq!(first.run().expect("first should rerun"), VmStatus::Halted); + assert_eq!( + second.stack().last(), + Some(&Value::string("\"second\"")), + "first's rerun must not disturb second's stack" + ); + assert_eq!( + first.stack().last(), + Some(&Value::string("\"first-again\"")) + ); +} + +/// Run input and events are run-scoped: a reset closes them, and a later run +/// starts with a clean context. +#[test] +fn run_input_and_events_do_not_leak_between_runs() { + let program = Arc::new( + compile_source( + r#" + use runtime; + let value: string = runtime::input_json(); + runtime::emit_json(value); + value; + "#, + ) + .expect("source should compile") + .program, + ); + let mut vm = Vm::new_shared(Arc::clone(&program)); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("runtime hosts should bind"); + + let seen = Arc::new(Mutex::new(Vec::::new())); + let sink_seen = Arc::clone(&seen); + vm.set_runtime_value_event_sink(move |value: Value| { + sink_seen.lock().expect("sink lock").push(value); + Ok(()) + }) + .expect("event sink should install"); + + vm.set_runtime_input(Value::string("run-one")) + .expect("input should be accepted"); + assert_eq!(vm.run().expect("first run should halt"), VmStatus::Halted); + assert_eq!(vm.stack().last(), Some(&Value::string("\"run-one\""))); + assert_eq!( + seen.lock().expect("sink lock").len(), + 1, + "first run must emit exactly one event" + ); + assert_eq!( + seen.lock().expect("sink lock")[0], + Value::string("\"run-one\"") + ); + + // A reset closes the run-scoped input: the next run must not see the + // previous run's input. + vm.reset_for_reuse(); + let missing = vm.run().expect_err("reset must close run input"); + assert!( + missing.to_string().contains("input_unavailable"), + "unexpected error after reset: {missing:?}" + ); + + // A fresh run (new instance from the same program) with fresh input sees + // neither the old input nor the old event stream. + let mut fresh = Vm::new_shared(Arc::clone(&program)); + HostFunctionRegistry::new() + .bind_vm_cached(&mut fresh) + .expect("runtime hosts should bind"); + let fresh_seen = Arc::new(Mutex::new(Vec::::new())); + let fresh_sink_seen = Arc::clone(&fresh_seen); + fresh + .set_runtime_value_event_sink(move |value: Value| { + fresh_sink_seen.lock().expect("sink lock").push(value); + Ok(()) + }) + .expect("event sink should install"); + fresh + .set_runtime_input(Value::string("run-two")) + .expect("input should be accepted"); + assert_eq!( + fresh.run().expect("fresh run should halt"), + VmStatus::Halted + ); + assert_eq!(fresh.stack().last(), Some(&Value::string("\"run-two\""))); + let events = fresh_seen.lock().expect("sink lock"); + assert_eq!(events.len(), 1, "fresh run must emit exactly one event"); + assert_eq!(events[0], Value::string("\"run-two\"")); +} +/// Fuel budgets are run-scoped: a reset clears the budget, and a new run +/// starts from its configured amount rather than inheriting leftovers. +#[test] +fn fuel_budgets_do_not_leak_between_runs() { + let program = compile_source( + r#" + fn action() -> int; + action(); + "#, + ) + .expect("source should compile") + .program; + let mut vm = Vm::new(program); + vm.bind_static_non_yielding_args_function("action", non_yielding_returns_zero); + + // A configured budget reads back as the configured amount. + vm.set_fuel(100); + assert_eq!(vm.get_fuel(), Some(100)); + + // Explicit consumption draws from the run-scoped budget. + vm.consume_fuel(30) + .expect("consumption within budget should succeed"); + assert_eq!(vm.get_fuel(), Some(70)); + + // A run charges the budget; the leftover is less than what was set. + assert_eq!(vm.run().expect("run should halt"), VmStatus::Halted); + let after_run = vm.get_fuel().expect("metering must still be active"); + assert!( + after_run < 70, + "run must consume from the active budget ({after_run} remaining)" + ); + + // Reset must clear the budget entirely (metering disabled, no leftovers). + vm.reset_for_reuse(); + assert_eq!(vm.get_fuel(), None, "reset must clear run-scoped fuel"); + + // A fresh budget starts from the configured amount, not from the prior + // run's leftover. + vm.set_fuel(200); + assert_eq!(vm.run().expect("run should halt"), VmStatus::Halted); + let fresh = vm.get_fuel().expect("metering must still be active"); + assert!( + fresh < 200, + "fresh budget must be consumed from its own amount ({fresh} remaining)" + ); + assert!( + fresh > after_run, + "fresh budget must not inherit the prior run's leftover" + ); +} + +/// The same immutable program can drive multiple VMs with independent stacks +/// and independent backend caches: one VM's reset and rerun never touches +/// another VM's execution state or cached artifacts. +#[test] +fn shared_program_backend_does_not_share_stacks_or_resources() { + let program = Arc::new( + compile_source( + r#" + fn action() -> int; + action(); + "#, + ) + .expect("source should compile") + .program, + ); + + let mut first = Vm::new_shared(Arc::clone(&program)); + let mut second = Vm::new_shared(Arc::clone(&program)); + first.bind_static_non_yielding_args_function("action", non_yielding_returns_seven); + second.bind_static_non_yielding_args_function("action", non_yielding_returns_nine); + + assert_eq!(first.run().expect("first should run"), VmStatus::Halted); + assert_eq!(second.run().expect("second should run"), VmStatus::Halted); + assert_eq!(first.stack(), &[Value::Int(7)]); + assert_eq!(second.stack(), &[Value::Int(9)]); + + // Reset + rerun on one VM must not change the other VM's stack. + first.reset_for_reuse(); + assert_eq!(first.run().expect("first should rerun"), VmStatus::Halted); + assert_eq!(first.stack(), &[Value::Int(7)]); + assert_eq!(second.stack(), &[Value::Int(9)]); +} + +/// Reset closes run-scoped state while retaining documented reusable state: +/// host bindings, backend configuration, and compiled artifacts survive, while +/// the interpreter state (ip/stack/locals) is rewound. +#[test] +fn reset_closes_run_scoped_state_and_retains_reusable_state() { + let program = compile_source( + r#" + fn action() -> int; + action(); + "#, + ) + .expect("source should compile") + .program; + let mut vm = Vm::new(program); + vm.set_regex_cache_capacity(8); + vm.bind_static_non_yielding_args_function("action", non_yielding_returns_forty_two); + + assert_eq!(vm.run().expect("first run should halt"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); + + vm.reset_for_reuse(); + + // Reusable state survives reset. + assert_eq!( + vm.regex_cache_capacity(), + 8, + "backend cache configuration is reusable across runs" + ); + assert_eq!( + vm.max_script_call_depth(), + vm::DEFAULT_MAX_SCRIPT_CALL_DEPTH, + "interpreter limits are reusable across runs" + ); + + // Run-scoped state is rewound: ip at entry, empty stack, null locals. + assert_eq!(vm.ip(), 0, "reset must rewind the instruction pointer"); + assert!(vm.stack().is_empty(), "reset must clear the stack"); + assert!( + vm.locals().iter().all(|value| *value == Value::Null), + "reset must restore null locals" + ); + + // The retained host binding still executes on the next run. + assert_eq!(vm.run().expect("second run should halt"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::Int(42)]); +} + +/// A yield caused by an explicit wait must leave the instance in a state that +/// reset can close, and a subsequent run must not inherit the wait. +#[test] +fn reset_closes_waiting_state_before_the_next_run() { + let program = compile_source( + r#" + fn action() -> int; + action(); + "#, + ) + .expect("source should compile") + .program; + let mut vm = Vm::new(program); + vm.bind_args_function("action", Box::new(PendingOneHost)); + + let status = vm.run().expect("run should yield"); + assert_eq!(status, VmStatus::Waiting(1)); + assert_eq!(vm.waiting_host_op_id(), Some(1)); + + vm.reset_for_reuse(); + assert_eq!( + vm.waiting_host_op_id(), + None, + "reset must close the pending host wait" + ); +} + +/// Regression pin for a pre-existing JIT issue that is NOT part of the +/// decomposition: a run that fails inside a host import can leave a native +/// trace/region that corrupts the *same instance's* next run after reset. +/// +/// The bug reproduces on the unmodified tree (HEAD cccdd2f + dirty host work): +/// run a program whose first host call errors, `reset_for_reuse()`, then run +/// again with valid input — the second run can fail with `StackUnderflow` +/// instead of executing. Clearing native traces between the runs makes the +/// rerun behave correctly, which isolates the cause to stale JIT state. +#[test] +#[ignore = "pre-existing JIT stale-trace replay after reset; tracked separately"] +fn reset_after_host_error_reruns_cleanly_on_the_same_instance() { + let program = compile_source( + r#" + use runtime; + let value: string = runtime::input_json(); + runtime::emit_json(value); + value; + "#, + ) + .expect("source should compile") + .program; + let mut vm = Vm::new(program); + HostFunctionRegistry::new() + .bind_vm_cached(&mut vm) + .expect("runtime hosts should bind"); + vm.set_runtime_value_event_sink(|_| Ok(())) + .expect("event sink should install"); + + vm.set_runtime_input(Value::string("run-one")) + .expect("input should be accepted"); + assert_eq!(vm.run().expect("first run should halt"), VmStatus::Halted); + + vm.reset_for_reuse(); + let missing = vm.run().expect_err("reset must close run input"); + assert!(missing.to_string().contains("input_unavailable")); + + vm.set_runtime_input(Value::string("run-two")) + .expect("input should be accepted"); + assert_eq!(vm.run().expect("rerun should halt"), VmStatus::Halted); + assert_eq!(vm.stack().last(), Some(&Value::string("\"run-two\""))); +} diff --git a/tests/vm/sqlite_host_tests.rs b/tests/vm/sqlite_host_tests.rs new file mode 100644 index 00000000..4551d827 --- /dev/null +++ b/tests/vm/sqlite_host_tests.rs @@ -0,0 +1,1038 @@ +extern crate vm as rustscript_vm; + +pub mod vm { + + pub use crate::rustscript_vm::{ + CallReturn, HostCallResult, HostOpId, OpCode, Program, Value, VmError, VmMap, VmResult, + }; + + use crate::builtins::runtime::cancellation::{CancellationToken, OperationRegistry}; + use crate::builtins::runtime::resource::ResourceArena; + + #[derive(Clone, Copy, Debug)] + pub struct SqliteLimits { + pub max_connections: usize, + pub max_statements: usize, + pub max_rows: usize, + pub max_columns: usize, + pub max_result_bytes: usize, + pub max_statement_bytes: usize, + pub max_parameters: usize, + pub max_parameter_bytes: usize, + pub max_pending_operations: usize, + pub max_transaction_ms: u64, + pub busy_timeout_ms: u64, + } + + impl Default for SqliteLimits { + fn default() -> Self { + Self { + max_connections: 16, + max_statements: 128, + max_rows: 1_000, + max_columns: 128, + max_result_bytes: 4 * 1024 * 1024, + max_statement_bytes: 1024 * 1024, + max_parameters: 128, + max_parameter_bytes: 1024 * 1024, + max_pending_operations: 32, + max_transaction_ms: 5_000, + busy_timeout_ms: 5_000, + } + } + } + + #[derive(Clone, Debug, Default)] + pub struct SqlitePolicy { + pub database_root: Option, + pub allow_unsafe_sql: bool, + pub limits: SqliteLimits, + } + + pub(crate) struct TestHostRuntime { + pub(crate) runtime_resources: ResourceArena, + pub(crate) runtime_operations: OperationRegistry, + + pub(crate) sqlite_policy: SqlitePolicy, + } + + pub(crate) struct TestRunContext { + pub(crate) cancellation: CancellationToken, + } + + pub struct Vm { + pub(crate) host: TestHostRuntime, + pub(crate) run_ctx: TestRunContext, + } + + impl Vm { + pub fn new(_program: Program) -> Self { + Self { + host: TestHostRuntime { + runtime_resources: ResourceArena::default(), + runtime_operations: OperationRegistry::default(), + + sqlite_policy: SqlitePolicy::default(), + }, + run_ctx: TestRunContext { + cancellation: CancellationToken::root(), + }, + } + } + + pub fn configure_sqlite(&mut self, policy: SqlitePolicy) { + self.host.sqlite_policy = policy; + } + } +} + +mod builtins { + pub use crate::vm::{Value, Vm, VmResult}; + + pub mod runtime { + pub use crate::vm::{HostCallResult, VmMap}; + + pub mod error { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/builtins/runtime/error.rs" + )); + } + + #[allow(dead_code)] + pub mod cancellation { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/builtins/runtime/cancellation.rs" + )); + } + + #[allow(dead_code)] + pub mod resource { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/builtins/runtime/resource.rs" + )); + } + + pub(crate) fn cancel_runtime_operation( + vm: &mut crate::vm::Vm, + op_id: cancellation::OperationId, + reason: cancellation::CancellationReason, + ) { + let payload = vm + .host + .runtime_operations + .get(op_id) + .ok() + .and_then(|operation| operation.payload()); + let _ = vm.host.runtime_operations.cancel(op_id, reason); + if let Some(payload) = payload { + let _ = close_runtime_resource(vm, payload, reason); + } + } + + pub(crate) fn close_runtime_resource( + vm: &mut crate::vm::Vm, + handle: resource::ResourceHandle, + reason: cancellation::CancellationReason, + ) -> error::RuntimeResult { + let operations = vm + .host + .runtime_operations + .operations_for_resource(handle) + .into_iter() + .map(|operation| { + let payload = operation.payload(); + (operation, payload) + }) + .collect::>(); + for (operation, _) in &operations { + operation.token().mark_cancelled(reason); + } + for (operation, _) in &operations { + let _ = vm.host.runtime_operations.cancel(operation.id(), reason); + } + for (_, payload) in operations { + if let Some(payload) = payload { + let _ = close_runtime_resource(vm, payload, reason); + } + } + vm.host.runtime_resources.close(handle, reason) + } + + pub mod typed { + pub type VmArrayRef<'a> = &'a [crate::vm::Value]; + pub type VmMapRef<'a> = &'a crate::vm::VmMap; + } + + pub trait TestBorrowArg<'a>: Sized { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult; + } + + impl<'a> TestBorrowArg<'a> for crate::vm::Value { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + args.get(index) + .cloned() + .ok_or(crate::vm::VmError::HostError(label.to_string())) + } + } + + impl<'a> TestBorrowArg<'a> for i64 { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + match args.get(index) { + Some(crate::vm::Value::Int(value)) => Ok(*value), + _ => Err(crate::vm::VmError::HostError(label.to_string())), + } + } + } + + impl<'a> TestBorrowArg<'a> for &'a str { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + match args.get(index) { + Some(crate::vm::Value::String(value)) => Ok(value.as_str()), + _ => Err(crate::vm::VmError::HostError(label.to_string())), + } + } + } + + impl<'a> TestBorrowArg<'a> for &'a [crate::vm::Value] { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + match args.get(index) { + Some(crate::vm::Value::Array(value)) => Ok(value.as_slice()), + _ => Err(crate::vm::VmError::HostError(label.to_string())), + } + } + } + + impl<'a> TestBorrowArg<'a> for &'a crate::vm::VmMap { + fn borrow_arg( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + match args.get(index) { + Some(crate::vm::Value::Map(value)) => Ok(value.as_ref()), + _ => Err(crate::vm::VmError::HostError(label.to_string())), + } + } + } + + pub fn borrow_arg<'a, T: TestBorrowArg<'a>>( + args: &'a [crate::vm::Value], + index: usize, + label: &'static str, + ) -> crate::vm::VmResult { + T::borrow_arg(args, index, label) + } + + pub mod sqlite { + include!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/src/builtins/runtime/sqlite.rs" + )); + } + + pub mod test_api { + use std::task::{Context, Poll}; + + use super::cancellation::{ + CancellationReason, OperationId, OperationOwner, OperationStatus, + }; + use super::resource::{ResourceHandle, ResourceTypeId}; + use super::{HostCallResult, VmMap}; + use crate::vm::{CallReturn, HostOpId, Value, Vm, VmResult}; + + pub fn open(vm: &mut Vm, args: &[Value]) -> VmResult { + super::sqlite::builtin_sqlite_open(vm, args) + } + + pub fn execute(vm: &mut Vm, args: &[Value]) -> VmResult> { + super::sqlite::builtin_sqlite_execute(vm, args) + } + + pub fn query(vm: &mut Vm, args: &[Value]) -> VmResult> { + super::sqlite::builtin_sqlite_query(vm, args) + } + + pub fn transaction( + vm: &mut Vm, + args: &[Value], + ) -> VmResult>> { + super::sqlite::builtin_sqlite_transaction(vm, args) + } + + pub fn close(vm: &mut Vm, args: &[Value]) -> VmResult<()> { + super::sqlite::builtin_sqlite_close(vm, args) + } + + pub fn poll( + vm: &mut Vm, + op_id: HostOpId, + cx: &mut Context<'_>, + ) -> Poll> { + super::sqlite::poll_pending_op(vm, op_id, cx) + } + + pub fn cancel(vm: &mut Vm, op_id: HostOpId) { + let Ok(id) = OperationId::from_raw(op_id) else { + return; + }; + let payload = vm + .host + .runtime_operations + .get(id) + .ok() + .filter(|operation| operation.owner() == OperationOwner::Sqlite) + .and_then(|operation| operation.payload()); + let _ = vm + .host + .runtime_operations + .cancel(id, CancellationReason::Requested); + if let Some(payload) = payload { + let _ = vm + .host + .runtime_resources + .close(payload, CancellationReason::Requested); + } + } + + pub fn active_operation_id(vm: &Vm, resource_id: i64) -> Option { + super::sqlite::active_operation_id(vm, resource_id) + } + + pub fn has_pending(vm: &Vm, op_id: HostOpId) -> bool { + OperationId::from_raw(op_id).is_ok_and(|id| { + vm.host.runtime_operations.get(id).is_ok_and(|operation| { + operation.owner() == OperationOwner::Sqlite + && matches!(operation.status(), OperationStatus::Pending) + && operation.payload().is_some() + }) + }) + } + + pub fn close_all(vm: &mut Vm) { + let _ = vm + .host + .runtime_operations + .cancel_all(CancellationReason::VmReset); + let _ = vm + .host + .runtime_resources + .close_all(CancellationReason::VmReset); + } + + pub fn has_sqlite_operation_owner(vm: &Vm, op_id: HostOpId) -> bool { + OperationId::from_raw(op_id) + .ok() + .and_then(|id| vm.host.runtime_operations.get(id).ok()) + .map(|operation| operation.owner()) + == Some(OperationOwner::Sqlite) + } + + pub fn is_sqlite_resource(handle: i64) -> bool { + ResourceHandle::from_value(&Value::Int(handle)) + .is_ok_and(|handle| handle.resource_type() == ResourceTypeId::SQLITE_CONNECTION) + } + + pub fn insert_wrong_type_resource(vm: &mut Vm) -> i64 { + let handle = vm + .host + .runtime_resources + .insert(ResourceTypeId::IO_FILE, 7_i64) + .expect("test resource should be inserted"); + match handle.as_value() { + Value::Int(value) => value, + _ => unreachable!(), + } + } + } + } +} + +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::task::{Context, Poll, Wake, Waker}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use builtins::runtime::test_api as sqlite; +use vm::{CallReturn, HostCallResult, OpCode, Program, Value, Vm, VmError}; + +struct NoopWake; + +impl Wake for NoopWake { + fn wake(self: Arc) {} +} + +fn noop_waker() -> Waker { + Waker::from(Arc::new(NoopWake)) +} + +fn new_vm() -> Vm { + Vm::new(Program::new(Vec::new(), vec![OpCode::Ret as u8])) +} + +fn temporary_root(label: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("system clock should be after the Unix epoch") + .as_nanos(); + let root = std::env::temp_dir().join(format!( + "rustscript-sqlite-{label}-{}-{nonce}", + std::process::id() + )); + fs::create_dir_all(&root).expect("temporary SQLite root should be created"); + root +} + +fn map_value(entries: impl IntoIterator) -> Value { + Value::map( + entries + .into_iter() + .map(|(key, value)| (Value::string(key), value)) + .collect(), + ) +} + +fn field<'a>(map: &'a vm::VmMap, key: &str) -> &'a Value { + map.get(&Value::string(key)) + .unwrap_or_else(|| panic!("missing SQLite result field {key}")) +} + +fn open_options(root: &Path, path: &str, mode: &str, limits: Value) -> Value { + map_value([ + ("root", Value::string(root.to_string_lossy().into_owned())), + ("path", Value::string(path)), + ("mode", Value::string(mode)), + ("limits", limits), + ]) +} + +fn limits(entries: impl IntoIterator) -> Value { + map_value( + entries + .into_iter() + .map(|(key, value)| (key, Value::Int(value))), + ) +} + +fn empty_params() -> Value { + Value::array(Vec::new()) +} + +fn wait_pending(vm: &mut Vm, op_id: vm::HostOpId) -> Result { + let waker = noop_waker(); + let mut cx = Context::from_waker(&waker); + loop { + match sqlite::poll(vm, op_id, &mut cx) { + Poll::Pending => std::thread::yield_now(), + Poll::Ready(Ok(CallReturn::None)) => return Ok(Value::Null), + Poll::Ready(Ok(CallReturn::One(value))) => return Ok(value), + Poll::Ready(Err(error)) => return Err(error), + } + } +} + +fn map_from_value(value: Value) -> vm::VmMap { + let Value::Map(map) = value else { + panic!("SQLite host result should be a map"); + }; + (*map).clone() +} + +fn host_map( + vm: &mut Vm, + result: Result, VmError>, +) -> Result { + let result = result?; + match result { + HostCallResult::Return(map) => Ok(map), + HostCallResult::Pending(op_id) => Ok(map_from_value(wait_pending(vm, op_id)?)), + } +} + +fn host_array( + vm: &mut Vm, + result: Result>, VmError>, +) -> Result, VmError> { + let result = result?; + match result { + HostCallResult::Return(values) => Ok(values), + HostCallResult::Pending(op_id) => { + let Value::Array(values) = wait_pending(vm, op_id)? else { + panic!("SQLite transaction result should be an array"); + }; + Ok((*values).clone()) + } + } +} + +fn open_db(vm: &mut Vm, options: Value) -> i64 { + if let Value::Map(options_map) = &options + && let Some(Value::String(root)) = options_map.get(&Value::string("root")) + { + vm.configure_sqlite(vm::SqlitePolicy { + database_root: Some(root.as_ref().clone()), + ..vm::SqlitePolicy::default() + }); + } + sqlite::open(vm, &[options]).expect("SQLite open should return") +} + +fn execute(vm: &mut Vm, db_id: i64, sql: &str, params: Value) -> Result { + let result = sqlite::execute(vm, &[Value::Int(db_id), Value::string(sql), params]); + host_map(vm, result) +} + +fn query( + vm: &mut Vm, + db_id: i64, + sql: &str, + params: Value, + query_limits: Value, +) -> Result { + let result = sqlite::query( + vm, + &[Value::Int(db_id), Value::string(sql), params, query_limits], + ); + host_map(vm, result) +} + +#[test] +fn sqlite_round_trip_supports_typed_values_and_ordered_transactions() { + let root = temporary_root("round-trip"); + let mut vm = new_vm(); + let db_id = open_db( + &mut vm, + open_options( + &root, + "state.db", + "read_write_create", + limits([ + ("max_rows", 128), + ("max_result_bytes", 64 * 1024), + ("max_statements", 16), + ("max_transaction_ms", 5_000), + ]), + ), + ); + + execute( + &mut vm, + db_id, + "CREATE TABLE values_table (id INTEGER PRIMARY KEY, n INTEGER, r REAL, s TEXT, b BLOB, z TEXT)", + empty_params(), + ) + .expect("table creation should succeed"); + execute( + &mut vm, + db_id, + "INSERT INTO values_table (n, r, s, b, z) VALUES (?1, ?2, ?3, ?4, ?5)", + Value::array(vec![ + Value::Int(7), + Value::Float(1.5), + Value::string("hello"), + Value::bytes(vec![0, 1, 2]), + Value::Null, + ]), + ) + .expect("typed parameter insert should succeed"); + + let rowset = query( + &mut vm, + db_id, + "SELECT n, r, s, b, z FROM values_table ORDER BY id", + empty_params(), + limits([("max_rows", 8), ("max_result_bytes", 64 * 1024)]), + ) + .expect("typed query should succeed"); + assert_eq!( + field(&rowset, "columns"), + &Value::array(vec![ + Value::string("n"), + Value::string("r"), + Value::string("s"), + Value::string("b"), + Value::string("z"), + ]) + ); + assert_eq!(field(&rowset, "truncated"), &Value::Bool(false)); + let Value::Array(rows) = field(&rowset, "rows") else { + panic!("SQLite rows should be an array"); + }; + assert_eq!(rows.len(), 1); + assert_eq!( + rows[0], + Value::array(vec![ + Value::Int(7), + Value::Float(1.5), + Value::string("hello"), + Value::bytes(vec![0, 1, 2]), + Value::Null, + ]) + ); + + let statements = Value::array(vec![ + map_value([ + ( + "sql", + Value::string("INSERT INTO values_table (n) VALUES (?1)"), + ), + ("params", Value::array(vec![Value::Int(8)])), + ]), + map_value([ + ( + "sql", + Value::string("INSERT INTO values_table (n) VALUES (?1)"), + ), + ("params", Value::array(vec![Value::Int(9)])), + ]), + ]); + let transaction = sqlite::transaction(&mut vm, &[Value::Int(db_id), statements]) + .expect("transaction should return"); + let transaction_value = + Value::array(host_array(&mut vm, Ok(transaction)).expect("transaction should complete")); + let Value::Array(results) = transaction_value else { + panic!("transaction should return ordered results"); + }; + assert_eq!(results.len(), 2); + for result in results.iter() { + let Value::Map(result) = result else { + panic!("transaction result should be a map"); + }; + assert_eq!(field(result, "rows_affected"), &Value::Int(1)); + } + + sqlite::close(&mut vm, &[Value::Int(db_id)]).expect("SQLite close should succeed"); + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_enforces_read_only_vm_local_ids_and_sql_safety() { + let root = temporary_root("policy"); + let mut writer_vm = new_vm(); + let db_id = open_db( + &mut writer_vm, + open_options(&root, "state.db", "read_write_create", limits([])), + ); + execute( + &mut writer_vm, + db_id, + "CREATE TABLE items (value INTEGER)", + empty_params(), + ) + .expect("table creation should succeed"); + + let mut other_vm = new_vm(); + let cross_vm_error = sqlite::execute( + &mut other_vm, + &[Value::Int(db_id), Value::string("SELECT 1"), empty_params()], + ) + .expect_err("a SQLite id must not cross VM instances"); + assert!( + cross_vm_error + .to_string() + .contains("unknown SQLite database") + ); + + let unsafe_sql = [ + "ATTACH DATABASE 'other.db' AS other", + "PRAGMA writable_schema = ON", + "SELECT load_extension('not-available')", + "CREATE TABLE first (id INTEGER); CREATE TABLE second (id INTEGER)", + ]; + for sql in unsafe_sql { + let error = sqlite::execute( + &mut writer_vm, + &[Value::Int(db_id), Value::string(sql), empty_params()], + ) + .expect_err("unsafe SQL should be rejected before execution"); + assert!( + error.to_string().contains("not allowed") + || error.to_string().contains("multiple statements") + || error.to_string().contains("disabled"), + "unexpected SQLite policy error: {error}" + ); + } + + let mut read_only_vm = new_vm(); + let read_only_id = open_db( + &mut read_only_vm, + open_options(&root, "state.db", "read_only", limits([])), + ); + let read_only_result = sqlite::execute( + &mut read_only_vm, + &[ + Value::Int(read_only_id), + Value::string("INSERT INTO items (value) VALUES (1)"), + empty_params(), + ], + ); + let read_only_error = host_map(&mut read_only_vm, read_only_result) + .expect_err("read-only SQLite handles must reject writes"); + assert!( + read_only_error.to_string().contains("ReadOnly") + || read_only_error.to_string().contains("readonly") + || read_only_error.to_string().contains("read-only"), + "unexpected read-only error: {read_only_error}" + ); + + sqlite::close_all(&mut writer_vm); + sqlite::close_all(&mut read_only_vm); + sqlite::close_all(&mut other_vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_query_reports_row_and_result_byte_truncation() { + let root = temporary_root("limits"); + let mut vm = new_vm(); + let db_id = open_db( + &mut vm, + open_options( + &root, + "state.db", + "read_write_create", + limits([("max_rows", 32), ("max_result_bytes", 32)]), + ), + ); + execute( + &mut vm, + db_id, + "CREATE TABLE items (value TEXT)", + empty_params(), + ) + .expect("table creation should succeed"); + for value in ["one", "two", "three"] { + execute( + &mut vm, + db_id, + "INSERT INTO items (value) VALUES (?1)", + Value::array(vec![Value::string(value)]), + ) + .expect("row insertion should succeed"); + } + + let row_limited = query( + &mut vm, + db_id, + "SELECT value FROM items ORDER BY rowid", + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ) + .expect("limited query should succeed"); + assert_eq!(field(&row_limited, "truncated"), &Value::Bool(true)); + let Value::Array(rows) = field(&row_limited, "rows") else { + panic!("SQLite rows should be an array"); + }; + assert_eq!(rows.len(), 1); + + let byte_limited = query( + &mut vm, + db_id, + "SELECT value FROM items ORDER BY rowid", + empty_params(), + limits([("max_rows", 32), ("max_result_bytes", 8)]), + ) + .expect("byte-limited query should succeed"); + assert_eq!(field(&byte_limited, "truncated"), &Value::Bool(true)); + + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_pending_operations_can_be_cancelled_and_cleaned_up() { + let root = temporary_root("cancel"); + let mut vm = new_vm(); + let db_id = open_db( + &mut vm, + open_options( + &root, + "state.db", + "read_write_create", + limits([ + ("max_transaction_ms", 10_000), + ("max_result_bytes", 64 * 1024), + ]), + ), + ); + let pending = sqlite::query( + &mut vm, + &[ + Value::Int(db_id), + Value::string( + "WITH RECURSIVE numbers(value) AS (\ + SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 100000000\ + ) SELECT sum(value) FROM numbers", + ), + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ], + ) + .expect("long SQLite query should be scheduled"); + let HostCallResult::Pending(op_id) = pending else { + panic!("long SQLite query should return a pending operation"); + }; + assert!(sqlite::has_pending(&vm, op_id)); + sqlite::cancel(&mut vm, op_id); + assert!(!sqlite::has_pending(&vm, op_id)); + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn cancelling_queued_sqlite_operation_does_not_interrupt_active_sibling() { + let root = temporary_root("queued_cancel"); + let mut vm = new_vm(); + let db_id = open_db( + &mut vm, + open_options( + &root, + "state.db", + "read_write_create", + limits([ + ("max_transaction_ms", 10_000), + ("max_result_bytes", 64 * 1024), + ]), + ), + ); + let active = sqlite::query( + &mut vm, + &[ + Value::Int(db_id), + Value::string( + "WITH RECURSIVE numbers(value) AS (\ + SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 2000000\ + ) SELECT sum(value) FROM numbers", + ), + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ], + ) + .expect("active query should schedule"); + let HostCallResult::Pending(active_id) = active else { + panic!("active query should be pending"); + }; + let wait_deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + while sqlite::active_operation_id(&vm, db_id) != Some(active_id) { + assert!( + std::time::Instant::now() < wait_deadline, + "active query should enter SQLite execution" + ); + std::thread::yield_now(); + } + + let queued = sqlite::query( + &mut vm, + &[ + Value::Int(db_id), + Value::string("SELECT 42"), + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ], + ) + .expect("queued query should schedule"); + let HostCallResult::Pending(queued_id) = queued else { + panic!("queued query should be pending"); + }; + sqlite::cancel(&mut vm, queued_id); + assert_eq!(sqlite::active_operation_id(&vm, db_id), Some(active_id)); + + wait_pending(&mut vm, active_id).expect("active sibling should complete successfully"); + assert!(!sqlite::has_pending(&vm, active_id)); + assert!(!sqlite::has_pending(&vm, queued_id)); + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +fn assert_sqlite_shutdown_cancels_all_siblings(close_all: bool) { + let root = temporary_root(if close_all { + "cancel_all_two_phase" + } else { + "close_two_phase" + }); + let mut vm = new_vm(); + let options = open_options( + &root, + "state.db", + "read_write_create", + limits([ + ("max_transaction_ms", 10_000), + ("max_result_bytes", 64 * 1024), + ]), + ); + let db_id = open_db(&mut vm, options.clone()); + execute( + &mut vm, + db_id, + "CREATE TABLE items (value INTEGER)", + empty_params(), + ) + .expect("table creation should succeed"); + + let active = sqlite::query( + &mut vm, + &[ + Value::Int(db_id), + Value::string( + "WITH RECURSIVE numbers(value) AS (\ + SELECT 1 UNION ALL SELECT value + 1 FROM numbers LIMIT 2000000\ + ) SELECT sum(value) FROM numbers", + ), + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ], + ) + .expect("active query should schedule"); + let HostCallResult::Pending(active_id) = active else { + panic!("active query should be pending"); + }; + let wait_deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + while sqlite::active_operation_id(&vm, db_id) != Some(active_id) { + assert!(std::time::Instant::now() < wait_deadline); + std::thread::yield_now(); + } + + let queued = sqlite::execute( + &mut vm, + &[ + Value::Int(db_id), + Value::string("INSERT INTO items (value) VALUES (42)"), + empty_params(), + ], + ) + .expect("queued insert should schedule"); + let HostCallResult::Pending(queued_id) = queued else { + panic!("queued insert should be pending"); + }; + + if close_all { + sqlite::close_all(&mut vm); + } else { + sqlite::close(&mut vm, &[Value::Int(db_id)]).expect("close should succeed"); + } + assert!(!sqlite::has_pending(&vm, active_id)); + assert!(!sqlite::has_pending(&vm, queued_id)); + + let reopened = open_db(&mut vm, options); + let result = query( + &mut vm, + reopened, + "SELECT count(*) AS count FROM items", + empty_params(), + limits([("max_rows", 1), ("max_result_bytes", 64 * 1024)]), + ) + .expect("count query should succeed"); + let Value::Array(rows) = field(&result, "rows") else { + panic!("rows should be an array"); + }; + assert_eq!(rows[0], Value::array(vec![Value::Int(0)])); + + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_close_cancels_all_siblings_before_waiting_for_workers() { + assert_sqlite_shutdown_cancels_all_siblings(false); +} + +#[test] +fn sqlite_cancel_all_broadcasts_before_waiting_for_workers() { + assert_sqlite_shutdown_cancels_all_siblings(true); +} + +#[test] +fn sqlite_uses_typed_generation_checked_resource_handles() { + let root = temporary_root("resource_handles"); + let mut vm = new_vm(); + let first = open_db( + &mut vm, + open_options(&root, "handles.db", "read_write_create", limits([])), + ); + assert!(sqlite::is_sqlite_resource(first)); + + sqlite::close(&mut vm, &[Value::Int(first)]).expect("first handle should close"); + let second = open_db( + &mut vm, + open_options(&root, "handles.db", "read_write_create", limits([])), + ); + assert_ne!( + first, second, + "slot reuse must advance the handle generation" + ); + + let stale = sqlite::execute( + &mut vm, + &[Value::Int(first), Value::string("SELECT 1"), empty_params()], + ) + .expect_err("a closed generation must stay invalid after slot reuse"); + assert!(stale.to_string().contains("unknown SQLite database")); + + let wrong_type = sqlite::insert_wrong_type_resource(&mut vm); + let wrong_type_error = sqlite::execute( + &mut vm, + &[ + Value::Int(wrong_type), + Value::string("SELECT 1"), + empty_params(), + ], + ) + .expect_err("a handle from another resource type must be rejected"); + assert!(wrong_type_error.to_string().contains("wrong resource type")); + + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} + +#[test] +fn sqlite_pending_work_is_registered_with_the_shared_owner() { + let root = temporary_root("operation_owner"); + let mut vm = new_vm(); + let db_id = open_db( + &mut vm, + open_options(&root, "operations.db", "read_write_create", limits([])), + ); + + let operation = sqlite::execute( + &mut vm, + &[ + Value::Int(db_id), + Value::string("CREATE TABLE items(id INTEGER PRIMARY KEY)"), + empty_params(), + ], + ) + .expect("execute should schedule"); + let HostCallResult::Pending(op_id) = operation else { + panic!("execute should return a pending operation"); + }; + assert!(sqlite::has_sqlite_operation_owner(&vm, op_id)); + let _ = wait_pending(&mut vm, op_id).expect("shared operation should complete"); + assert!(!sqlite::has_pending(&vm, op_id)); + + sqlite::close_all(&mut vm); + fs::remove_dir_all(root).expect("temporary SQLite root should be removed"); +} diff --git a/tests/vm/vm_async_runtime_tests.rs b/tests/vm/vm_async_runtime_tests.rs index b58f8fe8..7fb957ad 100644 --- a/tests/vm/vm_async_runtime_tests.rs +++ b/tests/vm/vm_async_runtime_tests.rs @@ -12,8 +12,8 @@ use std::{ use tokio::sync::oneshot; use vm::{ - BytecodeBuilder, CallOutcome, HostAsyncBridge, HostFunction, HostImport, HostOpId, Program, - Value, ValueType, Vm, VmError, VmStatus, + BytecodeBuilder, CallOutcome, CancellationReason, HostAsyncBridge, HostFunction, HostImport, + HostOpId, Program, Value, ValueType, Vm, VmError, VmStatus, }; type AsyncHostResult = Result; @@ -22,6 +22,7 @@ type SharedAsyncOps = Arc>; #[derive(Default)] struct TestAsyncOps { pending: HashMap>, + cancellations: Vec<(HostOpId, CancellationReason)>, } impl TestAsyncOps { @@ -96,6 +97,12 @@ impl HostAsyncBridge for TestAsyncBridge { .pending .remove(&op_id); } + + fn cancel_op_with_reason(&mut self, op_id: HostOpId, reason: CancellationReason) { + let mut ops = self.ops.lock().expect("test async ops lock poisoned"); + ops.pending.remove(&op_id); + ops.cancellations.push((op_id, reason)); + } } struct AsyncAddOneFunction { @@ -134,6 +141,14 @@ impl HostFunction for AsyncAddOneFunction { } } +struct InvalidPendingFunction; + +impl HostFunction for InvalidPendingFunction { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + Ok(CallOutcome::Pending(0)) + } +} + fn build_async_import_program(input: i64) -> Program { let constants = vec![Value::Int(input)]; let imports = vec![HostImport { @@ -226,8 +241,61 @@ async fn reset_cancels_pending_host_bridge_operation() { )); assert_eq!(ops.lock().unwrap().pending.len(), 1); vm.reset_for_reuse(); - assert_eq!(ops.lock().unwrap().pending.len(), 0); + let ops = ops.lock().unwrap(); + assert_eq!(ops.pending.len(), 0); + assert_eq!(ops.cancellations.len(), 1); + assert_eq!(ops.cancellations[0].1, CancellationReason::VmReset); + drop(ops); + assert_eq!(vm.waiting_host_op_id(), None); +} + +#[test] +fn rejected_pending_result_cancels_bridge_owned_work() { + let ops = Arc::new(Mutex::new(TestAsyncOps::default())); + let mut vm = Vm::new(build_async_import_program(41)); + vm.bind_function("edge::async_add_one", Box::new(InvalidPendingFunction)); + vm.set_async_bridge(Box::new(TestAsyncBridge::new(ops.clone()))); + + vm.run() + .expect_err("zero host operation id should be rejected"); + assert_eq!( + ops.lock().unwrap().cancellations, + vec![(0, CancellationReason::ResourceClosed)] + ); + assert_eq!(vm.waiting_host_op_id(), None); +} + +#[tokio::test(flavor = "current_thread")] +async fn user_cancellation_reaches_host_bridge_and_clears_waiting_state() { + let ops = Arc::new(Mutex::new(TestAsyncOps::default())); + let calls = Arc::new(AtomicUsize::new(0)); + let mut vm = Vm::new(build_async_import_program(41)); + vm.bind_function( + "edge::async_add_one", + Box::new(AsyncAddOneFunction::new( + ops.clone(), + calls, + Duration::from_secs(60), + )), + ); + vm.set_async_bridge(Box::new(TestAsyncBridge::new(ops.clone()))); + + let op_id = match vm.run().expect("pending call") { + VmStatus::Waiting(op_id) => op_id, + status => panic!("expected waiting status, got {status:?}"), + }; + let error = vm + .wait_for_host_op_blocking_with_cancel(|| true) + .expect_err("user cancellation should stop the wait"); + assert!(error.to_string().contains("cancelled")); + let ops = ops.lock().unwrap(); + assert_eq!(ops.pending.len(), 0); + assert_eq!( + ops.cancellations, + vec![(op_id, CancellationReason::Requested)] + ); assert_eq!(vm.waiting_host_op_id(), None); + assert!(op_id > 0); } #[tokio::test(flavor = "current_thread")] diff --git a/tests/vm/vm_runtime_tests.rs b/tests/vm/vm_runtime_tests.rs index f9c6fcb4..8fa22ef1 100644 --- a/tests/vm/vm_runtime_tests.rs +++ b/tests/vm/vm_runtime_tests.rs @@ -23,6 +23,66 @@ fn non_yielding_returns_bool(_: &[Value]) -> Result { Ok(CallOutcome::Return(vm::CallReturn::one(Value::Bool(true)))) } +fn returns_registered_value(_: &[Value]) -> Result { + Ok(CallOutcome::Return(vm::CallReturn::one(Value::Int(42)))) +} + +#[test] +fn empty_registry_allows_functions_registered_by_the_embedder() { + let compiled = + compile_source("fn action() -> int; action();").expect("host call source should compile"); + let mut registry = HostFunctionRegistry::empty(); + registry.register_static_args("action", 0, returns_registered_value); + let mut vm = Vm::new(compiled.program); + registry + .bind_vm_cached(&mut vm) + .expect("custom registry should bind its registered import"); + + assert_eq!( + vm.run().expect("custom host call should run"), + VmStatus::Halted + ); + assert_eq!(vm.stack(), &[Value::Int(42)]); +} + +#[test] +fn empty_registry_preserves_default_builtin_capabilities() { + let compiled = compile_source("use bytes; bytes::from_array_u8([1, 2, 3]);") + .expect("bytes source should compile"); + let mut vm = Vm::new(compiled.program); + HostFunctionRegistry::empty() + .bind_vm_cached(&mut vm) + .expect("empty registry should bind builtin calls"); + + assert_eq!(vm.run().expect("builtin call should run"), VmStatus::Halted); + assert_eq!(vm.stack(), &[Value::bytes(vec![1, 2, 3])]); +} + +#[cfg(feature = "cranelift-jit")] +#[test] +fn restricted_builtin_capabilities_match_between_interpreter_and_aot() { + let source = "use bytes; bytes::from_array_u8([1, 2, 3]);"; + let program = compile_source(source) + .expect("bytes source should compile") + .program; + + let mut interpreter = Vm::new(program.clone()); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut interpreter) + .expect("restricted registry should bind"); + assert!(matches!( + interpreter.run(), + Err(vm::VmError::UnboundImport(_)) + )); + + let mut aot = Vm::new(program); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut aot) + .expect("restricted registry should bind"); + aot.compile_aot().expect("AOT compile should succeed"); + assert!(matches!(aot.run(), Err(vm::VmError::UnboundImport(_)))); +} + #[test] fn non_yielding_args_return_type_contract_is_enforced_before_jit_compilation() { let compiled = @@ -324,6 +384,48 @@ fn namespaced_builtin_io_call_can_be_overridden_by_host_binding() { assert_eq!(vm.stack(), &[Value::Bool(false)]); } +#[test] +fn builtin_override_does_not_bypass_restricted_capability_profile() { + struct ExistsOverride; + + impl HostFunction for ExistsOverride { + fn call(&mut self, _vm: &mut Vm, _args: &[Value]) -> Result { + Ok(CallOutcome::Return(vec![Value::Bool(false)].into())) + } + } + + let program = compile_source( + r#" + use io; + io::exists("request_body"); + "#, + ) + .expect("source should compile") + .program; + + let mut denied = Vm::new(program.clone()); + HostFunctionRegistry::restricted() + .bind_vm_cached(&mut denied) + .expect("restricted registry should bind"); + denied.bind_function("io::exists", Box::new(ExistsOverride)); + assert!(matches!(denied.run(), Err(vm::VmError::UnboundImport(_)))); + + let mut allowed_registry = HostFunctionRegistry::restricted(); + allowed_registry + .allow_builtin("io::exists") + .expect("IO builtin should be known"); + let mut allowed = Vm::new(program); + allowed_registry + .bind_vm_cached(&mut allowed) + .expect("allowlisted registry should bind"); + allowed.bind_function("io::exists", Box::new(ExistsOverride)); + assert_eq!( + allowed.run().expect("override should run"), + VmStatus::Halted + ); + assert_eq!(allowed.stack(), &[Value::Bool(false)]); +} + #[test] fn namespaced_builtin_json_encode_call_can_be_overridden_by_host_binding() { struct JsonEncodeOverride; diff --git a/tests/vm_tests.rs b/tests/vm_tests.rs index 47a3a179..8856a0df 100644 --- a/tests/vm_tests.rs +++ b/tests/vm_tests.rs @@ -1,6 +1,9 @@ #![cfg(feature = "runtime")] #![allow(clippy::duplicate_mod)] +#[path = "vm/ownership_tests.rs"] +mod ownership_tests; + #[path = "vm/drop_contract_tests.rs"] mod drop_contract_tests;