From 518c52c80919b28bd7221502dab93d9bdfe08744 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:45:45 +0300 Subject: [PATCH 01/38] test(multimodal): expect zero files when disabled Update the disable sentinel test to assert that a disabled limit keeps zero files rather than being clamped up to one, matching the intended behaviour that files_disabled exists to preserve. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/mod_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/mod_tests.rs b/crates/tinyagents-harness/src/multimodal/mod_tests.rs index 1b8a7624c..ad39bbdd0 100644 --- a/crates/tinyagents-harness/src/multimodal/mod_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/mod_tests.rs @@ -353,8 +353,8 @@ fn the_disable_sentinel_survives_the_clamp() { assert!(disabled.files_disabled()); assert_eq!( disabled.effective().0, - 1, - "the clamp lifts 0 to 1 — which is exactly why files_disabled exists" + 0, + "zero files must remain disabled after applying runtime limits" ); assert!(!FileLimits::default().files_disabled()); } From e8af402a68718bfd3173aa6d98d4c8bdd9b55488 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:46:19 +0300 Subject: [PATCH 02/38] fix(multimodal): preserve zero max_files as hard-disable in effective limits `FileLimits::effective` now returns zero for `max_files` instead of clamping it up to one, so the hard-disable sentinel survives the clamp and callers no longer need to check `files_disabled` first. Documentation was updated to reflect that the sentinel is preserved rather than lifted. Auto-committed-on: dragonfly --- .../src/multimodal/config.rs | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/config.rs b/crates/tinyagents-harness/src/multimodal/config.rs index 4df018334..0381271b9 100644 --- a/crates/tinyagents-harness/src/multimodal/config.rs +++ b/crates/tinyagents-harness/src/multimodal/config.rs @@ -15,11 +15,9 @@ //! //! ## The one asymmetry worth knowing //! -//! `max_files == 0` is a **hard-disable sentinel**, and it is checked before -//! the clamp rather than after. The clamp lifts `0` to `1`, so a caller that -//! only consulted [`FileLimits::effective`] would admit a single file marker -//! from a source that asked for none. [`FileLimits::files_disabled`] is the -//! check that must run first; [`resolve`](super::resolve) does exactly that. +//! `max_files == 0` is a **hard-disable sentinel**. It remains zero in the +//! effective limits, so every caller can enforce the same hard stop before +//! resolving a source. use serde::{Deserialize, Serialize}; @@ -76,9 +74,8 @@ impl ImageLimits { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(default)] pub struct FileLimits { - /// Maximum files accepted in one turn. Clamped to `1..=16` — except `0`, - /// which is the hard-disable sentinel checked by - /// [`FileLimits::files_disabled`] before any clamp applies. + /// Maximum files accepted in one turn. Clamped to `1..=16`; `0` disables + /// file resolution outright. pub max_files: usize, /// Maximum size of a single file, in MiB. Clamped to `1..=50`. pub max_file_size_mb: usize, @@ -128,10 +125,15 @@ pub fn default_allowed_file_mime_types() -> Vec { impl FileLimits { /// Clamp configured values into runtime bounds: - /// `(max_files, max_file_size_mb, max_extracted_text_chars)`. + /// `(max_files, max_file_size_mb, max_extracted_text_chars)`. A zero file + /// count is preserved as the hard-disable setting. pub fn effective(&self) -> (usize, usize, usize) { ( - self.max_files.clamp(1, 16), + if self.max_files == 0 { + 0 + } else { + self.max_files.clamp(1, 16) + }, self.max_file_size_mb.clamp(1, 50), self.max_extracted_text_chars.clamp(1_000, 200_000), ) @@ -144,10 +146,9 @@ impl FileLimits { /// `true` when file markers are disabled outright. /// - /// The sentinel exists so a host can refuse `[FILE:…]` resolution for a - /// turn whose text came from somewhere it does not trust, without editing - /// the operator's own allowlist. It must be consulted **before** - /// [`FileLimits::effective`], whose clamp lifts `0` to `1`. + /// The sentinel lets a host refuse media resolution for an untrusted turn + /// without editing the operator's own allowlist. It is also preserved by + /// [`FileLimits::effective`]. pub fn files_disabled(&self) -> bool { self.max_files == 0 } From 2f9a8a2bacfb314a1b323417bb5e47c889437596 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:46:57 +0300 Subject: [PATCH 03/38] feat(multimodal): add media request module Expose a new request module with media request configuration, model, source resolver, and image readout types so callers can build and resolve multimodal media requests. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/mod.rs | 4 + .../src/multimodal/request.rs | 489 ++++++++++++++++++ 2 files changed, 493 insertions(+) create mode 100644 crates/tinyagents-harness/src/multimodal/request.rs diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index 0d7354a4f..96bd187c9 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -77,6 +77,7 @@ pub mod error; pub mod markers; pub mod mime; pub mod payload; +pub mod request; pub mod png; pub use png::optimize_png_lossless; pub mod blob_stash; @@ -101,6 +102,9 @@ pub use markers::{ text_has_image_placeholders, }; pub use payload::{FilePayload, compose_multimodal_message, sha256_prefix, truncate_chars}; +pub use request::{ + MediaImageReadout, MediaRequestConfig, MediaRequestModel, MediaSourceResolver, +}; pub use resolve::{ NoTextExtractor, TextExtractor, resolve_attachment, resolve_file, resolve_image, }; diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs new file mode 100644 index 000000000..013acb106 --- /dev/null +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -0,0 +1,489 @@ +//! Prepare a disposable model request by resolving its user media references. +//! +//! The wrapper owns generic marker counting, capability checks, payload +//! rendering, document extraction hooks, and non-vision text fallbacks. Hosts +//! provide the scope and source resolver because only a host can authorize a +//! path, fetch a remote source, or decide which credentials apply. Scope is +//! obtained from the immutable `ChatModel` state for each call; it is never +//! inferred from message text or a media reference. + +use std::sync::Arc; + +use async_trait::async_trait; +use base64::{Engine as _, engine::general_purpose::STANDARD}; +use tinyinference_llm::{ + message::{ContentBlock, ImageRef, MediaRef, Message}, + model::{ + ChatModel, InputModality, InputSource, ModelProfile, ModelRequest, ModelResponse, + ModelStream, + }, +}; + +use super::{ + ArchiveFormat, FileLimits, FilePayload, ImageLimits, ResolvedAttachment, + TextExtractor, UnknownMimePolicy, + archive::{ArchiveLimits, inspect_archive}, + data_uri::encode_data_uri, + markers::{parse_file_markers, parse_image_markers}, + mime::{detect_image_mime, is_extractable_text_mime}, + payload::compose_multimodal_message, +}; + +/// Per-turn attachment limits copied from host configuration. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct MediaRequestConfig { + /// Image count, size, and remote-fetch limits. + pub images: ImageLimits, + /// File count, size, extraction, and remote-fetch limits. + pub files: FileLimits, +} + +/// Resolve one media source after the host has applied its access policy. +/// +/// Implementations own all path authorization and source access. `max_bytes` +/// is an additional upper bound the resolver must honor before allocating or +/// returning the body. `scope` is produced once for this invocation from the +/// immutable model state and is shared only by that invocation. +#[async_trait] +pub trait MediaSourceResolver: Send + Sync { + /// Resolve a source into bounded bytes and display metadata. + async fn resolve( + &self, + scope: &Scope, + modality: InputModality, + source: &str, + max_bytes: usize, + ) -> std::result::Result; +} + +/// Produce a text description for an image when the selected model cannot +/// accept image input. Model selection and any credentials remain host-owned. +#[async_trait] +pub trait MediaImageReadout: Send + Sync { + /// Return a faithful image description for a host-authorized image. + async fn readout( + &self, + scope: &Scope, + image: &ResolvedAttachment, + ) -> std::result::Result; +} + +/// A `ChatModel` decorator that resolves media on a request copy. +pub struct MediaRequestModel { + inner: Arc>, + config: MediaRequestConfig, + scope_for_state: Arc std::result::Result + Send + Sync>, + resolver: Arc>, + extractor: Arc, + image_readout: Option>>, +} + +impl MediaRequestModel +where + State: Send + Sync, + Scope: Send + Sync, +{ + /// Wrap `inner` with generic request preparation. + /// + /// The scope callback must return the caller's already-authorized scope + /// from immutable model state. It is called separately for each invoke or + /// stream, so concurrent calls cannot overwrite one another's scope. + pub fn new( + inner: Arc>, + config: MediaRequestConfig, + scope_for_state: impl Fn(&State) -> std::result::Result + + Send + + Sync + + 'static, + resolver: Arc>, + extractor: Arc, + ) -> Self { + Self { + inner, + config, + scope_for_state: Arc::new(scope_for_state), + resolver, + extractor, + image_readout: None, + } + } + + /// Install the host's image-to-text fallback for models without vision. + pub fn with_image_readout(mut self, readout: Arc>) -> Self { + self.image_readout = Some(readout); + self + } + + async fn prepare(&self, state: &State, mut request: ModelRequest) -> tinyinference_llm::Result { + let Some(latest_user) = request + .messages + .iter() + .rposition(|message| matches!(message, Message::User(_))) + else { + return Ok(request); + }; + + // Count the raw latest-turn media before deriving scope or calling any + // source resolver. This keeps configured limits meaningful for reads. + let (images, files) = count_user_media(&request.messages[latest_user]); + let (max_images, _) = self.config.images.effective(); + let (max_files, _, _) = self.config.files.effective(); + if self.config.files.files_disabled() && images + files > 0 { + return Err(model_error("attachments are disabled for this request")); + } + if images > max_images { + return Err(model_error(&format!( + "image attachment count exceeds configured limit: max={max_images}, found={images}" + ))); + } + if files > max_files { + return Err(model_error(&format!( + "file attachment count exceeds configured limit: max={max_files}, found={files}" + ))); + } + + let scope = (self.scope_for_state)(state).map_err(|error| model_error(&error))?; + for message in &mut request.messages { + let Message::User(user) = message else { + continue; + }; + let mut prepared = Vec::with_capacity(user.content.len()); + for block in std::mem::take(&mut user.content) { + match block { + ContentBlock::Text(text) => { + prepared.extend(self.prepare_text(&scope, &text).await?); + } + block @ (ContentBlock::Image(_) + | ContentBlock::Audio(_) + | ContentBlock::Video(_) + | ContentBlock::Document(_)) => { + prepared.extend(self.prepare_block(&scope, block).await?); + } + other => prepared.push(other), + } + } + user.content = prepared; + } + Ok(request) + } + + async fn prepare_text( + &self, + scope: &Scope, + text: &str, + ) -> tinyinference_llm::Result> { + let (without_images, images) = parse_image_markers(text); + let (clean_text, files) = parse_file_markers(&without_images); + if images.is_empty() && files.is_empty() { + return Ok(vec![ContentBlock::Text(text.to_string())]); + } + + let mut native_images = Vec::new(); + let mut image_fallback = Vec::new(); + for source in images { + let attachment = self + .resolve(scope, InputModality::Image, &source, self.config.images.max_image_bytes()) + .await?; + if self.native(InputModality::Image, &attachment.mime) { + let mime = validated_image_mime(&attachment)?; + native_images.push(ContentBlock::Image(ImageRef { + url: encode_data_uri(&mime, &attachment.bytes), + mime_type: Some(mime), + })); + } else { + image_fallback.push(self.image_fallback(scope, &attachment).await?); + } + } + + let mut file_payloads = Vec::new(); + let mut file_fallback_text = Vec::new(); + for source in files { + let attachment = self + .resolve(scope, InputModality::Document, &source, self.config.files.max_file_bytes()) + .await?; + if self.native(InputModality::Document, &attachment.mime) { + native_images.push(ContentBlock::Document(MediaRef::base64( + STANDARD.encode(&attachment.bytes), + attachment.mime, + ))); + } else { + let (payload, archive_text) = self.file_fallback(&attachment).await; + file_payloads.push(payload); + if let Some(text) = archive_text { + file_fallback_text.push(text); + } + } + } + + let composed = compose_multimodal_message(&clean_text, &[], &file_payloads); + let mut output = Vec::new(); + let mut text_output = composed; + for fallback in image_fallback.into_iter().chain(file_fallback_text) { + if !text_output.is_empty() { + text_output.push_str("\n\n"); + } + text_output.push_str(&fallback); + } + if !text_output.is_empty() { + output.push(ContentBlock::Text(text_output)); + } + output.extend(native_images); + Ok(output) + } + + async fn prepare_block( + &self, + scope: &Scope, + block: ContentBlock, + ) -> tinyinference_llm::Result> { + let (modality, source, hinted_mime) = match block { + ContentBlock::Image(image) => (InputModality::Image, image.url, image.mime_type), + ContentBlock::Audio(media) => media_source(InputModality::Audio, media)?, + ContentBlock::Video(media) => media_source(InputModality::Video, media)?, + ContentBlock::Document(media) => media_source(InputModality::Document, media)?, + _ => unreachable!("prepare_block only receives media blocks"), + }; + let max_bytes = match modality { + InputModality::Image => self.config.images.max_image_bytes(), + _ => self.config.files.max_file_bytes(), + }; + let mut attachment = self.resolve(scope, modality, &source, max_bytes).await?; + if attachment.mime == "application/octet-stream" + && let Some(mime) = hinted_mime + { + attachment.mime = mime; + } + if self.native(modality, &attachment.mime) { + return Ok(vec![native_block(modality, attachment)?]); + } + let fallback = match modality { + InputModality::Image => self.image_fallback(scope, &attachment).await?, + _ => { + let (payload, archive_text) = self.file_fallback(&attachment).await; + let rendered = compose_multimodal_message("", &[], &[payload]); + match archive_text { + Some(archive) if !rendered.is_empty() => format!("{rendered}\n\n{archive}"), + Some(archive) => archive, + None => rendered, + } + } + }; + Ok(vec![ContentBlock::Text(fallback)]) + } + + async fn resolve( + &self, + scope: &Scope, + modality: InputModality, + source: &str, + max_bytes: usize, + ) -> tinyinference_llm::Result { + let attachment = self + .resolver + .resolve(scope, modality, source, max_bytes) + .await + .map_err(|error| model_error(&error))?; + if attachment.bytes.len() > max_bytes { + return Err(model_error("attachment exceeds configured byte limit")); + } + if modality == InputModality::Image && attachment.bytes.is_empty() { + return Err(model_error("image reference has no bytes")); + } + Ok(attachment) + } + + fn native(&self, modality: InputModality, mime: &str) -> bool { + let advertised = self.inner.profile().is_some_and(|profile| match modality { + InputModality::Image => profile.modalities.image_in, + InputModality::Audio => profile.modalities.audio_in, + InputModality::Video => profile.modalities.video_in, + InputModality::Document => profile.modalities.document_in, + }); + advertised && self.inner.supports_input(modality, mime, InputSource::Base64) + } + + async fn image_fallback( + &self, + scope: &Scope, + image: &ResolvedAttachment, + ) -> tinyinference_llm::Result { + let mime = validated_image_mime(image)?; + let details = match &self.image_readout { + Some(readout) => readout + .readout(scope, image) + .await + .map_err(|error| model_error(&error))?, + None => "Image contents are unavailable to this text-only model.".to_string(), + }; + Ok(format!( + "[Image attachment: {} bytes; mime={mime}]\nImage readout (untrusted):\n{details}", + image.bytes.len() + )) + } + + async fn file_fallback( + &self, + attachment: &ResolvedAttachment, + ) -> (FilePayload, Option) { + if let Some(format) = ArchiveFormat::detect(&attachment.name, &attachment.mime, &attachment.bytes) { + let text = match inspect_archive(&attachment.bytes, format, &ArchiveLimits::default()) { + Ok(listing) => { + let mut output = String::from( + "Archive listing (names are untrusted; original was not extracted):", + ); + for entry in listing.entries { + output.push_str(&format!( + "\n{:?} {} bytes {}", + entry.kind, + entry.declared_size, + serde_json::to_string(&entry.name).unwrap_or_default() + )); + } + if let Some(truncation) = listing.truncation { + output.push_str(&format!("\nListing truncated: {truncation:?}")); + } + output + } + Err(error) => format!("Archive inspection unavailable: {error}"), + }; + return ( + FilePayload::from_resolved( + &attachment.bytes, + attachment.name.clone(), + attachment.mime.clone(), + None, + self.config.files.effective().2, + ), + Some(text), + ); + } + + let extracted = if is_extractable_text_mime(&attachment.mime) + || !self.extractor.handles(&attachment.mime) + { + None + } else { + self.extractor + .extract(&attachment.mime, &attachment.bytes) + .await + .ok() + }; + ( + FilePayload::from_resolved( + &attachment.bytes, + attachment.name.clone(), + attachment.mime.clone(), + extracted, + self.config.files.effective().2, + ), + None, + ) + } +} + +#[async_trait] +impl ChatModel for MediaRequestModel +where + State: Send + Sync, + Scope: Send + Sync, +{ + fn profile(&self) -> Option<&ModelProfile> { + self.inner.profile() + } + + fn supports_input(&self, modality: InputModality, mime: &str, source: InputSource) -> bool { + self.inner.supports_input(modality, mime, source) + } + + fn cache_identity(&self) -> Option { + self.inner.cache_identity() + } + + async fn invoke( + &self, + state: &State, + request: ModelRequest, + ) -> tinyinference_llm::Result { + self.inner.invoke(state, self.prepare(state, request).await?).await + } + + async fn stream( + &self, + state: &State, + request: ModelRequest, + ) -> tinyinference_llm::Result { + self.inner.stream(state, self.prepare(state, request).await?).await + } +} + +fn count_user_media(message: &Message) -> (usize, usize) { + let Message::User(user) = message else { + return (0, 0); + }; + user.content.iter().fold((0, 0), |(images, files), block| { + let (image_count, file_count) = match block { + ContentBlock::Text(text) => { + let (_, images) = parse_image_markers(text); + let (_, files) = parse_file_markers(text); + (images.len(), files.len()) + } + ContentBlock::Image(_) => (1, 0), + ContentBlock::Audio(_) | ContentBlock::Video(_) | ContentBlock::Document(_) => (0, 1), + _ => (0, 0), + }; + (images + image_count, files + file_count) + }) +} + +fn media_source( + modality: InputModality, + media: MediaRef, +) -> tinyinference_llm::Result<(InputModality, String, Option)> { + Ok(match media { + MediaRef::Path { path, media_type } => (modality, path, media_type), + MediaRef::Url { url, media_type } => (modality, url, media_type), + MediaRef::Base64 { data, media_type } => ( + modality, + format!("data:{media_type};base64,{data}"), + Some(media_type), + ), + }) +} + +fn native_block( + modality: InputModality, + attachment: ResolvedAttachment, +) -> tinyinference_llm::Result { + let mime = if modality == InputModality::Image { + validated_image_mime(&attachment)? + } else { + attachment.mime + }; + let data = STANDARD.encode(&attachment.bytes); + Ok(match modality { + InputModality::Image => ContentBlock::Image(ImageRef { + url: encode_data_uri(&mime, &attachment.bytes), + mime_type: Some(mime), + }), + InputModality::Audio => ContentBlock::Audio(MediaRef::base64(data, mime)), + InputModality::Video => ContentBlock::Video(MediaRef::base64(data, mime)), + InputModality::Document => ContentBlock::Document(MediaRef::base64(data, mime)), + }) +} + +fn validated_image_mime(attachment: &ResolvedAttachment) -> tinyinference_llm::Result { + let mime = detect_image_mime(None, &attachment.bytes, Some(&attachment.mime)) + .ok_or_else(|| model_error("image attachment type cannot be determined"))?; + if !super::ALLOWED_IMAGE_MIME_TYPES.contains(&mime.as_str()) { + return Err(model_error("image attachment MIME type is not supported")); + } + Ok(mime) +} + +fn model_error(message: &str) -> tinyinference_llm::Error { + tinyinference_llm::Error::Model(message.to_string()) +} + +#[cfg(test)] +#[path = "request_tests.rs"] +mod tests; From d03be1477ee21bc447b464c0fd56b4e52ac91b47 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:47:35 +0300 Subject: [PATCH 04/38] test(multimodal): add request tests Add tests covering multimodal request construction and handling. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 318 ++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 crates/tinyagents-harness/src/multimodal/request_tests.rs diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs new file mode 100644 index 000000000..a89872c6e --- /dev/null +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -0,0 +1,318 @@ +use super::*; +use std::{collections::HashMap, io::Write, sync::atomic::{AtomicUsize, Ordering}, sync::Mutex}; + +use futures::StreamExt; +use tinyinference_llm::{ + message::{AssistantMessage, UserMessage}, + model::{Modalities, ModelStreamItem}, +}; + +#[derive(Default)] +struct CaptureModel { + requests: Mutex>, + profile: ModelProfile, + native_images: bool, +} + +#[async_trait] +impl ChatModel for CaptureModel { + fn profile(&self) -> Option<&ModelProfile> { + Some(&self.profile) + } + + fn supports_input(&self, modality: InputModality, _mime: &str, source: InputSource) -> bool { + modality == InputModality::Image && self.native_images && source == InputSource::Base64 + } + + fn cache_identity(&self) -> Option { + Some("capture-model".into()) + } + + async fn invoke( + &self, + _state: &String, + request: ModelRequest, + ) -> tinyinference_llm::Result { + self.requests.lock().unwrap().push(request); + Ok(ModelResponse { + message: AssistantMessage::default(), + ..ModelResponse::default() + }) + } +} + +impl CaptureModel { + fn new(native_images: bool) -> Self { + let mut profile = ModelProfile::default(); + profile.modalities = Modalities { + image_in: native_images, + ..Modalities::default() + }; + Self { + requests: Mutex::default(), + profile, + native_images, + } + } +} + +#[derive(Default)] +struct Resolver { + sources: HashMap, + calls: AtomicUsize, + scopes: Mutex>, +} + +#[async_trait] +impl MediaSourceResolver for Resolver { + async fn resolve( + &self, + scope: &String, + _modality: InputModality, + source: &str, + max_bytes: usize, + ) -> std::result::Result { + self.calls.fetch_add(1, Ordering::SeqCst); + self.scopes.lock().unwrap().push(scope.clone()); + let attachment = self + .sources + .get(source) + .cloned() + .ok_or_else(|| format!("missing test source {source}"))?; + if attachment.bytes.len() > max_bytes { + return Err("source exceeds test limit".into()); + } + Ok(attachment) + } +} + +#[derive(Default)] +struct Extractor { + inputs: Mutex)>>, +} + +#[async_trait] +impl TextExtractor for Extractor { + fn handles(&self, mime: &str) -> bool { + mime == "application/pdf" + } + + async fn extract(&self, mime: &str, bytes: &[u8]) -> std::result::Result { + self.inputs + .lock() + .unwrap() + .push((mime.to_string(), bytes.to_vec())); + Ok("PDF text layer from fake parser".into()) + } +} + +struct Readout; + +#[async_trait] +impl MediaImageReadout for Readout { + async fn readout( + &self, + _scope: &String, + _image: &ResolvedAttachment, + ) -> std::result::Result { + Ok("a small red square".into()) + } +} + +fn attachment(name: &str, mime: &str, bytes: &[u8]) -> ResolvedAttachment { + ResolvedAttachment { + bytes: bytes.to_vec(), + name: name.into(), + mime: mime.into(), + size_bytes: bytes.len(), + } +} + +fn request(text: &str) -> ModelRequest { + ModelRequest::new(vec![Message::User(UserMessage { + content: vec![ContentBlock::Text(text.into())], + })]) +} + +fn wrapper( + model: Arc, + config: MediaRequestConfig, + resolver: Arc, + extractor: Arc, +) -> MediaRequestModel { + MediaRequestModel::new( + model, + config, + |state: &String| Ok(state.clone()), + resolver, + extractor, + ) +} + +#[tokio::test] +async fn hard_zero_rejects_before_any_source_read() { + let model = Arc::new(CaptureModel::new(false)); + let resolver = Arc::new(Resolver::default()); + let disabled = MediaRequestConfig { + files: FileLimits { + max_files: 0, + ..FileLimits::default() + }, + ..MediaRequestConfig::default() + }; + let wrapped = wrapper(model, disabled, resolver.clone(), Arc::default()); + + let error = wrapped + .invoke(&"tenant-a".into(), request("see [IMAGE:secret]")) + .await + .expect_err("hard-zero disables all media reads"); + + assert!(error.to_string().contains("attachments are disabled")); + assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn ordinary_count_limit_is_checked_before_resolving_any_marker() { + let model = Arc::new(CaptureModel::new(true)); + let resolver = Arc::new(Resolver::default()); + let config = MediaRequestConfig { + images: ImageLimits { + max_images: 1, + ..ImageLimits::default() + }, + ..MediaRequestConfig::default() + }; + let wrapped = wrapper(model, config, resolver.clone(), Arc::default()); + + let error = wrapped + .invoke(&"tenant-a".into(), request("[IMAGE:first] [IMAGE:second]")) + .await + .expect_err("two images exceed the configured cap"); + + assert!(error.to_string().contains("max=1, found=2")); + assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn invoke_and_stream_prepare_disposable_requests_without_changing_history() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + sources.insert( + "image".into(), + attachment("tiny.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + resolver, + Arc::default(), + ); + let original = request("caption [IMAGE:image]"); + let canonical = original.clone(); + + wrapped.invoke(&"tenant-a".into(), original.clone()).await.unwrap(); + let stream = wrapped.stream(&"tenant-a".into(), original.clone()).await.unwrap(); + let items: Vec = stream.collect().await; + + assert!(matches!(items.first(), Some(ModelStreamItem::Started))); + assert_eq!(original, canonical, "the caller's request remains canonical"); + let calls = model.requests.lock().unwrap(); + assert_eq!(calls.len(), 2); + for prepared in calls.iter() { + let Message::User(user) = &prepared.messages[0] else { + panic!("user request retained") + }; + assert!(user.content.iter().any(|block| matches!(block, ContentBlock::Image(_)))); + assert!(!user.content.iter().any(|block| { + matches!(block, ContentBlock::Text(text) if text.contains("[IMAGE:")) + })); + } + assert_eq!(wrapped.cache_identity().as_deref(), Some("capture-model")); + assert!(wrapped.profile().is_some()); +} + +#[tokio::test] +async fn concurrent_calls_keep_their_immutable_scope_separate() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + sources.insert("image".into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); + let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let wrapped = wrapper(model, MediaRequestConfig::default(), resolver.clone(), Arc::default()); + + let (a, b) = tokio::join!( + wrapped.invoke(&"scope-a".into(), request("[IMAGE:image]")), + wrapped.invoke(&"scope-b".into(), request("[IMAGE:image]")), + ); + + a.unwrap(); + b.unwrap(); + let mut scopes = resolver.scopes.lock().unwrap().clone(); + scopes.sort(); + assert_eq!(scopes, vec!["scope-a", "scope-b"]); +} + +#[tokio::test] +async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { + let model = Arc::new(CaptureModel::new(false)); + let pdf = b"%PDF-1.7 a meaningful fake PDF body"; + let mut sources = HashMap::new(); + sources.insert("image".into(), attachment("red.png", "image/png", b"\x89PNG\r\n\x1a\n")); + sources.insert("pdf".into(), attachment("brief.pdf", "application/pdf", pdf)); + let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let extractor = Arc::new(Extractor::default()); + let wrapped = wrapper(model.clone(), MediaRequestConfig::default(), resolver, extractor.clone()) + .with_image_readout(Arc::new(Readout)); + + wrapped + .invoke(&"tenant-a".into(), request("[IMAGE:image] [FILE:pdf]")) + .await + .unwrap(); + + let calls = model.requests.lock().unwrap(); + let rendered = calls[0].messages[0].text(); + assert!(rendered.contains("a small red square")); + assert!(rendered.contains("PDF text layer from fake parser")); + assert_eq!( + extractor.inputs.lock().unwrap().as_slice(), + &[("application/pdf".into(), pdf.to_vec())] + ); +} + +#[tokio::test] +async fn nonvision_archive_fallback_lists_entries_without_extracting_them() { + let mut bytes = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut bytes); + zip.start_file("notes/readme.txt", zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(b"do not extract this archive").unwrap(); + zip.finish().unwrap(); + } + let model = Arc::new(CaptureModel::new(false)); + let mut sources = HashMap::new(); + sources.insert( + "archive".into(), + attachment("bundle.zip", "application/zip", bytes.get_ref()), + ); + let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + resolver, + Arc::default(), + ); + + wrapped + .invoke(&"tenant-a".into(), request("[FILE:archive]")) + .await + .unwrap(); + + let rendered = model.requests.lock().unwrap()[0].messages[0].text(); + assert!(rendered.contains("Archive listing")); + assert!(rendered.contains("notes/readme.txt")); + assert!(rendered.contains("original was not extracted")); +} From 3c20e3b07ac7f9d46dfb95a0d5282dc618c399eb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:48:13 +0300 Subject: [PATCH 05/38] test(multimodal): build requests via struct literal and inline text extraction Test helpers now construct ModelRequest with a struct literal and default fields, and the two fallback tests extract rendered text directly from the user message content blocks instead of relying on a removed helper. This keeps the tests compiling after the request type and text accessor changed. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 2 +- .../src/multimodal/request_tests.rs | 36 ++++++++++++++++--- 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 013acb106..94552a0b1 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -21,7 +21,7 @@ use tinyinference_llm::{ use super::{ ArchiveFormat, FileLimits, FilePayload, ImageLimits, ResolvedAttachment, - TextExtractor, UnknownMimePolicy, + TextExtractor, archive::{ArchiveLimits, inspect_archive}, data_uri::encode_data_uri, markers::{parse_file_markers, parse_image_markers}, diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index a89872c6e..397708f65 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -129,9 +129,12 @@ fn attachment(name: &str, mime: &str, bytes: &[u8]) -> ResolvedAttachment { } fn request(text: &str) -> ModelRequest { - ModelRequest::new(vec![Message::User(UserMessage { - content: vec![ContentBlock::Text(text.into())], - })]) + ModelRequest { + messages: vec![Message::User(UserMessage { + content: vec![ContentBlock::Text(text.into())], + })], + ..ModelRequest::default() + } } fn wrapper( @@ -273,7 +276,18 @@ async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { .unwrap(); let calls = model.requests.lock().unwrap(); - let rendered = calls[0].messages[0].text(); + let rendered = match &calls[0].messages[0] { + Message::User(user) => user + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"), + _ => panic!("prepared user message retained"), + }; assert!(rendered.contains("a small red square")); assert!(rendered.contains("PDF text layer from fake parser")); assert_eq!( @@ -311,7 +325,19 @@ async fn nonvision_archive_fallback_lists_entries_without_extracting_them() { .await .unwrap(); - let rendered = model.requests.lock().unwrap()[0].messages[0].text(); + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("prepared user message retained") + }; + let rendered = user + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); assert!(rendered.contains("Archive listing")); assert!(rendered.contains("notes/readme.txt")); assert!(rendered.contains("original was not extracted")); From f0eb6cc7d57a13859f7c05212e07af20446f319b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:48:54 +0300 Subject: [PATCH 06/38] fix(multimodal): clamp max_files to at least one The effective file limit no longer special-cases zero, so a configured max_files of 0 now clamps to 1 instead of disabling uploads. Tests were updated to compare serialized requests and to use owned scope strings. Auto-committed-on: dragonfly --- .../src/multimodal/config.rs | 6 +----- .../src/multimodal/request_tests.rs | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/config.rs b/crates/tinyagents-harness/src/multimodal/config.rs index 0381271b9..1614c2984 100644 --- a/crates/tinyagents-harness/src/multimodal/config.rs +++ b/crates/tinyagents-harness/src/multimodal/config.rs @@ -129,11 +129,7 @@ impl FileLimits { /// count is preserved as the hard-disable setting. pub fn effective(&self) -> (usize, usize, usize) { ( - if self.max_files == 0 { - 0 - } else { - self.max_files.clamp(1, 16) - }, + self.max_files.clamp(1, 16), self.max_file_size_mb.clamp(1, 50), self.max_extracted_text_chars.clamp(1_000, 200_000), ) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 397708f65..7d942a0f3 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, io::Write, sync::atomic::{AtomicUsize, Ordering} use futures::StreamExt; use tinyinference_llm::{ - message::{AssistantMessage, UserMessage}, + message::UserMessage, model::{Modalities, ModelStreamItem}, }; @@ -34,10 +34,7 @@ impl ChatModel for CaptureModel { request: ModelRequest, ) -> tinyinference_llm::Result { self.requests.lock().unwrap().push(request); - Ok(ModelResponse { - message: AssistantMessage::default(), - ..ModelResponse::default() - }) + Ok(ModelResponse::assistant("")) } } @@ -222,7 +219,11 @@ async fn invoke_and_stream_prepare_disposable_requests_without_changing_history( let items: Vec = stream.collect().await; assert!(matches!(items.first(), Some(ModelStreamItem::Started))); - assert_eq!(original, canonical, "the caller's request remains canonical"); + assert_eq!( + serde_json::to_value(&original).unwrap(), + serde_json::to_value(&canonical).unwrap(), + "the caller's request remains canonical" + ); let calls = model.requests.lock().unwrap(); assert_eq!(calls.len(), 2); for prepared in calls.iter() { @@ -246,9 +247,11 @@ async fn concurrent_calls_keep_their_immutable_scope_separate() { let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); let wrapped = wrapper(model, MediaRequestConfig::default(), resolver.clone(), Arc::default()); + let scope_a = "scope-a".to_string(); + let scope_b = "scope-b".to_string(); let (a, b) = tokio::join!( - wrapped.invoke(&"scope-a".into(), request("[IMAGE:image]")), - wrapped.invoke(&"scope-b".into(), request("[IMAGE:image]")), + wrapped.invoke(&scope_a, request("[IMAGE:image]")), + wrapped.invoke(&scope_b, request("[IMAGE:image]")), ); a.unwrap(); From d63e9a6b8b3ac76adc5e67db524eb5410ecfb0f5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:49:05 +0300 Subject: [PATCH 07/38] fix(multimodal): preserve zero max_files as hard disable The effective file limit now returns zero when max_files is zero instead of clamping it up to one, so a zero setting keeps disabling file attachments rather than silently allowing a single file. The remaining changes are formatting and import reordering with no behaviour change. Auto-committed-on: dragonfly --- .../src/multimodal/config.rs | 6 +- .../tinyagents-harness/src/multimodal/mod.rs | 6 +- .../src/multimodal/request.rs | 45 ++++++++---- .../src/multimodal/request_tests.rs | 69 +++++++++++++++---- 4 files changed, 95 insertions(+), 31 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/config.rs b/crates/tinyagents-harness/src/multimodal/config.rs index 1614c2984..0381271b9 100644 --- a/crates/tinyagents-harness/src/multimodal/config.rs +++ b/crates/tinyagents-harness/src/multimodal/config.rs @@ -129,7 +129,11 @@ impl FileLimits { /// count is preserved as the hard-disable setting. pub fn effective(&self) -> (usize, usize, usize) { ( - self.max_files.clamp(1, 16), + if self.max_files == 0 { + 0 + } else { + self.max_files.clamp(1, 16) + }, self.max_file_size_mb.clamp(1, 50), self.max_extracted_text_chars.clamp(1_000, 200_000), ) diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index 96bd187c9..6c36e4b7b 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -77,8 +77,8 @@ pub mod error; pub mod markers; pub mod mime; pub mod payload; -pub mod request; pub mod png; +pub mod request; pub use png::optimize_png_lossless; pub mod blob_stash; pub mod resolve; @@ -102,9 +102,7 @@ pub use markers::{ text_has_image_placeholders, }; pub use payload::{FilePayload, compose_multimodal_message, sha256_prefix, truncate_chars}; -pub use request::{ - MediaImageReadout, MediaRequestConfig, MediaRequestModel, MediaSourceResolver, -}; +pub use request::{MediaImageReadout, MediaRequestConfig, MediaRequestModel, MediaSourceResolver}; pub use resolve::{ NoTextExtractor, TextExtractor, resolve_attachment, resolve_file, resolve_image, }; diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 94552a0b1..cfa4719ee 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -20,8 +20,7 @@ use tinyinference_llm::{ }; use super::{ - ArchiveFormat, FileLimits, FilePayload, ImageLimits, ResolvedAttachment, - TextExtractor, + ArchiveFormat, FileLimits, FilePayload, ImageLimits, ResolvedAttachment, TextExtractor, archive::{ArchiveLimits, inspect_archive}, data_uri::encode_data_uri, markers::{parse_file_markers, parse_image_markers}, @@ -91,10 +90,7 @@ where pub fn new( inner: Arc>, config: MediaRequestConfig, - scope_for_state: impl Fn(&State) -> std::result::Result - + Send - + Sync - + 'static, + scope_for_state: impl Fn(&State) -> std::result::Result + Send + Sync + 'static, resolver: Arc>, extractor: Arc, ) -> Self { @@ -114,7 +110,11 @@ where self } - async fn prepare(&self, state: &State, mut request: ModelRequest) -> tinyinference_llm::Result { + async fn prepare( + &self, + state: &State, + mut request: ModelRequest, + ) -> tinyinference_llm::Result { let Some(latest_user) = request .messages .iter() @@ -182,7 +182,12 @@ where let mut image_fallback = Vec::new(); for source in images { let attachment = self - .resolve(scope, InputModality::Image, &source, self.config.images.max_image_bytes()) + .resolve( + scope, + InputModality::Image, + &source, + self.config.images.max_image_bytes(), + ) .await?; if self.native(InputModality::Image, &attachment.mime) { let mime = validated_image_mime(&attachment)?; @@ -199,7 +204,12 @@ where let mut file_fallback_text = Vec::new(); for source in files { let attachment = self - .resolve(scope, InputModality::Document, &source, self.config.files.max_file_bytes()) + .resolve( + scope, + InputModality::Document, + &source, + self.config.files.max_file_bytes(), + ) .await?; if self.native(InputModality::Document, &attachment.mime) { native_images.push(ContentBlock::Document(MediaRef::base64( @@ -299,7 +309,10 @@ where InputModality::Video => profile.modalities.video_in, InputModality::Document => profile.modalities.document_in, }); - advertised && self.inner.supports_input(modality, mime, InputSource::Base64) + advertised + && self + .inner + .supports_input(modality, mime, InputSource::Base64) } async fn image_fallback( @@ -325,7 +338,9 @@ where &self, attachment: &ResolvedAttachment, ) -> (FilePayload, Option) { - if let Some(format) = ArchiveFormat::detect(&attachment.name, &attachment.mime, &attachment.bytes) { + if let Some(format) = + ArchiveFormat::detect(&attachment.name, &attachment.mime, &attachment.bytes) + { let text = match inspect_archive(&attachment.bytes, format, &ArchiveLimits::default()) { Ok(listing) => { let mut output = String::from( @@ -404,7 +419,9 @@ where state: &State, request: ModelRequest, ) -> tinyinference_llm::Result { - self.inner.invoke(state, self.prepare(state, request).await?).await + self.inner + .invoke(state, self.prepare(state, request).await?) + .await } async fn stream( @@ -412,7 +429,9 @@ where state: &State, request: ModelRequest, ) -> tinyinference_llm::Result { - self.inner.stream(state, self.prepare(state, request).await?).await + self.inner + .stream(state, self.prepare(state, request).await?) + .await } } diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 7d942a0f3..6ada56380 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -1,5 +1,10 @@ use super::*; -use std::{collections::HashMap, io::Write, sync::atomic::{AtomicUsize, Ordering}, sync::Mutex}; +use std::{ + collections::HashMap, + io::Write, + sync::Mutex, + sync::atomic::{AtomicUsize, Ordering}, +}; use futures::StreamExt; use tinyinference_llm::{ @@ -214,8 +219,14 @@ async fn invoke_and_stream_prepare_disposable_requests_without_changing_history( let original = request("caption [IMAGE:image]"); let canonical = original.clone(); - wrapped.invoke(&"tenant-a".into(), original.clone()).await.unwrap(); - let stream = wrapped.stream(&"tenant-a".into(), original.clone()).await.unwrap(); + wrapped + .invoke(&"tenant-a".into(), original.clone()) + .await + .unwrap(); + let stream = wrapped + .stream(&"tenant-a".into(), original.clone()) + .await + .unwrap(); let items: Vec = stream.collect().await; assert!(matches!(items.first(), Some(ModelStreamItem::Started))); @@ -230,7 +241,11 @@ async fn invoke_and_stream_prepare_disposable_requests_without_changing_history( let Message::User(user) = &prepared.messages[0] else { panic!("user request retained") }; - assert!(user.content.iter().any(|block| matches!(block, ContentBlock::Image(_)))); + assert!( + user.content + .iter() + .any(|block| matches!(block, ContentBlock::Image(_))) + ); assert!(!user.content.iter().any(|block| { matches!(block, ContentBlock::Text(text) if text.contains("[IMAGE:")) })); @@ -243,9 +258,20 @@ async fn invoke_and_stream_prepare_disposable_requests_without_changing_history( async fn concurrent_calls_keep_their_immutable_scope_separate() { let model = Arc::new(CaptureModel::new(true)); let mut sources = HashMap::new(); - sources.insert("image".into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); - let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); - let wrapped = wrapper(model, MediaRequestConfig::default(), resolver.clone(), Arc::default()); + sources.insert( + "image".into(), + attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let wrapped = wrapper( + model, + MediaRequestConfig::default(), + resolver.clone(), + Arc::default(), + ); let scope_a = "scope-a".to_string(); let scope_b = "scope-b".to_string(); @@ -266,12 +292,26 @@ async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { let model = Arc::new(CaptureModel::new(false)); let pdf = b"%PDF-1.7 a meaningful fake PDF body"; let mut sources = HashMap::new(); - sources.insert("image".into(), attachment("red.png", "image/png", b"\x89PNG\r\n\x1a\n")); - sources.insert("pdf".into(), attachment("brief.pdf", "application/pdf", pdf)); - let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + sources.insert( + "image".into(), + attachment("red.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + sources.insert( + "pdf".into(), + attachment("brief.pdf", "application/pdf", pdf), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); let extractor = Arc::new(Extractor::default()); - let wrapped = wrapper(model.clone(), MediaRequestConfig::default(), resolver, extractor.clone()) - .with_image_readout(Arc::new(Readout)); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + resolver, + extractor.clone(), + ) + .with_image_readout(Arc::new(Readout)); wrapped .invoke(&"tenant-a".into(), request("[IMAGE:image] [FILE:pdf]")) @@ -315,7 +355,10 @@ async fn nonvision_archive_fallback_lists_entries_without_extracting_them() { "archive".into(), attachment("bundle.zip", "application/zip", bytes.get_ref()), ); - let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); let wrapped = wrapper( model.clone(), MediaRequestConfig::default(), From af5f06cef7dfe030bbd8721e133df2ec7184c28c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:50:16 +0300 Subject: [PATCH 08/38] fix(multimodal): enforce remote fetch policy and preserve hard disable Remote attachment sources are now rejected before resolution when the relevant allow_remote_fetch setting is off, and max_files == 0 is kept at zero by effective() so a source asking for no attachments cannot admit one. The deferred fetch call is also forwarded to the inner model. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/mod.rs | 7 +++---- .../src/multimodal/request.rs | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index 6c36e4b7b..68b37b567 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -65,10 +65,9 @@ //! 1. **Count before resolving.** [`config::FileLimits::files_disabled`] and the //! per-turn caps are checked against the raw markers, before any read //! happens. A cap enforced after the fetch is not a cap. -//! 2. **Check the sentinel before the clamp.** `max_files == 0` means *none*, -//! and [`config::FileLimits::effective`] clamps it to 1. Consulting only the -//! clamped value admits exactly one attachment from a source that asked for -//! zero. +//! 2. **Preserve the hard disable.** `max_files == 0` means *none* and stays +//! zero in [`config::FileLimits::effective`]. The request decorator checks +//! this before calling a host resolver. pub mod archive; pub mod config; diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index cfa4719ee..7fbb189e9 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -288,6 +288,16 @@ where source: &str, max_bytes: usize, ) -> tinyinference_llm::Result { + let remote = source.starts_with("http://") || source.starts_with("https://"); + let remote_allowed = match modality { + InputModality::Image => self.config.images.allow_remote_fetch, + InputModality::Audio | InputModality::Video | InputModality::Document => { + self.config.files.allow_remote_fetch + } + }; + if remote && !remote_allowed { + return Err(model_error("remote attachment fetch is disabled")); + } let attachment = self .resolver .resolve(scope, modality, source, max_bytes) @@ -433,6 +443,13 @@ where .stream(state, self.prepare(state, request).await?) .await } + + async fn fetch_deferred( + &self, + handle: &tinyinference_llm::model::DeferredHandle, + ) -> tinyinference_llm::Result { + self.inner.fetch_deferred(handle).await + } } fn count_user_media(message: &Message) -> (usize, usize) { From 2d16afc07248e08de296d5744e68012e5e3f85fe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:50:29 +0300 Subject: [PATCH 09/38] docs(multimodal): document the media request model and host seams Adds the request module to the architecture overview and file table, covering MediaRequestModel and the resolver/readout seams the host supplies. Rewrites the limits note to reflect that max_files == 0 now stays zero in FileLimits::effective and is checked before any source read, and clarifies which responsibilities remain with the host. Auto-committed-on: dragonfly --- .../src/multimodal/README.md | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/README.md b/crates/tinyagents-harness/src/multimodal/README.md index 03958304e..e77058f30 100644 --- a/crates/tinyagents-harness/src/multimodal/README.md +++ b/crates/tinyagents-harness/src/multimodal/README.md @@ -45,6 +45,12 @@ format contributes a header naming it plus a content hash. [`compose_multimodal_message`] (renders the final provider-bound message), and supporting helpers (`truncate_chars`, `sha256_prefix`, `format_size`, `escape_attr`). +- [`request`] — [`MediaRequestModel`], a `ChatModel` decorator that prepares a + disposable request, checks counts before reads, preserves per-call scope, + dispatches native media only when both model and transport support it, and + renders text/archive/image fallbacks. The host supplies + [`MediaSourceResolver`] for authorized source access and may supply + [`MediaImageReadout`] for its chosen non-vision image description path. - [`error`] — [`MultimodalError`] and its [`Result`] alias; every variant carries the offending `input` verbatim so a multi-attachment turn's failure is attributable. @@ -66,6 +72,7 @@ format contributes a header naming it plus a content hash. | `archive/zip_admission.rs` | Allocation-free ZIP preflight and admitted reader. | | `png.rs` | Optional lossless PNG optimization. | | `payload.rs` | `FilePayload`, message composition, truncation, hashing. | +| `request.rs` | `MediaRequestModel` and host resolver/readout seams. | | `error.rs` | `MultimodalError`, `Result`. | | `mod_tests.rs`, `resolve_tests.rs`, `archive/ops_tests.rs`, `png_tests.rs`, `blob_stash_tests.rs` | Marker, MIME, size, path, intake, archive, PNG and stash behavior. | @@ -74,15 +81,14 @@ format contributes a header naming it plus a content hash. - **Count before resolving.** `FileLimits::files_disabled` and the per-turn count caps must be checked against the raw markers, before any read happens — a cap enforced after the fetch is not a cap. -- **Check the sentinel before the clamp.** `max_files == 0` means *none*; - `FileLimits::effective` clamps it up to `1`. Consulting only the clamped - value would admit one attachment from a source that asked for zero. -- What stays with the host, deliberately: the `TextExtractor` implementation (which parser, if any, and its - timeout), the attachment stash (where bytes live between ingress and - dispatch), and message-level marker counting (only the host knows its - message type). This module never decides which local paths may be read — - that is `FileLimits::files_disabled`'s lever, not a filesystem allowlist - here. +- **Preserve the hard disable.** `max_files == 0` means *none* and remains + zero in `FileLimits::effective`. `MediaRequestModel` checks it before asking + the host source resolver to read anything. +- What stays with the host, deliberately: the `TextExtractor` implementation + (which parser, if any, and its timeout), the attachment stash (where bytes + live between ingress and dispatch), image-readout model choice, source + authorization, and access scope. `MediaRequestModel` counts typed message + markers but never decides which local paths may be read. - Opt-in remote image and file URLs pass TinyTools' URL and DNS guard. The resolver pins the vetted addresses in its own direct HTTP client and refuses redirects. It uses 30-second request and 10-second connection timeouts. From b92bca7dedf5d109ed8bb78b8a2f87bc41dd6d05 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:52:11 +0300 Subject: [PATCH 10/38] fix(multimodal): reject media anywhere in the request when files are disabled The hard-zero check only inspected the latest user message, so attachments referenced earlier in the conversation slipped past it. It now scans every message before rejecting, and tests cover the historical-media case plus request immutability on resolution errors and cancellation. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 6 +- .../src/multimodal/request_tests.rs | 90 +++++++++++++++++++ 2 files changed, 95 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 7fbb189e9..847affd63 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -128,7 +128,11 @@ where let (images, files) = count_user_media(&request.messages[latest_user]); let (max_images, _) = self.config.images.effective(); let (max_files, _, _) = self.config.files.effective(); - if self.config.files.files_disabled() && images + files > 0 { + let request_has_media = request.messages.iter().any(|message| { + let (images, files) = count_user_media(message); + images + files > 0 + }); + if self.config.files.files_disabled() && request_has_media { return Err(model_error("attachments are disabled for this request")); } if images > max_images { diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 6ada56380..afd594757 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -65,6 +65,25 @@ struct Resolver { scopes: Mutex>, } +#[derive(Default)] +struct PendingResolver { + entered: tokio::sync::Notify, +} + +#[async_trait] +impl MediaSourceResolver for PendingResolver { + async fn resolve( + &self, + _scope: &String, + _modality: InputModality, + _source: &str, + _max_bytes: usize, + ) -> std::result::Result { + self.entered.notify_one(); + std::future::pending().await + } +} + #[async_trait] impl MediaSourceResolver for Resolver { async fn resolve( @@ -176,6 +195,35 @@ async fn hard_zero_rejects_before_any_source_read() { assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); } +#[tokio::test] +async fn hard_zero_also_blocks_media_only_present_in_history() { + let model = Arc::new(CaptureModel::new(false)); + let resolver = Arc::new(Resolver::default()); + let disabled = MediaRequestConfig { + files: FileLimits { + max_files: 0, + ..FileLimits::default() + }, + ..MediaRequestConfig::default() + }; + let wrapped = wrapper(model, disabled, resolver.clone(), Arc::default()); + let mut request = request("latest text only"); + request.messages.insert( + 0, + Message::User(UserMessage { + content: vec![ContentBlock::Text("earlier [IMAGE:secret]".into())], + }), + ); + + let error = wrapped + .invoke(&"tenant-a".into(), request) + .await + .expect_err("hard-zero disables historical media too"); + + assert!(error.to_string().contains("attachments are disabled")); + assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn ordinary_count_limit_is_checked_before_resolving_any_marker() { let model = Arc::new(CaptureModel::new(true)); @@ -254,6 +302,48 @@ async fn invoke_and_stream_prepare_disposable_requests_without_changing_history( assert!(wrapped.profile().is_some()); } +#[tokio::test] +async fn resolution_error_and_cancellation_leave_the_canonical_request_unchanged() { + let model = Arc::new(CaptureModel::new(true)); + let missing_resolver = Arc::new(Resolver::default()); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + missing_resolver, + Arc::default(), + ); + let original = request("[IMAGE:unavailable]"); + let canonical = serde_json::to_value(&original).unwrap(); + assert!( + wrapped + .invoke(&"tenant-a".into(), original.clone()) + .await + .is_err() + ); + assert_eq!(serde_json::to_value(&original).unwrap(), canonical); + assert!(model.requests.lock().unwrap().is_empty()); + + let pending = Arc::new(PendingResolver::default()); + let cancel_model = Arc::new(CaptureModel::new(true)); + let canceled_wrapper = MediaRequestModel::new( + cancel_model, + MediaRequestConfig::default(), + |state: &String| Ok(state.clone()), + pending.clone(), + Arc::default(), + ); + let canonical_after_cancel = serde_json::to_value(&original).unwrap(); + let call = { + let wrapped = canceled_wrapper; + let request = original.clone(); + tokio::spawn(async move { wrapped.invoke(&"tenant-a".into(), request).await }) + }; + pending.entered.notified().await; + call.abort(); + assert!(call.await.unwrap_err().is_cancelled()); + assert_eq!(serde_json::to_value(&original).unwrap(), canonical_after_cancel); +} + #[tokio::test] async fn concurrent_calls_keep_their_immutable_scope_separate() { let model = Arc::new(CaptureModel::new(true)); From 5c019bd13f31fa22ea48b9a2dd08ea9d3023fc04 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:53:20 +0300 Subject: [PATCH 11/38] feat(multimodal): bound historical media resolution with host fallback Historical media blocks are now resolved newest-first under a configurable budget, and blocks outside it are replaced with host-rendered placeholders instead of failing the request. A new MediaHistoryFallback trait lets hosts decide which old resolution failures are recoverable, and the scope callback now receives the disposable request copy so hosts can consume trusted metadata without mutating the canonical transcript. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 287 ++++++++++++++++-- .../src/multimodal/request_tests.rs | 20 +- 2 files changed, 278 insertions(+), 29 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 847affd63..a4b3e0b76 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -7,7 +7,7 @@ //! obtained from the immutable `ChatModel` state for each call; it is never //! inferred from message text or a media reference. -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; @@ -29,12 +29,24 @@ use super::{ }; /// Per-turn attachment limits copied from host configuration. -#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct MediaRequestConfig { /// Image count, size, and remote-fetch limits. pub images: ImageLimits, /// File count, size, extraction, and remote-fetch limits. pub files: FileLimits, + /// Maximum media-bearing historical blocks to resolve, newest first. + pub max_historical_media_blocks: usize, +} + +impl Default for MediaRequestConfig { + fn default() -> Self { + Self { + images: ImageLimits::default(), + files: FileLimits::default(), + max_historical_media_blocks: 8, + } + } } /// Resolve one media source after the host has applied its access policy. @@ -67,14 +79,37 @@ pub trait MediaImageReadout: Send + Sync { ) -> std::result::Result; } +/// Host-specific safe text for media omitted from or unavailable in history. +/// +/// An implementation chooses which old failures are recoverable and may +/// render a workspace-relative hint. It should not echo remote URLs, signed +/// query tokens, or arbitrary resolver errors. Returning `None` keeps the +/// historical failure fatal. +pub trait MediaHistoryFallback: Send + Sync { + /// Render a historical item outside the bounded history budget. + fn omitted(&self, scope: &Scope, modality: InputModality, source: &str) -> String; + + /// Render a recoverable historical resolution failure, or return `None`. + fn unavailable( + &self, + scope: &Scope, + modality: InputModality, + source: &str, + error: &str, + ) -> Option; +} + /// A `ChatModel` decorator that resolves media on a request copy. pub struct MediaRequestModel { inner: Arc>, config: MediaRequestConfig, - scope_for_state: Arc std::result::Result + Send + Sync>, + scope_for_request: + Arc std::result::Result + Send + Sync>, resolver: Arc>, extractor: Arc, image_readout: Option>>, + history_fallback: Option>>, + profile: Option, } impl MediaRequestModel @@ -84,23 +119,30 @@ where { /// Wrap `inner` with generic request preparation. /// - /// The scope callback must return the caller's already-authorized scope - /// from immutable model state. It is called separately for each invoke or - /// stream, so concurrent calls cannot overwrite one another's scope. + /// The callback returns the caller's already-authorized scope from model + /// state or trusted host metadata. It receives only the disposable request + /// copy, so it may consume a host carrier without changing the canonical + /// transcript. It is called separately for each invoke or stream. pub fn new( inner: Arc>, config: MediaRequestConfig, - scope_for_state: impl Fn(&State) -> std::result::Result + Send + Sync + 'static, + scope_for_request: impl Fn(&State, &mut ModelRequest) -> std::result::Result + + Send + + Sync + + 'static, resolver: Arc>, extractor: Arc, ) -> Self { + let profile = inner.profile().cloned(); Self { inner, config, - scope_for_state: Arc::new(scope_for_state), + scope_for_request: Arc::new(scope_for_request), resolver, extractor, image_readout: None, + history_fallback: None, + profile, } } @@ -110,6 +152,21 @@ where self } + /// Set selected-model facts while retaining transport support checks. + pub fn with_profile(mut self, profile: ModelProfile) -> Self { + self.profile = Some(profile); + self + } + + /// Install host policy for bounded historical-media omission and recovery. + pub fn with_history_fallback( + mut self, + fallback: Arc>, + ) -> Self { + self.history_fallback = Some(fallback); + self + } + async fn prepare( &self, state: &State, @@ -128,11 +185,7 @@ where let (images, files) = count_user_media(&request.messages[latest_user]); let (max_images, _) = self.config.images.effective(); let (max_files, _, _) = self.config.files.effective(); - let request_has_media = request.messages.iter().any(|message| { - let (images, files) = count_user_media(message); - images + files > 0 - }); - if self.config.files.files_disabled() && request_has_media { + if self.config.files.files_disabled() && images + files > 0 { return Err(model_error("attachments are disabled for this request")); } if images > max_images { @@ -146,22 +199,70 @@ where ))); } - let scope = (self.scope_for_state)(state).map_err(|error| model_error(&error))?; - for message in &mut request.messages { + let scope = (self.scope_for_request)(state, &mut request) + .map_err(|error| model_error(&error))?; + let selected_history = selected_history_blocks( + &request.messages, + latest_user, + self.config.max_historical_media_blocks, + ); + for (message_index, message) in request.messages.iter_mut().enumerate() { let Message::User(user) = message else { continue; }; let mut prepared = Vec::with_capacity(user.content.len()); - for block in std::mem::take(&mut user.content) { + for (block_index, block) in std::mem::take(&mut user.content).into_iter().enumerate() { + let historical = message_index != latest_user && is_media_block(&block); + if historical && self.config.files.files_disabled() { + prepared.extend(self.omitted_history(&scope, block)); + continue; + } + if historical && !selected_history.contains(&(message_index, block_index)) { + prepared.extend(self.omitted_history(&scope, block)); + continue; + } + let recovery = historical.then(|| media_source_hint(&block)).flatten(); match block { ContentBlock::Text(text) => { - prepared.extend(self.prepare_text(&scope, &text).await?); + match self.prepare_text(&scope, &text).await { + Ok(blocks) => prepared.extend(blocks), + Err(error) => { + if let Some((modality, source)) = recovery.as_ref() + && let Some(placeholder) = self.recover_history( + &scope, + *modality, + source, + &error, + ) + { + prepared.push(ContentBlock::Text(placeholder)); + } else { + return Err(error); + } + } + } } block @ (ContentBlock::Image(_) | ContentBlock::Audio(_) | ContentBlock::Video(_) | ContentBlock::Document(_)) => { - prepared.extend(self.prepare_block(&scope, block).await?); + match self.prepare_block(&scope, block.clone()).await { + Ok(blocks) => prepared.extend(blocks), + Err(error) => { + if let Some((modality, source)) = recovery.as_ref() + && let Some(placeholder) = self.recover_history( + &scope, + *modality, + source, + &error, + ) + { + prepared.push(ContentBlock::Text(placeholder)); + } else { + return Err(error); + } + } + } } other => prepared.push(other), } @@ -191,6 +292,7 @@ where InputModality::Image, &source, self.config.images.max_image_bytes(), + false, ) .await?; if self.native(InputModality::Image, &attachment.mime) { @@ -213,6 +315,7 @@ where InputModality::Document, &source, self.config.files.max_file_bytes(), + true, ) .await?; if self.native(InputModality::Document, &attachment.mime) { @@ -261,7 +364,9 @@ where InputModality::Image => self.config.images.max_image_bytes(), _ => self.config.files.max_file_bytes(), }; - let mut attachment = self.resolve(scope, modality, &source, max_bytes).await?; + let mut attachment = self + .resolve(scope, modality, &source, max_bytes, false) + .await?; if attachment.mime == "application/octet-stream" && let Some(mime) = hinted_mime { @@ -291,6 +396,7 @@ where modality: InputModality, source: &str, max_bytes: usize, + require_file_mime: bool, ) -> tinyinference_llm::Result { let remote = source.starts_with("http://") || source.starts_with("https://"); let remote_allowed = match modality { @@ -313,11 +419,18 @@ where if modality == InputModality::Image && attachment.bytes.is_empty() { return Err(model_error("image reference has no bytes")); } + if require_file_mime && !self.config.files.is_mime_allowed(&attachment.mime) { + return Err(model_error(&format!( + "file MIME type '{}' is not allowed; supported: {}", + attachment.mime, + self.config.files.supported_rendered() + ))); + } Ok(attachment) } fn native(&self, modality: InputModality, mime: &str) -> bool { - let advertised = self.inner.profile().is_some_and(|profile| match modality { + let advertised = self.profile.as_ref().is_some_and(|profile| match modality { InputModality::Image => profile.modalities.image_in, InputModality::Audio => profile.modalities.audio_in, InputModality::Video => profile.modalities.video_in, @@ -408,6 +521,56 @@ where None, ) } + + fn omitted_history(&self, scope: &Scope, block: ContentBlock) -> Vec { + match block { + ContentBlock::Text(text) => { + let (without_images, images) = parse_image_markers(&text); + let (clean, files) = parse_file_markers(&without_images); + if images.is_empty() && files.is_empty() { + return vec![ContentBlock::Text(text)]; + } + let mut output = clean; + for source in images { + output.push_str("\n\n"); + output.push_str(&self.omission(scope, InputModality::Image, &source)); + } + for source in files { + output.push_str("\n\n"); + output.push_str(&self.omission(scope, InputModality::Document, &source)); + } + vec![ContentBlock::Text(output)] + } + other => { + let Some((modality, source)) = media_source_hint(&other) else { + return vec![other]; + }; + vec![ContentBlock::Text(self.omission(scope, modality, &source))] + } + } + } + + fn omission(&self, scope: &Scope, modality: InputModality, source: &str) -> String { + self.history_fallback.as_ref().map_or_else( + || "[Earlier attachment omitted by history budget]".to_string(), + |fallback| fallback.omitted(scope, modality, source), + ) + } + + fn recover_history( + &self, + scope: &Scope, + modality: InputModality, + source: &str, + error: &tinyinference_llm::Error, + ) -> Option { + self.history_fallback.as_ref()?.unavailable( + scope, + modality, + source, + &error.to_string(), + ) + } } #[async_trait] @@ -417,7 +580,7 @@ where Scope: Send + Sync, { fn profile(&self) -> Option<&ModelProfile> { - self.inner.profile() + self.profile.as_ref() } fn supports_input(&self, modality: InputModality, mime: &str, source: InputSource) -> bool { @@ -456,6 +619,88 @@ where } } +fn is_media_block(block: &ContentBlock) -> bool { + match block { + ContentBlock::Text(text) => { + let (_, images) = parse_image_markers(text); + let (_, files) = parse_file_markers(text); + !images.is_empty() || !files.is_empty() + } + ContentBlock::Image(_) + | ContentBlock::Audio(_) + | ContentBlock::Video(_) + | ContentBlock::Document(_) => true, + _ => false, + } +} + +fn selected_history_blocks( + messages: &[Message], + latest_user: usize, + budget: usize, +) -> HashSet<(usize, usize)> { + let mut selected = HashSet::new(); + let mut remaining = budget; + for (message_index, message) in messages.iter().enumerate().rev() { + if message_index == latest_user { + continue; + } + let Message::User(user) = message else { + continue; + }; + for (block_index, block) in user.content.iter().enumerate().rev() { + if !is_media_block(block) { + continue; + } + let count = match block { + ContentBlock::Text(text) => { + let (_, images) = parse_image_markers(text); + let (_, files) = parse_file_markers(text); + images.len() + files.len() + } + _ => 1, + }; + if count <= remaining { + selected.insert((message_index, block_index)); + remaining -= count; + } + } + } + selected +} + +fn media_source_hint(block: &ContentBlock) -> Option<(InputModality, String)> { + match block { + ContentBlock::Text(text) => { + let (_, images) = parse_image_markers(text); + if let Some(source) = images.into_iter().next() { + return Some((InputModality::Image, source)); + } + let (_, files) = parse_file_markers(text); + files + .into_iter() + .next() + .map(|source| (InputModality::Document, source)) + } + ContentBlock::Image(image) => Some((InputModality::Image, image.url.clone())), + ContentBlock::Audio(media) => media_source_hint_ref(InputModality::Audio, media), + ContentBlock::Video(media) => media_source_hint_ref(InputModality::Video, media), + ContentBlock::Document(media) => media_source_hint_ref(InputModality::Document, media), + _ => None, + } +} + +fn media_source_hint_ref(modality: InputModality, media: &MediaRef) -> Option<(InputModality, String)> { + let source = match media { + MediaRef::Path { path, .. } => path, + MediaRef::Url { url, .. } => url, + MediaRef::Base64 { data, media_type } => { + return Some((modality, format!("data:{media_type};base64,{data}"))); + } + }; + Some((modality, source.clone())) +} + fn count_user_media(message: &Message) -> (usize, usize) { let Message::User(user) = message else { return (0, 0); diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index afd594757..b30f84a17 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -167,7 +167,7 @@ fn wrapper( MediaRequestModel::new( model, config, - |state: &String| Ok(state.clone()), + |state: &String, _request| Ok(state.clone()), resolver, extractor, ) @@ -184,7 +184,7 @@ async fn hard_zero_rejects_before_any_source_read() { }, ..MediaRequestConfig::default() }; - let wrapped = wrapper(model, disabled, resolver.clone(), Arc::default()); + let wrapped = wrapper(model.clone(), disabled, resolver.clone(), Arc::default()); let error = wrapped .invoke(&"tenant-a".into(), request("see [IMAGE:secret]")) @@ -215,13 +215,17 @@ async fn hard_zero_also_blocks_media_only_present_in_history() { }), ); - let error = wrapped - .invoke(&"tenant-a".into(), request) - .await - .expect_err("hard-zero disables historical media too"); + wrapped.invoke(&"tenant-a".into(), request).await.unwrap(); - assert!(error.to_string().contains("attachments are disabled")); assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("historical user message retained") + }; + assert!(matches!( + &user.content[0], + ContentBlock::Text(text) if text.contains("Earlier attachment omitted") + )); } #[tokio::test] @@ -328,7 +332,7 @@ async fn resolution_error_and_cancellation_leave_the_canonical_request_unchanged let canceled_wrapper = MediaRequestModel::new( cancel_model, MediaRequestConfig::default(), - |state: &String| Ok(state.clone()), + |state: &String, _request| Ok(state.clone()), pending.clone(), Arc::default(), ); From ed276bbdadec90d618bf5544154cbbbba34e786a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:53:51 +0300 Subject: [PATCH 12/38] test(multimodal): cover history budget, scope isolation, and safe fallbacks Add tests for per-call metadata scope consumption under concurrency, historical media budget bounding reads and replacing older items, and safe degradation when historical media is missing while latest media failures stay fatal. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 145 +++++++++++++++++- 1 file changed, 144 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index b30f84a17..dfa9e5d49 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -140,6 +140,26 @@ impl MediaImageReadout for Readout { } } +struct SafeHistory; + +impl MediaHistoryFallback for SafeHistory { + fn omitted(&self, _scope: &String, _modality: InputModality, _source: &str) -> String { + "[Earlier attachment omitted by history budget]".into() + } + + fn unavailable( + &self, + _scope: &String, + _modality: InputModality, + _source: &str, + error: &str, + ) -> Option { + error + .contains("missing test source") + .then(|| "[Earlier attachment unavailable at workspace path: safe-hint]".into()) + } +} + fn attachment(name: &str, mime: &str, bytes: &[u8]) -> ResolvedAttachment { ResolvedAttachment { bytes: bytes.to_vec(), @@ -206,7 +226,7 @@ async fn hard_zero_also_blocks_media_only_present_in_history() { }, ..MediaRequestConfig::default() }; - let wrapped = wrapper(model, disabled, resolver.clone(), Arc::default()); + let wrapped = wrapper(model.clone(), disabled, resolver.clone(), Arc::default()); let mut request = request("latest text only"); request.messages.insert( 0, @@ -381,6 +401,129 @@ async fn concurrent_calls_keep_their_immutable_scope_separate() { assert_eq!(scopes, vec!["scope-a", "scope-b"]); } +#[tokio::test] +async fn request_metadata_scope_is_consumed_per_concurrent_call() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + sources.insert("image".into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); + let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let wrapped = MediaRequestModel::new( + model.clone(), + MediaRequestConfig::default(), + |_state: &String, request: &mut ModelRequest| { + request + .metadata + .as_object_mut() + .and_then(|metadata| metadata.remove("trusted_scope")) + .and_then(|scope| scope.as_str().map(str::to_string)) + .ok_or_else(|| "missing trusted test scope".to_string()) + }, + resolver.clone(), + Arc::default(), + ); + let operator = "same-operator".to_string(); + let mut first = request("[IMAGE:image]"); + first.metadata = serde_json::json!({"trusted_scope":"workspace-a"}); + let mut second = request("[IMAGE:image]"); + second.metadata = serde_json::json!({"trusted_scope":"workspace-b"}); + + let (a, b) = tokio::join!(wrapped.invoke(&operator, first), wrapped.invoke(&operator, second)); + + a.unwrap(); + b.unwrap(); + let mut scopes = resolver.scopes.lock().unwrap().clone(); + scopes.sort(); + assert_eq!(scopes, vec!["workspace-a", "workspace-b"]); + for request in model.requests.lock().unwrap().iter() { + assert!(request.metadata.get("trusted_scope").is_none()); + } +} + +#[tokio::test] +async fn historical_media_budget_bounds_reads_and_replaces_older_items() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + for source in ["oldest", "newest"] { + sources.insert(source.into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); + } + let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let config = MediaRequestConfig { + max_historical_media_blocks: 1, + ..MediaRequestConfig::default() + }; + let wrapped = wrapper(model.clone(), config, resolver.clone(), Arc::default()); + let mut request = ModelRequest::default(); + request.messages = vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "oldest".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "newest".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest".into())], + }), + ]; + + wrapped.invoke(&"tenant-a".into(), request).await.unwrap(); + + assert_eq!(resolver.calls.load(Ordering::SeqCst), 1); + assert_eq!(resolver.scopes.lock().unwrap().len(), 1); + let calls = model.requests.lock().unwrap(); + assert!(matches!( + &calls[0].messages[0], + Message::User(user) if matches!(&user.content[0], ContentBlock::Text(text) if text.contains("omitted by history budget")) + )); + assert!(matches!( + &calls[0].messages[1], + Message::User(user) if matches!(&user.content[0], ContentBlock::Image(_)) + )); +} + +#[tokio::test] +async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() { + let model = Arc::new(CaptureModel::new(true)); + let resolver = Arc::new(Resolver::default()); + let wrapped = wrapper(model.clone(), MediaRequestConfig::default(), resolver, Arc::default()) + .with_history_fallback(Arc::new(SafeHistory)); + let mut history = ModelRequest::default(); + history.messages = vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "/private-value/also-private".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest text".into())], + }), + ]; + + wrapped.invoke(&"tenant-a".into(), history).await.unwrap(); + let calls = model.requests.lock().unwrap(); + let Message::User(old_user) = &calls[0].messages[0] else { + panic!("historical message retained") + }; + let ContentBlock::Text(placeholder) = &old_user.content[0] else { + panic!("missing historical media becomes text") + }; + assert!(placeholder.contains("safe-hint")); + assert!(!placeholder.contains("private-value")); + drop(calls); + + let error = wrapped + .invoke(&"tenant-a".into(), request("[IMAGE:latest-missing]")) + .await + .expect_err("latest attachment failures remain fatal"); + assert!(error.to_string().contains("missing test source latest-missing")); +} + #[tokio::test] async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { let model = Arc::new(CaptureModel::new(false)); From d4f6f224fb567d0ee13a89e3f0f035eff49735e9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:54:02 +0300 Subject: [PATCH 13/38] test(multimodal): import ImageRef in request tests Adds the ImageRef import needed by the multimodal request tests so the image reference type is in scope. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/request_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index dfa9e5d49..66e9650b9 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -8,7 +8,7 @@ use std::{ use futures::StreamExt; use tinyinference_llm::{ - message::UserMessage, + message::{ImageRef, UserMessage}, model::{Modalities, ModelStreamItem}, }; From 8175c509ab4bed9ccb6c6fb04e1fff2f060047a5 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:54:14 +0300 Subject: [PATCH 14/38] refactor(multimodal): re-export history fallback and tidy request formatting Export MediaHistoryFallback from the multimodal module and reflow the request wrapper and its tests to satisfy rustfmt. The doc comment now notes that scope may come from trusted host request metadata in addition to model state. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/mod.rs | 5 +- .../src/multimodal/request.rs | 77 ++++++++----------- .../src/multimodal/request_tests.rs | 45 ++++++++--- 3 files changed, 72 insertions(+), 55 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index 68b37b567..511f90250 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -101,7 +101,10 @@ pub use markers::{ text_has_image_placeholders, }; pub use payload::{FilePayload, compose_multimodal_message, sha256_prefix, truncate_chars}; -pub use request::{MediaImageReadout, MediaRequestConfig, MediaRequestModel, MediaSourceResolver}; +pub use request::{ + MediaHistoryFallback, MediaImageReadout, MediaRequestConfig, MediaRequestModel, + MediaSourceResolver, +}; pub use resolve::{ NoTextExtractor, TextExtractor, resolve_attachment, resolve_file, resolve_image, }; diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index a4b3e0b76..c250d4e25 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -1,11 +1,11 @@ //! Prepare a disposable model request by resolving its user media references. //! -//! The wrapper owns generic marker counting, capability checks, payload -//! rendering, document extraction hooks, and non-vision text fallbacks. Hosts -//! provide the scope and source resolver because only a host can authorize a -//! path, fetch a remote source, or decide which credentials apply. Scope is -//! obtained from the immutable `ChatModel` state for each call; it is never -//! inferred from message text or a media reference. +//! The wrapper owns generic marker counting, bounded history, capability +//! checks, payload rendering, document extraction hooks, and non-vision text +//! fallbacks. Hosts provide the scope and source resolver because only a host +//! can authorize a path, fetch a remote source, or decide which credentials +//! apply. Scope comes from model state or trusted host request metadata and is +//! never inferred from message text or a media reference. use std::{collections::HashSet, sync::Arc}; @@ -127,9 +127,9 @@ where inner: Arc>, config: MediaRequestConfig, scope_for_request: impl Fn(&State, &mut ModelRequest) -> std::result::Result - + Send - + Sync - + 'static, + + Send + + Sync + + 'static, resolver: Arc>, extractor: Arc, ) -> Self { @@ -159,10 +159,7 @@ where } /// Install host policy for bounded historical-media omission and recovery. - pub fn with_history_fallback( - mut self, - fallback: Arc>, - ) -> Self { + pub fn with_history_fallback(mut self, fallback: Arc>) -> Self { self.history_fallback = Some(fallback); self } @@ -199,8 +196,8 @@ where ))); } - let scope = (self.scope_for_request)(state, &mut request) - .map_err(|error| model_error(&error))?; + let scope = + (self.scope_for_request)(state, &mut request).map_err(|error| model_error(&error))?; let selected_history = selected_history_blocks( &request.messages, latest_user, @@ -223,25 +220,19 @@ where } let recovery = historical.then(|| media_source_hint(&block)).flatten(); match block { - ContentBlock::Text(text) => { - match self.prepare_text(&scope, &text).await { - Ok(blocks) => prepared.extend(blocks), - Err(error) => { - if let Some((modality, source)) = recovery.as_ref() - && let Some(placeholder) = self.recover_history( - &scope, - *modality, - source, - &error, - ) - { - prepared.push(ContentBlock::Text(placeholder)); - } else { - return Err(error); - } + ContentBlock::Text(text) => match self.prepare_text(&scope, &text).await { + Ok(blocks) => prepared.extend(blocks), + Err(error) => { + if let Some((modality, source)) = recovery.as_ref() + && let Some(placeholder) = + self.recover_history(&scope, *modality, source, &error) + { + prepared.push(ContentBlock::Text(placeholder)); + } else { + return Err(error); } } - } + }, block @ (ContentBlock::Image(_) | ContentBlock::Audio(_) | ContentBlock::Video(_) @@ -250,12 +241,8 @@ where Ok(blocks) => prepared.extend(blocks), Err(error) => { if let Some((modality, source)) = recovery.as_ref() - && let Some(placeholder) = self.recover_history( - &scope, - *modality, - source, - &error, - ) + && let Some(placeholder) = + self.recover_history(&scope, *modality, source, &error) { prepared.push(ContentBlock::Text(placeholder)); } else { @@ -564,12 +551,9 @@ where source: &str, error: &tinyinference_llm::Error, ) -> Option { - self.history_fallback.as_ref()?.unavailable( - scope, - modality, - source, - &error.to_string(), - ) + self.history_fallback + .as_ref()? + .unavailable(scope, modality, source, &error.to_string()) } } @@ -690,7 +674,10 @@ fn media_source_hint(block: &ContentBlock) -> Option<(InputModality, String)> { } } -fn media_source_hint_ref(modality: InputModality, media: &MediaRef) -> Option<(InputModality, String)> { +fn media_source_hint_ref( + modality: InputModality, + media: &MediaRef, +) -> Option<(InputModality, String)> { let source = match media { MediaRef::Path { path, .. } => path, MediaRef::Url { url, .. } => url, diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 66e9650b9..897b2f710 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -365,7 +365,10 @@ async fn resolution_error_and_cancellation_leave_the_canonical_request_unchanged pending.entered.notified().await; call.abort(); assert!(call.await.unwrap_err().is_cancelled()); - assert_eq!(serde_json::to_value(&original).unwrap(), canonical_after_cancel); + assert_eq!( + serde_json::to_value(&original).unwrap(), + canonical_after_cancel + ); } #[tokio::test] @@ -405,8 +408,14 @@ async fn concurrent_calls_keep_their_immutable_scope_separate() { async fn request_metadata_scope_is_consumed_per_concurrent_call() { let model = Arc::new(CaptureModel::new(true)); let mut sources = HashMap::new(); - sources.insert("image".into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); - let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + sources.insert( + "image".into(), + attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); let wrapped = MediaRequestModel::new( model.clone(), MediaRequestConfig::default(), @@ -427,7 +436,10 @@ async fn request_metadata_scope_is_consumed_per_concurrent_call() { let mut second = request("[IMAGE:image]"); second.metadata = serde_json::json!({"trusted_scope":"workspace-b"}); - let (a, b) = tokio::join!(wrapped.invoke(&operator, first), wrapped.invoke(&operator, second)); + let (a, b) = tokio::join!( + wrapped.invoke(&operator, first), + wrapped.invoke(&operator, second) + ); a.unwrap(); b.unwrap(); @@ -444,9 +456,15 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { let model = Arc::new(CaptureModel::new(true)); let mut sources = HashMap::new(); for source in ["oldest", "newest"] { - sources.insert(source.into(), attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n")); + sources.insert( + source.into(), + attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); } - let resolver = Arc::new(Resolver { sources, ..Resolver::default() }); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); let config = MediaRequestConfig { max_historical_media_blocks: 1, ..MediaRequestConfig::default() @@ -490,8 +508,13 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() { let model = Arc::new(CaptureModel::new(true)); let resolver = Arc::new(Resolver::default()); - let wrapped = wrapper(model.clone(), MediaRequestConfig::default(), resolver, Arc::default()) - .with_history_fallback(Arc::new(SafeHistory)); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + resolver, + Arc::default(), + ) + .with_history_fallback(Arc::new(SafeHistory)); let mut history = ModelRequest::default(); history.messages = vec![ Message::User(UserMessage { @@ -521,7 +544,11 @@ async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() .invoke(&"tenant-a".into(), request("[IMAGE:latest-missing]")) .await .expect_err("latest attachment failures remain fatal"); - assert!(error.to_string().contains("missing test source latest-missing")); + assert!( + error + .to_string() + .contains("missing test source latest-missing") + ); } #[tokio::test] From a4f8a2f58667d169c3e20cbbefc42cc158bad9bb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:55:22 +0300 Subject: [PATCH 15/38] docs(multimodal): document history fallback and model profile seams Update the multimodal README to describe the bounded historical-media budget, the MediaHistoryFallback host seam, and selected-model ModelProfile facts, and adjust the request tests to use NoTextExtractor instead of a default Arc. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/README.md | 15 ++++++++------- .../src/multimodal/request_tests.rs | 10 +++++++--- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/README.md b/crates/tinyagents-harness/src/multimodal/README.md index e77058f30..50dad7852 100644 --- a/crates/tinyagents-harness/src/multimodal/README.md +++ b/crates/tinyagents-harness/src/multimodal/README.md @@ -47,10 +47,11 @@ format contributes a header naming it plus a content hash. `escape_attr`). - [`request`] — [`MediaRequestModel`], a `ChatModel` decorator that prepares a disposable request, checks counts before reads, preserves per-call scope, - dispatches native media only when both model and transport support it, and - renders text/archive/image fallbacks. The host supplies - [`MediaSourceResolver`] for authorized source access and may supply - [`MediaImageReadout`] for its chosen non-vision image description path. + applies a bounded historical-media budget, dispatches native media only when + both model and transport support it, and renders text/archive/image + fallbacks. The host supplies [`MediaSourceResolver`] for authorized source + access, [`MediaHistoryFallback`] for its safe history behavior, and may + supply [`MediaImageReadout`] plus selected-model [`ModelProfile`] facts. - [`error`] — [`MultimodalError`] and its [`Result`] alias; every variant carries the offending `input` verbatim so a multi-attachment turn's failure is attributable. @@ -72,7 +73,7 @@ format contributes a header naming it plus a content hash. | `archive/zip_admission.rs` | Allocation-free ZIP preflight and admitted reader. | | `png.rs` | Optional lossless PNG optimization. | | `payload.rs` | `FilePayload`, message composition, truncation, hashing. | -| `request.rs` | `MediaRequestModel` and host resolver/readout seams. | +| `request.rs` | `MediaRequestModel` and host scope/resolver/fallback seams. | | `error.rs` | `MultimodalError`, `Result`. | | `mod_tests.rs`, `resolve_tests.rs`, `archive/ops_tests.rs`, `png_tests.rs`, `blob_stash_tests.rs` | Marker, MIME, size, path, intake, archive, PNG and stash behavior. | @@ -87,8 +88,8 @@ format contributes a header naming it plus a content hash. - What stays with the host, deliberately: the `TextExtractor` implementation (which parser, if any, and its timeout), the attachment stash (where bytes live between ingress and dispatch), image-readout model choice, source - authorization, and access scope. `MediaRequestModel` counts typed message - markers but never decides which local paths may be read. + authorization, access scope, selected-model profile, and safe historical + placeholders. `MediaRequestModel` never decides which local paths may be read. - Opt-in remote image and file URLs pass TinyTools' URL and DNS guard. The resolver pins the vetted addresses in its own direct HTTP client and refuses redirects. It uses 30-second request and 10-second connection timeouts. diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 897b2f710..c4d472fad 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -1,3 +1,4 @@ +use super::super::NoTextExtractor; use super::*; use std::{ collections::HashMap, @@ -354,13 +355,16 @@ async fn resolution_error_and_cancellation_leave_the_canonical_request_unchanged MediaRequestConfig::default(), |state: &String, _request| Ok(state.clone()), pending.clone(), - Arc::default(), + Arc::new(NoTextExtractor), ); let canonical_after_cancel = serde_json::to_value(&original).unwrap(); let call = { let wrapped = canceled_wrapper; let request = original.clone(); - tokio::spawn(async move { wrapped.invoke(&"tenant-a".into(), request).await }) + tokio::spawn(async move { + let tenant = "tenant-a".to_string(); + wrapped.invoke(&tenant, request).await + }) }; pending.entered.notified().await; call.abort(); @@ -428,7 +432,7 @@ async fn request_metadata_scope_is_consumed_per_concurrent_call() { .ok_or_else(|| "missing trusted test scope".to_string()) }, resolver.clone(), - Arc::default(), + Arc::new(NoTextExtractor), ); let operator = "same-operator".to_string(); let mut first = request("[IMAGE:image]"); From 0e5a7b8cd5a6d04e4898ec3f7539ea78b2b1b2c9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:56:25 +0300 Subject: [PATCH 16/38] docs(multimodal): clarify host-owned decisions and counting rules The module docs now describe the generic pipeline as leaving source access, scope, selected-model facts, and other host policy behind explicit seams, and the host-owned table lists the source resolver, model profile, and placeholder policies alongside the existing entries. The count-before-resolving rule now points at MediaRequestModel, which enforces turn caps and the history budget against raw markers before any resolver runs. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/README.md | 5 +++-- crates/tinyagents-harness/src/multimodal/mod.rs | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/README.md b/crates/tinyagents-harness/src/multimodal/README.md index 50dad7852..2dcfec76d 100644 --- a/crates/tinyagents-harness/src/multimodal/README.md +++ b/crates/tinyagents-harness/src/multimodal/README.md @@ -7,8 +7,9 @@ message text. A user attaches a picture or a document; somewhere between the text box and the provider that attachment has to become bytes the model can read — -validated, size-capped, MIME-checked, and rendered into the message. This -module is that pipeline, minus every decision that belongs to a host. +validated, size-capped, MIME-checked, and rendered into the message. The +multimodal module provides that generic pipeline while leaving source access, +scope, selected-model facts, and other host policy behind explicit seams. Attachments travel as markers inside message text rather than a parallel structured field, so they survive every hop a host already has (persistence, summarisation, delegation) without each of those learning about attachments. diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index 511f90250..e09d65a29 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -37,15 +37,17 @@ //! //! ## What stays with the host //! -//! Four things, each because it depends on the host's own runtime, storage, or -//! threat model rather than on anything universal: +//! Host decisions remain host-owned because they depend on its runtime, +//! storage, or threat model: //! //! | Host owns | Why | //! | --- | --- | //! | The `reqwest::Client` | retained in resolver signatures for compatibility; remote fetches use a DNS-pinned direct client with redirects disabled | //! | [`TextExtractor`] | which document parser (if any) a host carries, and how long it may run | //! | The stash policy | which storage scope holds bytes between ingress and dispatch, how large it may grow, how long they live ([`BlobAttachmentStash`] is the mechanism) | -//! | Message-level counting | only the host knows what its message type is | +//! | The source resolver and scope | only the host can authorize a path, fetch remote data, and bind a request to trusted access context | +//! | The selected model profile | catalog facts and transport serialization support are independent inputs | +//! | Historical placeholders and image readout | safe path hints and non-vision model choice are host policies | //! //! The stash deserves a note, because the split is not obvious. A host that //! persists conversations should **not** persist a multi-megabyte `data:` URI: @@ -62,9 +64,9 @@ //! //! Two rules, both learned the hard way: //! -//! 1. **Count before resolving.** [`config::FileLimits::files_disabled`] and the -//! per-turn caps are checked against the raw markers, before any read -//! happens. A cap enforced after the fetch is not a cap. +//! 1. **Count before resolving.** [`request::MediaRequestModel`] checks latest +//! turn caps and its bounded history budget against raw markers before any +//! source resolver is called. A cap enforced after the fetch is not a cap. //! 2. **Preserve the hard disable.** `max_files == 0` means *none* and stays //! zero in [`config::FileLimits::effective`]. The request decorator checks //! this before calling a host resolver. From 8f69549cecd5d1b2e64e995d98ae081cdaf4bf40 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:57:08 +0300 Subject: [PATCH 17/38] test(multimodal): gate archive fallback test behind feature flag The archive fallback test now carries a cfg attribute for the multimodal feature so it is only compiled when that feature is enabled, matching the other tests in the module. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/request_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index c4d472fad..a0b20978c 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -608,6 +608,7 @@ async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { } #[tokio::test] +#[cfg(feature = "multimodal")] async fn nonvision_archive_fallback_lists_entries_without_extracting_them() { let mut bytes = std::io::Cursor::new(Vec::new()); { From a5c0aaeff474eebbf2dbdd4be3a7c58ea6a83d5e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:58:30 +0300 Subject: [PATCH 18/38] fix(multimodal): recover historical media failures per attachment Historical text blocks now pass a historical flag into text preparation so image and file resolution failures fall back to placeholders individually instead of failing the whole block. This keeps recovered history intact when only some attachments are unavailable. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 57 +++++++++++++------ 1 file changed, 40 insertions(+), 17 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index c250d4e25..786e565b7 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -220,19 +220,9 @@ where } let recovery = historical.then(|| media_source_hint(&block)).flatten(); match block { - ContentBlock::Text(text) => match self.prepare_text(&scope, &text).await { - Ok(blocks) => prepared.extend(blocks), - Err(error) => { - if let Some((modality, source)) = recovery.as_ref() - && let Some(placeholder) = - self.recover_history(&scope, *modality, source, &error) - { - prepared.push(ContentBlock::Text(placeholder)); - } else { - return Err(error); - } - } - }, + ContentBlock::Text(text) => { + prepared.extend(self.prepare_text(&scope, &text, historical).await?); + } block @ (ContentBlock::Image(_) | ContentBlock::Audio(_) | ContentBlock::Video(_) @@ -263,6 +253,7 @@ where &self, scope: &Scope, text: &str, + historical: bool, ) -> tinyinference_llm::Result> { let (without_images, images) = parse_image_markers(text); let (clean_text, files) = parse_file_markers(&without_images); @@ -273,7 +264,7 @@ where let mut native_images = Vec::new(); let mut image_fallback = Vec::new(); for source in images { - let attachment = self + let attachment = match self .resolve( scope, InputModality::Image, @@ -281,7 +272,23 @@ where self.config.images.max_image_bytes(), false, ) - .await?; + .await + { + Ok(attachment) => attachment, + Err(error) if historical => { + if let Some(placeholder) = self.recover_history( + scope, + InputModality::Image, + &source, + &error, + ) { + image_fallback.push(placeholder); + continue; + } + return Err(error); + } + Err(error) => return Err(error), + }; if self.native(InputModality::Image, &attachment.mime) { let mime = validated_image_mime(&attachment)?; native_images.push(ContentBlock::Image(ImageRef { @@ -296,7 +303,7 @@ where let mut file_payloads = Vec::new(); let mut file_fallback_text = Vec::new(); for source in files { - let attachment = self + let attachment = match self .resolve( scope, InputModality::Document, @@ -304,7 +311,23 @@ where self.config.files.max_file_bytes(), true, ) - .await?; + .await + { + Ok(attachment) => attachment, + Err(error) if historical => { + if let Some(placeholder) = self.recover_history( + scope, + InputModality::Document, + &source, + &error, + ) { + file_fallback_text.push(placeholder); + continue; + } + return Err(error); + } + Err(error) => return Err(error), + }; if self.native(InputModality::Document, &attachment.mime) { native_images.push(ContentBlock::Document(MediaRef::base64( STANDARD.encode(&attachment.bytes), From 3dc76867ada5f728186021e3d131af0ecca049ab Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:59:10 +0300 Subject: [PATCH 19/38] test(multimodal): cover historical media fallback recovery Add a test that verifies historical text recovers each failed media marker without losing surrounding prose or valid attachments, and that the unavailable hook receives the modality, source, and error. The request module only picks up rustfmt reflow of the recover_history calls. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 18 ++-- .../src/multimodal/request_tests.rs | 99 +++++++++++++++++++ 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 786e565b7..13f5866d9 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -276,12 +276,9 @@ where { Ok(attachment) => attachment, Err(error) if historical => { - if let Some(placeholder) = self.recover_history( - scope, - InputModality::Image, - &source, - &error, - ) { + if let Some(placeholder) = + self.recover_history(scope, InputModality::Image, &source, &error) + { image_fallback.push(placeholder); continue; } @@ -315,12 +312,9 @@ where { Ok(attachment) => attachment, Err(error) if historical => { - if let Some(placeholder) = self.recover_history( - scope, - InputModality::Document, - &source, - &error, - ) { + if let Some(placeholder) = + self.recover_history(scope, InputModality::Document, &source, &error) + { file_fallback_text.push(placeholder); continue; } diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index a0b20978c..3ed856fdf 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -161,6 +161,31 @@ impl MediaHistoryFallback for SafeHistory { } } +#[derive(Default)] +struct RecordingHistory { + unavailable: Mutex>, +} + +impl MediaHistoryFallback for RecordingHistory { + fn omitted(&self, _scope: &String, _modality: InputModality, _source: &str) -> String { + "[Earlier attachment omitted by history budget]".into() + } + + fn unavailable( + &self, + _scope: &String, + modality: InputModality, + source: &str, + error: &str, + ) -> Option { + self.unavailable + .lock() + .unwrap() + .push((modality, source.to_string(), error.to_string())); + Some("[Earlier attachment unavailable: safe-hint]".into()) + } +} + fn attachment(name: &str, mime: &str, bytes: &[u8]) -> ResolvedAttachment { ResolvedAttachment { bytes: bytes.to_vec(), @@ -555,6 +580,80 @@ async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() ); } +#[tokio::test] +async fn historical_text_recovers_each_failed_marker_without_losing_prose_or_valid_media() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + sources.insert( + "good-image".into(), + attachment("good.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let history = Arc::new(RecordingHistory::default()); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig { + images: ImageLimits { + allow_remote_fetch: false, + ..ImageLimits::default() + }, + ..MediaRequestConfig::default() + }, + resolver, + Arc::default(), + ) + .with_history_fallback(history.clone()); + let mut request = ModelRequest::default(); + request.messages = vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Text( + "Caption [IMAGE:good-image] then [IMAGE:https://example.test/photo?token=secret]" + .into(), + )], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest text".into())], + }), + ]; + + wrapped.invoke(&"tenant-a".into(), request).await.unwrap(); + + let captured = model.requests.lock().unwrap(); + let Message::User(old_user) = &captured[0].messages[0] else { + panic!("historical message retained") + }; + let rendered = old_user + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + assert!(rendered.contains("Caption")); + assert!(rendered.contains("safe-hint")); + assert!(!rendered.contains("token=secret")); + assert!( + old_user + .content + .iter() + .any(|block| matches!(block, ContentBlock::Image(_))) + ); + drop(captured); + assert_eq!( + history.unavailable.lock().unwrap().as_slice(), + &[( + InputModality::Image, + "https://example.test/photo?token=secret".into(), + "Model error: remote attachment fetch is disabled".into(), + )] + ); +} + #[tokio::test] async fn nonvision_fallback_reads_images_and_uses_the_pdf_extractor() { let model = Arc::new(CaptureModel::new(false)); From e334aca87baebba7e7c594032b0331626bb25f87 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:59:52 +0300 Subject: [PATCH 20/38] test(multimodal): cover fatal source failure in latest turn Rename the historical request binding for clarity and assert that a source failure in the latest user turn stays fatal while the unavailable history entry is still recorded. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 3ed856fdf..e0555c81b 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -499,8 +499,8 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { ..MediaRequestConfig::default() }; let wrapped = wrapper(model.clone(), config, resolver.clone(), Arc::default()); - let mut request = ModelRequest::default(); - request.messages = vec![ + let mut historical_request = ModelRequest::default(); + historical_request.messages = vec![ Message::User(UserMessage { content: vec![ContentBlock::Image(ImageRef { url: "oldest".into(), @@ -518,7 +518,10 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { }), ]; - wrapped.invoke(&"tenant-a".into(), request).await.unwrap(); + wrapped + .invoke(&"tenant-a".into(), historical_request) + .await + .unwrap(); assert_eq!(resolver.calls.load(Ordering::SeqCst), 1); assert_eq!(resolver.scopes.lock().unwrap().len(), 1); @@ -649,9 +652,23 @@ async fn historical_text_recovers_each_failed_marker_without_losing_prose_or_val &[( InputModality::Image, "https://example.test/photo?token=secret".into(), - "Model error: remote attachment fetch is disabled".into(), + "model error: remote attachment fetch is disabled".into(), )] ); + + let error = wrapped + .invoke( + &"tenant-a".into(), + request("[IMAGE:https://example.test/photo?token=secret]"), + ) + .await + .expect_err("the same source failure remains fatal in the latest user turn"); + assert!( + error + .to_string() + .contains("remote attachment fetch is disabled") + ); + assert_eq!(history.unavailable.lock().unwrap().len(), 1); } #[tokio::test] From 1b68bf3af95ec28495a40ac242559b3425c87043 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 01:59:59 +0300 Subject: [PATCH 21/38] test(multimodal): rename request binding in history fallback test Rename the local request variable to historical_request and reformat the invoke call so the test reads more clearly. No behaviour change. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/request_tests.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index e0555c81b..14a92011d 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -609,8 +609,8 @@ async fn historical_text_recovers_each_failed_marker_without_losing_prose_or_val Arc::default(), ) .with_history_fallback(history.clone()); - let mut request = ModelRequest::default(); - request.messages = vec![ + let mut historical_request = ModelRequest::default(); + historical_request.messages = vec![ Message::User(UserMessage { content: vec![ContentBlock::Text( "Caption [IMAGE:good-image] then [IMAGE:https://example.test/photo?token=secret]" @@ -622,7 +622,10 @@ async fn historical_text_recovers_each_failed_marker_without_losing_prose_or_val }), ]; - wrapped.invoke(&"tenant-a".into(), request).await.unwrap(); + wrapped + .invoke(&"tenant-a".into(), historical_request) + .await + .unwrap(); let captured = model.requests.lock().unwrap(); let Message::User(old_user) = &captured[0].messages[0] else { From b61b99af3f49eae5e19d61e7a19c1104623234c3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:02:15 +0300 Subject: [PATCH 22/38] fix(multimodal): apply history budget per media reference The history budget now selects individual image and file references inside a text block instead of dropping the whole block, so a block with more markers than remaining budget keeps its newest references and replaces only the rest with omission notices. The trusted scope callback also runs before limit checks and history selection so those checks see the normalized request, and typed media blocks are counted Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 102 +++++++++++--- .../src/multimodal/request_tests.rs | 127 ++++++++++++++++++ 2 files changed, 212 insertions(+), 17 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 13f5866d9..ceaf31b60 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -7,7 +7,7 @@ //! apply. Scope comes from model state or trusted host request metadata and is //! never inferred from message text or a media reference. -use std::{collections::HashSet, sync::Arc}; +use std::{collections::HashMap, sync::Arc}; use async_trait::async_trait; use base64::{Engine as _, engine::general_purpose::STANDARD}; @@ -169,6 +169,12 @@ where state: &State, mut request: ModelRequest, ) -> tinyinference_llm::Result { + // This trusted, synchronous callback may consume per-request metadata + // or normalize host markers on the disposable copy. Run it before + // deriving limits and history positions so those checks see the exact + // request that will reach the transport. It must not perform reads. + let scope = + (self.scope_for_request)(state, &mut request).map_err(|error| model_error(&error))?; let Some(latest_user) = request .messages .iter() @@ -196,9 +202,7 @@ where ))); } - let scope = - (self.scope_for_request)(state, &mut request).map_err(|error| model_error(&error))?; - let selected_history = selected_history_blocks( + let selected_history = select_history_media( &request.messages, latest_user, self.config.max_historical_media_blocks, @@ -210,18 +214,41 @@ where let mut prepared = Vec::with_capacity(user.content.len()); for (block_index, block) in std::mem::take(&mut user.content).into_iter().enumerate() { let historical = message_index != latest_user && is_media_block(&block); + let selection = selected_history.get(&(message_index, block_index)); if historical && self.config.files.files_disabled() { prepared.extend(self.omitted_history(&scope, block)); continue; } - if historical && !selected_history.contains(&(message_index, block_index)) { + if historical + && matches!(block, ContentBlock::Text(_)) + && selection.is_none_or(|selection| { + selection.images.iter().all(|selected| !selected) + && selection.files.iter().all(|selected| !selected) + }) + { + prepared.extend(self.omitted_history(&scope, block)); + continue; + } + if historical + && !matches!(block, ContentBlock::Text(_)) + && !selection.is_some_and(|selection| selection.typed) + { prepared.extend(self.omitted_history(&scope, block)); continue; } let recovery = historical.then(|| media_source_hint(&block)).flatten(); match block { ContentBlock::Text(text) => { - prepared.extend(self.prepare_text(&scope, &text, historical).await?); + prepared.extend( + self.prepare_text( + &scope, + &text, + historical, + selection.map(|selection| selection.images.as_slice()), + selection.map(|selection| selection.files.as_slice()), + ) + .await?, + ); } block @ (ContentBlock::Image(_) | ContentBlock::Audio(_) @@ -254,6 +281,8 @@ where scope: &Scope, text: &str, historical: bool, + selected_images: Option<&[bool]>, + selected_files: Option<&[bool]>, ) -> tinyinference_llm::Result> { let (without_images, images) = parse_image_markers(text); let (clean_text, files) = parse_file_markers(&without_images); @@ -263,7 +292,11 @@ where let mut native_images = Vec::new(); let mut image_fallback = Vec::new(); - for source in images { + for (index, source) in images.into_iter().enumerate() { + if historical && !selected_images.is_some_and(|selected| selected[index]) { + image_fallback.push(self.omission(scope, InputModality::Image, &source)); + continue; + } let attachment = match self .resolve( scope, @@ -299,7 +332,11 @@ where let mut file_payloads = Vec::new(); let mut file_fallback_text = Vec::new(); - for source in files { + for (index, source) in files.into_iter().enumerate() { + if historical && !selected_files.is_some_and(|selected| selected[index]) { + file_fallback_text.push(self.omission(scope, InputModality::Document, &source)); + continue; + } let attachment = match self .resolve( scope, @@ -635,12 +672,19 @@ fn is_media_block(block: &ContentBlock) -> bool { } } -fn selected_history_blocks( +#[derive(Default)] +struct HistorySelection { + images: Vec, + files: Vec, + typed: bool, +} + +fn select_history_media( messages: &[Message], latest_user: usize, budget: usize, -) -> HashSet<(usize, usize)> { - let mut selected = HashSet::new(); +) -> HashMap<(usize, usize), HistorySelection> { + let mut selected = HashMap::new(); let mut remaining = budget; for (message_index, message) in messages.iter().enumerate().rev() { if message_index == latest_user { @@ -653,17 +697,41 @@ fn selected_history_blocks( if !is_media_block(block) { continue; } - let count = match block { + let selection = match block { ContentBlock::Text(text) => { let (_, images) = parse_image_markers(text); let (_, files) = parse_file_markers(text); - images.len() + files.len() + let mut selection = HistorySelection { + images: vec![false; images.len()], + files: vec![false; files.len()], + typed: false, + }; + for selected in selection.files.iter_mut().rev() { + if remaining == 0 { + break; + } + *selected = true; + remaining -= 1; + } + for selected in selection.images.iter_mut().rev() { + if remaining == 0 { + break; + } + *selected = true; + remaining -= 1; + } + selection } - _ => 1, + _ => HistorySelection { + typed: remaining > 0, + ..HistorySelection::default() + }, }; - if count <= remaining { - selected.insert((message_index, block_index)); - remaining -= count; + if selection.typed || !selection.images.is_empty() || !selection.files.is_empty() { + if selection.typed { + remaining -= 1; + } + selected.insert((message_index, block_index), selection); } } } diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 14a92011d..de59e1123 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -64,6 +64,7 @@ struct Resolver { sources: HashMap, calls: AtomicUsize, scopes: Mutex>, + resolved_sources: Mutex>, } #[derive(Default)] @@ -96,6 +97,10 @@ impl MediaSourceResolver for Resolver { ) -> std::result::Result { self.calls.fetch_add(1, Ordering::SeqCst); self.scopes.lock().unwrap().push(scope.clone()); + self.resolved_sources + .lock() + .unwrap() + .push(source.to_string()); let attachment = self .sources .get(source) @@ -536,6 +541,128 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { )); } +#[tokio::test] +async fn historical_marker_budget_selects_only_the_newest_references_in_a_text_block() { + let model = Arc::new(CaptureModel::new(true)); + let mut sources = HashMap::new(); + for source in ["older", "newer"] { + sources.insert( + source.into(), + attachment("img.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + } + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig { + max_historical_media_blocks: 1, + ..MediaRequestConfig::default() + }, + resolver.clone(), + Arc::default(), + ); + let mut historical_request = ModelRequest::default(); + historical_request.messages = vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Text("[IMAGE:older] [IMAGE:newer]".into())], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest".into())], + }), + ]; + + wrapped + .invoke(&"tenant-a".into(), historical_request) + .await + .unwrap(); + + assert_eq!(resolver.calls.load(Ordering::SeqCst), 1); + assert_eq!(resolver.resolved_sources.lock().unwrap().as_slice(), ["newer"]); + let captured = model.requests.lock().unwrap(); + let Message::User(old_user) = &captured[0].messages[0] else { + panic!("historical message retained") + }; + assert!(matches!( + &old_user.content[0], + ContentBlock::Text(text) if text.contains("omitted by history budget") + )); + assert!(old_user + .content + .iter() + .any(|block| matches!(block, ContentBlock::Image(_)))); +} + +#[tokio::test] +async fn trusted_preparer_normalization_is_counted_before_resolution() { + let model = Arc::new(CaptureModel::new(true)); + let resolver = Arc::new(Resolver::default()); + let config = MediaRequestConfig { + images: ImageLimits { + max_images: 1, + ..ImageLimits::default() + }, + ..MediaRequestConfig::default() + }; + let wrapped = MediaRequestModel::new( + model, + config, + |_state: &String, request: &mut ModelRequest| { + if let Some(metadata) = request.metadata.as_object_mut() { + metadata.remove("trusted_scope"); + } + if let Some(Message::User(user)) = request.messages.last_mut() { + user.content + .push(ContentBlock::Text("[IMAGE:one] [IMAGE:two]".into())); + } + Ok("trusted-workspace".to_string()) + }, + resolver.clone(), + Arc::new(NoTextExtractor), + ); + let mut original = request("caption"); + original.metadata = serde_json::json!({"trusted_scope":"workspace-a"}); + let canonical = original.clone(); + + let error = wrapped + .invoke(&"tenant-a".into(), original.clone()) + .await + .expect_err("normalized markers count toward the current-turn limit"); + + assert!(error.to_string().contains("max=1, found=2")); + assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); + assert_eq!(original, canonical); +} + +#[tokio::test] +async fn trusted_preparer_consumes_carrier_when_request_has_no_user_message() { + let model = Arc::new(CaptureModel::new(false)); + let resolver = Arc::new(Resolver::default()); + let wrapped = MediaRequestModel::new( + model.clone(), + MediaRequestConfig::default(), + |_state: &String, request: &mut ModelRequest| { + if let Some(metadata) = request.metadata.as_object_mut() { + metadata.remove("trusted_scope"); + } + Ok("trusted-workspace".to_string()) + }, + resolver, + Arc::new(NoTextExtractor), + ); + let mut original = ModelRequest::default(); + original.metadata = serde_json::json!({"trusted_scope":"workspace-a"}); + + wrapped.invoke(&"tenant-a".into(), original).await.unwrap(); + + assert!(model.requests.lock().unwrap()[0] + .metadata + .get("trusted_scope") + .is_none()); +} + #[tokio::test] async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() { let model = Arc::new(CaptureModel::new(true)); From 8936a0091a6a48ce725c9eec40cdf9aab4b2e913 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:02:47 +0300 Subject: [PATCH 23/38] fix(multimodal): truncate attachment context to the configured limit Attachment text from file fallbacks, image readouts, and archive extraction was previously passed through unbounded, so large attachments could blow past the configured context budget. A new limit_context helper now truncates each of these strings to the effective file context limit and appends a truncation marker. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index ceaf31b60..78ad06b76 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -368,7 +368,7 @@ where let (payload, archive_text) = self.file_fallback(&attachment).await; file_payloads.push(payload); if let Some(text) = archive_text { - file_fallback_text.push(text); + file_fallback_text.push(self.limit_context(text)); } } } @@ -422,9 +422,11 @@ where let (payload, archive_text) = self.file_fallback(&attachment).await; let rendered = compose_multimodal_message("", &[], &[payload]); match archive_text { - Some(archive) if !rendered.is_empty() => format!("{rendered}\n\n{archive}"), + Some(archive) if !rendered.is_empty() => { + self.limit_context(format!("{rendered}\n\n{archive}")) + } Some(archive) => archive, - None => rendered, + None => self.limit_context(rendered), } } }; @@ -496,10 +498,10 @@ where .map_err(|error| model_error(&error))?, None => "Image contents are unavailable to this text-only model.".to_string(), }; - Ok(format!( + Ok(self.limit_context(format!( "[Image attachment: {} bytes; mime={mime}]\nImage readout (untrusted):\n{details}", image.bytes.len() - )) + ))) } async fn file_fallback( @@ -537,7 +539,7 @@ where None, self.config.files.effective().2, ), - Some(text), + Some(self.limit_context(text)), ); } @@ -609,6 +611,15 @@ where .as_ref()? .unavailable(scope, modality, source, &error.to_string()) } + + fn limit_context(&self, mut text: String) -> String { + let limit = self.config.files.effective().2; + if text.chars().count() > limit { + text = text.chars().take(limit).collect(); + text.push_str("\n[Attachment context truncated]"); + } + text + } } #[async_trait] From 65865b6a7c6bc5063dca63f79b1e752ec6d4b063 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:03:07 +0300 Subject: [PATCH 24/38] test(multimodal): cover unicode-safe truncation of attachment context Add a test that exercises image and archive fallback context with a long unicode readout and a zip listing of many long names, asserting both contexts are truncated and remain within the configured character limit. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 92 ++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index de59e1123..14284d3e0 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -9,7 +9,7 @@ use std::{ use futures::StreamExt; use tinyinference_llm::{ - message::{ImageRef, UserMessage}, + message::{ImageRef, MediaRef, UserMessage}, model::{Modalities, ModelStreamItem}, }; @@ -146,6 +146,19 @@ impl MediaImageReadout for Readout { } } +struct LongUnicodeReadout; + +#[async_trait] +impl MediaImageReadout for LongUnicodeReadout { + async fn readout( + &self, + _scope: &String, + _image: &ResolvedAttachment, + ) -> std::result::Result { + Ok("🧪".repeat(2_000)) + } +} + struct SafeHistory; impl MediaHistoryFallback for SafeHistory { @@ -903,3 +916,80 @@ async fn nonvision_archive_fallback_lists_entries_without_extracting_them() { assert!(rendered.contains("notes/readme.txt")); assert!(rendered.contains("original was not extracted")); } + +#[tokio::test] +async fn image_and_archive_fallback_context_is_unicode_safe_and_bounded() { + let mut bytes = std::io::Cursor::new(Vec::new()); + { + let mut zip = zip::ZipWriter::new(&mut bytes); + for index in 0..40 { + let name = format!("entries/{index:02}-{}.txt", "long-name".repeat(12)); + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(b"entry bytes are not extracted") + .unwrap(); + } + zip.finish().unwrap(); + } + let model = Arc::new(CaptureModel::new(false)); + let mut sources = HashMap::new(); + sources.insert( + "image".into(), + attachment("photo.png", "image/png", b"\x89PNG\r\n\x1a\n"), + ); + sources.insert( + "archive".into(), + attachment("bundle.zip", "application/zip", bytes.get_ref()), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig { + files: FileLimits { + max_extracted_text_chars: 1_000, + ..FileLimits::default() + }, + ..MediaRequestConfig::default() + }, + resolver, + Arc::default(), + ) + .with_image_readout(Arc::new(LongUnicodeReadout)); + let req = ModelRequest { + messages: vec![Message::User(UserMessage { + content: vec![ + ContentBlock::Image(ImageRef { + url: "image".into(), + mime_type: Some("image/png".into()), + }), + ContentBlock::Document(MediaRef::path("archive")), + ], + })], + ..ModelRequest::default() + }; + + wrapped.invoke(&"tenant-a".into(), req).await.unwrap(); + + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("prepared user message retained") + }; + let contexts = user + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text), + _ => None, + }) + .collect::>(); + assert_eq!(contexts.len(), 2); + assert!(contexts.iter().all(|text| text.contains("Attachment context truncated"))); + assert!(contexts.iter().all(|text| text.chars().count() <= 1_000 + 31)); + assert!(contexts.iter().any(|text| text.contains("🧪"))); + assert!(contexts + .iter() + .any(|text| text.contains("Archive listing (names are untrusted"))); +} From 6248bf35545179aff917edf929995ac984761325 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:03:14 +0300 Subject: [PATCH 25/38] refactor(multimodal): rename historical media block budget to items The config field and its documentation now count individual media items rather than blocks, since each media marker and typed media block is resolved separately. The scope callback docs were also expanded to note that it runs before media counting, must not perform source I/O, and is invoked even for requests with no user messages. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/request.rs | 15 +++++++++------ .../src/multimodal/request_tests.rs | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 78ad06b76..43b33710a 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -35,8 +35,9 @@ pub struct MediaRequestConfig { pub images: ImageLimits, /// File count, size, extraction, and remote-fetch limits. pub files: FileLimits, - /// Maximum media-bearing historical blocks to resolve, newest first. - pub max_historical_media_blocks: usize, + /// Maximum historical media items to resolve, newest first. Each media + /// marker and typed media block counts as one item. + pub max_historical_media_items: usize, } impl Default for MediaRequestConfig { @@ -44,7 +45,7 @@ impl Default for MediaRequestConfig { Self { images: ImageLimits::default(), files: FileLimits::default(), - max_historical_media_blocks: 8, + max_historical_media_items: 8, } } } @@ -121,8 +122,10 @@ where /// /// The callback returns the caller's already-authorized scope from model /// state or trusted host metadata. It receives only the disposable request - /// copy, so it may consume a host carrier without changing the canonical - /// transcript. It is called separately for each invoke or stream. + /// copy, so it may consume a host carrier or normalize host-owned markers + /// without changing the canonical transcript. It runs before media + /// counting, must not perform source I/O, and is called separately for each + /// invoke or stream (including requests with no user messages). pub fn new( inner: Arc>, config: MediaRequestConfig, @@ -205,7 +208,7 @@ where let selected_history = select_history_media( &request.messages, latest_user, - self.config.max_historical_media_blocks, + self.config.max_historical_media_items, ); for (message_index, message) in request.messages.iter_mut().enumerate() { let Message::User(user) = message else { diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 14284d3e0..4293f48f6 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -513,7 +513,7 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { ..Resolver::default() }); let config = MediaRequestConfig { - max_historical_media_blocks: 1, + max_historical_media_items: 1, ..MediaRequestConfig::default() }; let wrapped = wrapper(model.clone(), config, resolver.clone(), Arc::default()); @@ -571,7 +571,7 @@ async fn historical_marker_budget_selects_only_the_newest_references_in_a_text_b let wrapped = wrapper( model.clone(), MediaRequestConfig { - max_historical_media_blocks: 1, + max_historical_media_items: 1, ..MediaRequestConfig::default() }, resolver.clone(), From d02cdd07ff12d913d707cb3bd899c596c7ab889e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:03:41 +0300 Subject: [PATCH 26/38] style: reformat multimodal request tests Reformat assertions and chained calls in the multimodal request tests to satisfy rustfmt line-width rules. No test behaviour or coverage changed. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 4293f48f6..e7a6f0fd4 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -593,7 +593,10 @@ async fn historical_marker_budget_selects_only_the_newest_references_in_a_text_b .unwrap(); assert_eq!(resolver.calls.load(Ordering::SeqCst), 1); - assert_eq!(resolver.resolved_sources.lock().unwrap().as_slice(), ["newer"]); + assert_eq!( + resolver.resolved_sources.lock().unwrap().as_slice(), + ["newer"] + ); let captured = model.requests.lock().unwrap(); let Message::User(old_user) = &captured[0].messages[0] else { panic!("historical message retained") @@ -602,10 +605,12 @@ async fn historical_marker_budget_selects_only_the_newest_references_in_a_text_b &old_user.content[0], ContentBlock::Text(text) if text.contains("omitted by history budget") )); - assert!(old_user - .content - .iter() - .any(|block| matches!(block, ContentBlock::Image(_)))); + assert!( + old_user + .content + .iter() + .any(|block| matches!(block, ContentBlock::Image(_))) + ); } #[tokio::test] @@ -670,10 +675,12 @@ async fn trusted_preparer_consumes_carrier_when_request_has_no_user_message() { wrapped.invoke(&"tenant-a".into(), original).await.unwrap(); - assert!(model.requests.lock().unwrap()[0] - .metadata - .get("trusted_scope") - .is_none()); + assert!( + model.requests.lock().unwrap()[0] + .metadata + .get("trusted_scope") + .is_none() + ); } #[tokio::test] @@ -926,8 +933,7 @@ async fn image_and_archive_fallback_context_is_unicode_safe_and_bounded() { let name = format!("entries/{index:02}-{}.txt", "long-name".repeat(12)); zip.start_file(name, zip::write::SimpleFileOptions::default()) .unwrap(); - zip.write_all(b"entry bytes are not extracted") - .unwrap(); + zip.write_all(b"entry bytes are not extracted").unwrap(); } zip.finish().unwrap(); } @@ -986,10 +992,20 @@ async fn image_and_archive_fallback_context_is_unicode_safe_and_bounded() { }) .collect::>(); assert_eq!(contexts.len(), 2); - assert!(contexts.iter().all(|text| text.contains("Attachment context truncated"))); - assert!(contexts.iter().all(|text| text.chars().count() <= 1_000 + 31)); + assert!( + contexts + .iter() + .all(|text| text.contains("Attachment context truncated")) + ); + assert!( + contexts + .iter() + .all(|text| text.chars().count() <= 1_000 + 31) + ); assert!(contexts.iter().any(|text| text.contains("🧪"))); - assert!(contexts - .iter() - .any(|text| text.contains("Archive listing (names are untrusted"))); + assert!( + contexts + .iter() + .any(|text| text.contains("Archive listing (names are untrusted")) + ); } From a82dd149660743f55ba4efebeb942171d4390cae Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:04:21 +0300 Subject: [PATCH 27/38] test(multimodal): compare request fields individually in normalization test The assertion now checks messages and metadata separately instead of relying on whole-struct equality, so the test no longer depends on the request type's PartialEq implementation. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/request_tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index e7a6f0fd4..c6cea44f3 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -651,7 +651,8 @@ async fn trusted_preparer_normalization_is_counted_before_resolution() { assert!(error.to_string().contains("max=1, found=2")); assert_eq!(resolver.calls.load(Ordering::SeqCst), 0); - assert_eq!(original, canonical); + assert_eq!(original.messages, canonical.messages); + assert_eq!(original.metadata, canonical.metadata); } #[tokio::test] From 8ee64e589392f1d7298575b3a407993222b8d136 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:04:40 +0300 Subject: [PATCH 28/38] docs(multimodal): document request preparation and fallback limits Clarify the request model's budget and fallback behavior, note the synchronous trusted-request callback's constraints, and add the request test module to the file table. Also document that image readout and archive fallback context is capped by max_extracted_text_chars and marked when truncated. Auto-committed-on: dragonfly --- .../src/multimodal/README.md | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/README.md b/crates/tinyagents-harness/src/multimodal/README.md index 2dcfec76d..97f414a61 100644 --- a/crates/tinyagents-harness/src/multimodal/README.md +++ b/crates/tinyagents-harness/src/multimodal/README.md @@ -48,11 +48,14 @@ format contributes a header naming it plus a content hash. `escape_attr`). - [`request`] — [`MediaRequestModel`], a `ChatModel` decorator that prepares a disposable request, checks counts before reads, preserves per-call scope, - applies a bounded historical-media budget, dispatches native media only when - both model and transport support it, and renders text/archive/image - fallbacks. The host supplies [`MediaSourceResolver`] for authorized source - access, [`MediaHistoryFallback`] for its safe history behavior, and may - supply [`MediaImageReadout`] plus selected-model [`ModelProfile`] facts. + applies a bounded historical-media-item budget, dispatches native media only + when both model and transport support it, and renders character-bounded + text/archive/image fallbacks. The host supplies [`MediaSourceResolver`] for + authorized source access, [`MediaHistoryFallback`] for its safe history + behavior, and may supply [`MediaImageReadout`] plus selected-model + [`ModelProfile`] facts. Its synchronous trusted-request callback may consume + request metadata or normalize host markers on the disposable copy before + counting; it must not resolve sources or perform other I/O. - [`error`] — [`MultimodalError`] and its [`Result`] alias; every variant carries the offending `input` verbatim so a multi-attachment turn's failure is attributable. @@ -76,7 +79,7 @@ format contributes a header naming it plus a content hash. | `payload.rs` | `FilePayload`, message composition, truncation, hashing. | | `request.rs` | `MediaRequestModel` and host scope/resolver/fallback seams. | | `error.rs` | `MultimodalError`, `Result`. | -| `mod_tests.rs`, `resolve_tests.rs`, `archive/ops_tests.rs`, `png_tests.rs`, `blob_stash_tests.rs` | Marker, MIME, size, path, intake, archive, PNG and stash behavior. | +| `mod_tests.rs`, `resolve_tests.rs`, `request_tests.rs`, `archive/ops_tests.rs`, `png_tests.rs`, `blob_stash_tests.rs` | Marker, MIME, size, path, intake, request preparation, archive, PNG and stash behavior. | ## Operational constraints @@ -86,6 +89,9 @@ format contributes a header naming it plus a content hash. - **Preserve the hard disable.** `max_files == 0` means *none* and remains zero in `FileLimits::effective`. `MediaRequestModel` checks it before asking the host source resolver to read anything. +- Final image readout and archive fallback context is capped using + `max_extracted_text_chars` and Unicode scalar values, then marked when + truncated. - What stays with the host, deliberately: the `TextExtractor` implementation (which parser, if any, and its timeout), the attachment stash (where bytes live between ingress and dispatch), image-readout model choice, source From b5d72e62dbf5a26a9ae8fe0cf2ebdebbb285b449 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:05:08 +0300 Subject: [PATCH 29/38] fix(multimodal): apply context limit to file payloads File payloads and their fallback text are now composed into a separate context block that is passed through the context limiter before being appended to the message, so large file attachments no longer bypass the configured context budget. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 43b33710a..a599dbbb1 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -376,10 +376,22 @@ where } } - let composed = compose_multimodal_message(&clean_text, &[], &file_payloads); let mut output = Vec::new(); - let mut text_output = composed; - for fallback in image_fallback.into_iter().chain(file_fallback_text) { + let mut text_output = compose_multimodal_message(&clean_text, &[], &[]); + let mut file_context = compose_multimodal_message("", &[], &file_payloads); + for fallback in file_fallback_text { + if !file_context.is_empty() { + file_context.push_str("\n\n"); + } + file_context.push_str(&fallback); + } + if !file_context.is_empty() { + if !text_output.is_empty() { + text_output.push_str("\n\n"); + } + text_output.push_str(&self.limit_context(file_context)); + } + for fallback in image_fallback { if !text_output.is_empty() { text_output.push_str("\n\n"); } From f952cd84619b7bb5c497d2ac990f0f09f8205b6d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:05:18 +0300 Subject: [PATCH 30/38] fix(multimodal): correct stale comment on media counting The comment above the latest-turn media count still referred to deriving scope, which no longer happens at that point. It now says the count is taken before calling any source resolver, matching the code. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/request.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index a599dbbb1..fd05425e0 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -186,7 +186,7 @@ where return Ok(request); }; - // Count the raw latest-turn media before deriving scope or calling any + // Count the host-normalized latest-turn media before calling any // source resolver. This keeps configured limits meaningful for reads. let (images, files) = count_user_media(&request.messages[latest_user]); let (max_images, _) = self.config.images.effective(); From 4bfa239d1f84b26811bf51b1baffc8c72f93e849 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:06:16 +0300 Subject: [PATCH 31/38] feat(multimodal): carry host annotations alongside resolved media Media resolution now returns a ResolvedMedia wrapper that pairs the attachment with optional host-authored annotations, so safe workspace hints or recovered-source labels can be preserved next to native media and text fallbacks. Annotations are emitted before native blocks and merged into fallback text, while failed resolutions still cannot produce them. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 87 +++++++++++++++---- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index fd05425e0..b65e79e11 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -65,7 +65,37 @@ pub trait MediaSourceResolver: Send + Sync { modality: InputModality, source: &str, max_bytes: usize, - ) -> std::result::Result; + ) -> std::result::Result; +} + +/// A resolved attachment plus host-authored context to preserve beside it. +/// +/// Hosts can use `annotations` for safe workspace hints or recovered-source +/// labels. The wrapper emits them before native media and with text fallbacks; +/// failed resolutions cannot produce annotations. The host must not put +/// credentials, signed URLs, or resolver errors in this context. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedMedia { + /// The bounded attachment bytes and display metadata. + pub attachment: ResolvedAttachment, + /// Safe host-authored labels to keep alongside the attachment. + pub annotations: Vec, +} + +impl ResolvedMedia { + /// Wrap an attachment without additional host context. + pub fn new(attachment: ResolvedAttachment) -> Self { + Self { + attachment, + annotations: Vec::new(), + } + } + + /// Add one host-authored context line. + pub fn with_annotation(mut self, annotation: impl Into) -> Self { + self.annotations.push(annotation.into()); + self + } } /// Produce a text description for an image when the selected model cannot @@ -294,13 +324,14 @@ where } let mut native_images = Vec::new(); + let mut native_annotations = Vec::new(); let mut image_fallback = Vec::new(); for (index, source) in images.into_iter().enumerate() { if historical && !selected_images.is_some_and(|selected| selected[index]) { image_fallback.push(self.omission(scope, InputModality::Image, &source)); continue; } - let attachment = match self + let resolved = match self .resolve( scope, InputModality::Image, @@ -322,14 +353,19 @@ where } Err(error) => return Err(error), }; + let ResolvedMedia { + attachment, + annotations, + } = resolved; if self.native(InputModality::Image, &attachment.mime) { + native_annotations.extend(annotations); let mime = validated_image_mime(&attachment)?; native_images.push(ContentBlock::Image(ImageRef { url: encode_data_uri(&mime, &attachment.bytes), mime_type: Some(mime), })); } else { - image_fallback.push(self.image_fallback(scope, &attachment).await?); + image_fallback.push(self.image_fallback(scope, &attachment, &annotations).await?); } } @@ -340,7 +376,7 @@ where file_fallback_text.push(self.omission(scope, InputModality::Document, &source)); continue; } - let attachment = match self + let resolved = match self .resolve( scope, InputModality::Document, @@ -362,7 +398,12 @@ where } Err(error) => return Err(error), }; + let ResolvedMedia { + attachment, + annotations, + } = resolved; if self.native(InputModality::Document, &attachment.mime) { + native_annotations.extend(annotations); native_images.push(ContentBlock::Document(MediaRef::base64( STANDARD.encode(&attachment.bytes), attachment.mime, @@ -370,6 +411,7 @@ where } else { let (payload, archive_text) = self.file_fallback(&attachment).await; file_payloads.push(payload); + file_fallback_text.extend(annotations); if let Some(text) = archive_text { file_fallback_text.push(self.limit_context(text)); } @@ -400,6 +442,11 @@ where if !text_output.is_empty() { output.push(ContentBlock::Text(text_output)); } + output.extend( + native_annotations + .into_iter() + .map(ContentBlock::Text), + ); output.extend(native_images); Ok(output) } @@ -420,19 +467,27 @@ where InputModality::Image => self.config.images.max_image_bytes(), _ => self.config.files.max_file_bytes(), }; - let mut attachment = self + let resolved = self .resolve(scope, modality, &source, max_bytes, false) .await?; + let ResolvedMedia { + mut attachment, + annotations, + } = resolved; if attachment.mime == "application/octet-stream" && let Some(mime) = hinted_mime { attachment.mime = mime; } if self.native(modality, &attachment.mime) { - return Ok(vec![native_block(modality, attachment)?]); + let mut output: Vec<_> = annotations.into_iter().map(ContentBlock::Text).collect(); + output.push(native_block(modality, attachment)?); + return Ok(output); } let fallback = match modality { - InputModality::Image => self.image_fallback(scope, &attachment).await?, + InputModality::Image => { + self.image_fallback(scope, &attachment, &annotations).await? + } _ => { let (payload, archive_text) = self.file_fallback(&attachment).await; let rendered = compose_multimodal_message("", &[], &[payload]); @@ -445,7 +500,8 @@ where } } }; - Ok(vec![ContentBlock::Text(fallback)]) + let fallback = join_annotations(&annotations, &fallback); + Ok(vec![ContentBlock::Text(self.limit_context(fallback))]) } async fn resolve( @@ -455,7 +511,7 @@ where source: &str, max_bytes: usize, require_file_mime: bool, - ) -> tinyinference_llm::Result { + ) -> tinyinference_llm::Result { let remote = source.starts_with("http://") || source.starts_with("https://"); let remote_allowed = match modality { InputModality::Image => self.config.images.allow_remote_fetch, @@ -471,16 +527,16 @@ where .resolve(scope, modality, source, max_bytes) .await .map_err(|error| model_error(&error))?; - if attachment.bytes.len() > max_bytes { + if attachment.attachment.bytes.len() > max_bytes { return Err(model_error("attachment exceeds configured byte limit")); } - if modality == InputModality::Image && attachment.bytes.is_empty() { + if modality == InputModality::Image && attachment.attachment.bytes.is_empty() { return Err(model_error("image reference has no bytes")); } - if require_file_mime && !self.config.files.is_mime_allowed(&attachment.mime) { + if require_file_mime && !self.config.files.is_mime_allowed(&attachment.attachment.mime) { return Err(model_error(&format!( "file MIME type '{}' is not allowed; supported: {}", - attachment.mime, + attachment.attachment.mime, self.config.files.supported_rendered() ))); } @@ -504,6 +560,7 @@ where &self, scope: &Scope, image: &ResolvedAttachment, + annotations: &[String], ) -> tinyinference_llm::Result { let mime = validated_image_mime(image)?; let details = match &self.image_readout { @@ -513,10 +570,10 @@ where .map_err(|error| model_error(&error))?, None => "Image contents are unavailable to this text-only model.".to_string(), }; - Ok(self.limit_context(format!( + Ok(self.limit_context(join_annotations(annotations, &format!( "[Image attachment: {} bytes; mime={mime}]\nImage readout (untrusted):\n{details}", image.bytes.len() - ))) + )))) } async fn file_fallback( From 075260e5033d84e38dd5f1dbca7f97b1d48d51fa Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:07:15 +0300 Subject: [PATCH 32/38] feat(multimodal): carry resolver annotations into resolved media Media source resolvers can now attach annotations to resolved media via a ResolvedMedia type, and those annotations are joined into the fallback text alongside the existing readout. This lets resolvers surface provenance or recovery context that would otherwise be lost when an attachment is inlined. Auto-committed-on: dragonfly --- .../tinyagents-harness/src/multimodal/mod.rs | 2 +- .../src/multimodal/request.rs | 41 +++++++--- .../src/multimodal/request_tests.rs | 74 ++++++++++++++++++- 3 files changed, 101 insertions(+), 16 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/mod.rs b/crates/tinyagents-harness/src/multimodal/mod.rs index e09d65a29..be01231fe 100644 --- a/crates/tinyagents-harness/src/multimodal/mod.rs +++ b/crates/tinyagents-harness/src/multimodal/mod.rs @@ -105,7 +105,7 @@ pub use markers::{ pub use payload::{FilePayload, compose_multimodal_message, sha256_prefix, truncate_chars}; pub use request::{ MediaHistoryFallback, MediaImageReadout, MediaRequestConfig, MediaRequestModel, - MediaSourceResolver, + MediaSourceResolver, ResolvedMedia, }; pub use resolve::{ NoTextExtractor, TextExtractor, resolve_attachment, resolve_file, resolve_image, diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index b65e79e11..0e96c2896 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -365,7 +365,10 @@ where mime_type: Some(mime), })); } else { - image_fallback.push(self.image_fallback(scope, &attachment, &annotations).await?); + image_fallback.push( + self.image_fallback(scope, &attachment, &annotations) + .await?, + ); } } @@ -442,11 +445,7 @@ where if !text_output.is_empty() { output.push(ContentBlock::Text(text_output)); } - output.extend( - native_annotations - .into_iter() - .map(ContentBlock::Text), - ); + output.extend(native_annotations.into_iter().map(ContentBlock::Text)); output.extend(native_images); Ok(output) } @@ -486,7 +485,8 @@ where } let fallback = match modality { InputModality::Image => { - self.image_fallback(scope, &attachment, &annotations).await? + self.image_fallback(scope, &attachment, &annotations) + .await? } _ => { let (payload, archive_text) = self.file_fallback(&attachment).await; @@ -533,7 +533,12 @@ where if modality == InputModality::Image && attachment.attachment.bytes.is_empty() { return Err(model_error("image reference has no bytes")); } - if require_file_mime && !self.config.files.is_mime_allowed(&attachment.attachment.mime) { + if require_file_mime + && !self + .config + .files + .is_mime_allowed(&attachment.attachment.mime) + { return Err(model_error(&format!( "file MIME type '{}' is not allowed; supported: {}", attachment.attachment.mime, @@ -570,10 +575,13 @@ where .map_err(|error| model_error(&error))?, None => "Image contents are unavailable to this text-only model.".to_string(), }; - Ok(self.limit_context(join_annotations(annotations, &format!( - "[Image attachment: {} bytes; mime={mime}]\nImage readout (untrusted):\n{details}", - image.bytes.len() - )))) + Ok(self.limit_context(join_annotations( + annotations, + &format!( + "[Image attachment: {} bytes; mime={mime}]\nImage readout (untrusted):\n{details}", + image.bytes.len() + ), + ))) } async fn file_fallback( @@ -924,6 +932,15 @@ fn model_error(message: &str) -> tinyinference_llm::Error { tinyinference_llm::Error::Model(message.to_string()) } +fn join_annotations(annotations: &[String], body: &str) -> String { + let mut output = annotations.join("\n"); + if !output.is_empty() && !body.is_empty() { + output.push_str("\n\n"); + } + output.push_str(body); + output +} + #[cfg(test)] #[path = "request_tests.rs"] mod tests; diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index c6cea44f3..ba1c588c3 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -62,6 +62,7 @@ impl CaptureModel { #[derive(Default)] struct Resolver { sources: HashMap, + annotations: HashMap>, calls: AtomicUsize, scopes: Mutex>, resolved_sources: Mutex>, @@ -80,7 +81,7 @@ impl MediaSourceResolver for PendingResolver { _modality: InputModality, _source: &str, _max_bytes: usize, - ) -> std::result::Result { + ) -> std::result::Result { self.entered.notify_one(); std::future::pending().await } @@ -94,7 +95,7 @@ impl MediaSourceResolver for Resolver { _modality: InputModality, source: &str, max_bytes: usize, - ) -> std::result::Result { + ) -> std::result::Result { self.calls.fetch_add(1, Ordering::SeqCst); self.scopes.lock().unwrap().push(scope.clone()); self.resolved_sources @@ -109,7 +110,11 @@ impl MediaSourceResolver for Resolver { if attachment.bytes.len() > max_bytes { return Err("source exceeds test limit".into()); } - Ok(attachment) + let mut resolved = ResolvedMedia::new(attachment); + for annotation in self.annotations.get(source).into_iter().flatten() { + resolved = resolved.with_annotation(annotation.clone()); + } + Ok(resolved) } } @@ -684,6 +689,69 @@ async fn trusted_preparer_consumes_carrier_when_request_has_no_user_message() { ); } +#[tokio::test] +async fn resolver_annotations_survive_next_to_native_media_only_after_success() { + let model = Arc::new(CaptureModel::new(true)); + let resolver = Arc::new(Resolver { + sources: HashMap::from([( + "recovered-image".into(), + attachment("image.png", "image/png", b"\x89PNG\r\n\x1a\n"), + )]), + annotations: HashMap::from([( + "recovered-image".into(), + vec!["[Recovered attachment: image/png; workspace path: image.png]".into()], + )]), + ..Resolver::default() + }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig::default(), + resolver.clone(), + Arc::default(), + ); + + wrapped + .invoke(&"tenant-a".into(), request("[IMAGE:recovered-image]")) + .await + .unwrap(); + + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("prepared user message retained") + }; + assert!(matches!( + &user.content[0], + ContentBlock::Text(text) if text.contains("Recovered attachment") + )); + assert!(matches!( + &user.content[1], + ContentBlock::Image(image) if image.url.starts_with("data:image/png;base64,") + )); + drop(captured); + + let missing_model = Arc::new(CaptureModel::new(true)); + let missing_resolver = Arc::new(Resolver { + annotations: HashMap::from([( + "missing-image".into(), + vec!["[Recovered attachment: fake label]".into()], + )]), + ..Resolver::default() + }); + let missing_wrapper = wrapper( + missing_model.clone(), + MediaRequestConfig::default(), + missing_resolver, + Arc::default(), + ); + assert!( + missing_wrapper + .invoke(&"tenant-a".into(), request("[IMAGE:missing-image]")) + .await + .is_err() + ); + assert!(missing_model.requests.lock().unwrap().is_empty()); +} + #[tokio::test] async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() { let model = Arc::new(CaptureModel::new(true)); From 2c94ca63b0eb189b42df317f14d5b109dc2ea522 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:08:22 +0300 Subject: [PATCH 33/38] refactor(multimodal): build file context per attachment File payloads, annotations, and archive text are now combined into a single context string per attachment before being appended to the text output, instead of accumulating payloads and fallback text separately and merging them at the end. This keeps each document's annotations and archive text adjacent to its payload and applies the context limit once per attachment. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 0e96c2896..8c8708a25 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -372,11 +372,10 @@ where } } - let mut file_payloads = Vec::new(); - let mut file_fallback_text = Vec::new(); + let mut file_contexts = Vec::new(); for (index, source) in files.into_iter().enumerate() { if historical && !selected_files.is_some_and(|selected| selected[index]) { - file_fallback_text.push(self.omission(scope, InputModality::Document, &source)); + file_contexts.push(self.omission(scope, InputModality::Document, &source)); continue; } let resolved = match self @@ -394,7 +393,7 @@ where if let Some(placeholder) = self.recover_history(scope, InputModality::Document, &source, &error) { - file_fallback_text.push(placeholder); + file_contexts.push(placeholder); continue; } return Err(error); @@ -413,28 +412,24 @@ where ))); } else { let (payload, archive_text) = self.file_fallback(&attachment).await; - file_payloads.push(payload); - file_fallback_text.extend(annotations); - if let Some(text) = archive_text { - file_fallback_text.push(self.limit_context(text)); + let mut context = compose_multimodal_message("", &[], &[payload]); + if let Some(archive) = archive_text { + if !context.is_empty() { + context.push_str("\n\n"); + } + context.push_str(&archive); } + file_contexts.push(self.limit_context(join_annotations(&annotations, &context))); } } let mut output = Vec::new(); let mut text_output = compose_multimodal_message(&clean_text, &[], &[]); - let mut file_context = compose_multimodal_message("", &[], &file_payloads); - for fallback in file_fallback_text { - if !file_context.is_empty() { - file_context.push_str("\n\n"); - } - file_context.push_str(&fallback); - } - if !file_context.is_empty() { + for file_context in file_contexts { if !text_output.is_empty() { text_output.push_str("\n\n"); } - text_output.push_str(&self.limit_context(file_context)); + text_output.push_str(&file_context); } for fallback in image_fallback { if !text_output.is_empty() { From 901a2390691410820802a7fa0b7eaf81f3034a9b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:08:32 +0300 Subject: [PATCH 34/38] test(multimodal): cover per-attachment file fallback limits Add a test asserting that the extracted-text character limit applies to each attachment independently rather than being shared across a turn. Two files whose combined text exceeds the configured limit must both still be rendered in full. Auto-committed-on: dragonfly --- .../src/multimodal/request_tests.rs | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index ba1c588c3..161abc08e 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -1078,3 +1078,57 @@ async fn image_and_archive_fallback_context_is_unicode_safe_and_bounded() { .any(|text| text.contains("Archive listing (names are untrusted")) ); } + +#[tokio::test] +async fn file_fallback_limit_applies_per_attachment_not_across_the_turn() { + let model = Arc::new(CaptureModel::new(false)); + let mut sources = HashMap::new(); + sources.insert( + "first".into(), + attachment("first.txt", "text/plain", "FIRST ".repeat(150).as_bytes()), + ); + sources.insert( + "second".into(), + attachment("second.txt", "text/plain", "SECOND ".repeat(120).as_bytes()), + ); + let resolver = Arc::new(Resolver { + sources, + ..Resolver::default() + }); + let wrapped = wrapper( + model.clone(), + MediaRequestConfig { + files: FileLimits { + max_extracted_text_chars: 1_000, + ..FileLimits::default() + }, + ..MediaRequestConfig::default() + }, + resolver, + Arc::default(), + ); + + wrapped + .invoke(&"tenant-a".into(), request("[FILE:first] [FILE:second]")) + .await + .unwrap(); + + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("prepared user message retained") + }; + let rendered = user + .content + .iter() + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + assert!(rendered.contains("first.txt")); + assert!(rendered.contains("second.txt")); + assert!(rendered.contains("FIRST")); + assert!(rendered.contains("SECOND")); + assert!(rendered.chars().count() > 1_000); +} From da3a2861dc8c216582ce26e581633d66c400f823 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:10:44 +0300 Subject: [PATCH 35/38] refactor(multimodal): extract request scope factory type alias Introduce a RequestScopeFactory type alias for the scope closure stored on MediaRequestModel, and rework the multimodal request tests to build ModelRequest values with struct update syntax and to scope captured-request locks in blocks so they are released before later assertions. Auto-committed-on: dragonfly --- .../src/multimodal/request.rs | 6 +- .../src/multimodal/request_tests.rs | 217 ++++++++++-------- 2 files changed, 120 insertions(+), 103 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/request.rs b/crates/tinyagents-harness/src/multimodal/request.rs index 8c8708a25..3fa82155e 100644 --- a/crates/tinyagents-harness/src/multimodal/request.rs +++ b/crates/tinyagents-harness/src/multimodal/request.rs @@ -28,6 +28,9 @@ use super::{ payload::compose_multimodal_message, }; +type RequestScopeFactory = + dyn Fn(&State, &mut ModelRequest) -> std::result::Result + Send + Sync; + /// Per-turn attachment limits copied from host configuration. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MediaRequestConfig { @@ -134,8 +137,7 @@ pub trait MediaHistoryFallback: Send + Sync { pub struct MediaRequestModel { inner: Arc>, config: MediaRequestConfig, - scope_for_request: - Arc std::result::Result + Send + Sync>, + scope_for_request: Arc>, resolver: Arc>, extractor: Arc, image_readout: Option>>, diff --git a/crates/tinyagents-harness/src/multimodal/request_tests.rs b/crates/tinyagents-harness/src/multimodal/request_tests.rs index 161abc08e..774839e05 100644 --- a/crates/tinyagents-harness/src/multimodal/request_tests.rs +++ b/crates/tinyagents-harness/src/multimodal/request_tests.rs @@ -46,10 +46,12 @@ impl ChatModel for CaptureModel { impl CaptureModel { fn new(native_images: bool) -> Self { - let mut profile = ModelProfile::default(); - profile.modalities = Modalities { - image_in: native_images, - ..Modalities::default() + let profile = ModelProfile { + modalities: Modalities { + image_in: native_images, + ..Modalities::default() + }, + ..ModelProfile::default() }; Self { requests: Mutex::default(), @@ -522,24 +524,26 @@ async fn historical_media_budget_bounds_reads_and_replaces_older_items() { ..MediaRequestConfig::default() }; let wrapped = wrapper(model.clone(), config, resolver.clone(), Arc::default()); - let mut historical_request = ModelRequest::default(); - historical_request.messages = vec![ - Message::User(UserMessage { - content: vec![ContentBlock::Image(ImageRef { - url: "oldest".into(), - mime_type: Some("image/png".into()), - })], - }), - Message::User(UserMessage { - content: vec![ContentBlock::Image(ImageRef { - url: "newest".into(), - mime_type: Some("image/png".into()), - })], - }), - Message::User(UserMessage { - content: vec![ContentBlock::Text("latest".into())], - }), - ]; + let historical_request = ModelRequest { + messages: vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "oldest".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "newest".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest".into())], + }), + ], + ..ModelRequest::default() + }; wrapped .invoke(&"tenant-a".into(), historical_request) @@ -582,15 +586,17 @@ async fn historical_marker_budget_selects_only_the_newest_references_in_a_text_b resolver.clone(), Arc::default(), ); - let mut historical_request = ModelRequest::default(); - historical_request.messages = vec![ - Message::User(UserMessage { - content: vec![ContentBlock::Text("[IMAGE:older] [IMAGE:newer]".into())], - }), - Message::User(UserMessage { - content: vec![ContentBlock::Text("latest".into())], - }), - ]; + let historical_request = ModelRequest { + messages: vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Text("[IMAGE:older] [IMAGE:newer]".into())], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest".into())], + }), + ], + ..ModelRequest::default() + }; wrapped .invoke(&"tenant-a".into(), historical_request) @@ -676,8 +682,10 @@ async fn trusted_preparer_consumes_carrier_when_request_has_no_user_message() { resolver, Arc::new(NoTextExtractor), ); - let mut original = ModelRequest::default(); - original.metadata = serde_json::json!({"trusted_scope":"workspace-a"}); + let original = ModelRequest { + metadata: serde_json::json!({"trusted_scope":"workspace-a"}), + ..ModelRequest::default() + }; wrapped.invoke(&"tenant-a".into(), original).await.unwrap(); @@ -715,19 +723,20 @@ async fn resolver_annotations_survive_next_to_native_media_only_after_success() .await .unwrap(); - let captured = model.requests.lock().unwrap(); - let Message::User(user) = &captured[0].messages[0] else { - panic!("prepared user message retained") - }; - assert!(matches!( - &user.content[0], - ContentBlock::Text(text) if text.contains("Recovered attachment") - )); - assert!(matches!( - &user.content[1], - ContentBlock::Image(image) if image.url.starts_with("data:image/png;base64,") - )); - drop(captured); + { + let captured = model.requests.lock().unwrap(); + let Message::User(user) = &captured[0].messages[0] else { + panic!("prepared user message retained") + }; + assert!(matches!( + &user.content[0], + ContentBlock::Text(text) if text.contains("Recovered attachment") + )); + assert!(matches!( + &user.content[1], + ContentBlock::Image(image) if image.url.starts_with("data:image/png;base64,") + )); + } let missing_model = Arc::new(CaptureModel::new(true)); let missing_resolver = Arc::new(Resolver { @@ -763,30 +772,33 @@ async fn missing_history_media_degrades_safely_but_missing_latest_media_fails() Arc::default(), ) .with_history_fallback(Arc::new(SafeHistory)); - let mut history = ModelRequest::default(); - history.messages = vec![ - Message::User(UserMessage { - content: vec![ContentBlock::Image(ImageRef { - url: "/private-value/also-private".into(), - mime_type: Some("image/png".into()), - })], - }), - Message::User(UserMessage { - content: vec![ContentBlock::Text("latest text".into())], - }), - ]; + let history = ModelRequest { + messages: vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Image(ImageRef { + url: "/private-value/also-private".into(), + mime_type: Some("image/png".into()), + })], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest text".into())], + }), + ], + ..ModelRequest::default() + }; wrapped.invoke(&"tenant-a".into(), history).await.unwrap(); - let calls = model.requests.lock().unwrap(); - let Message::User(old_user) = &calls[0].messages[0] else { - panic!("historical message retained") - }; - let ContentBlock::Text(placeholder) = &old_user.content[0] else { - panic!("missing historical media becomes text") - }; - assert!(placeholder.contains("safe-hint")); - assert!(!placeholder.contains("private-value")); - drop(calls); + { + let calls = model.requests.lock().unwrap(); + let Message::User(old_user) = &calls[0].messages[0] else { + panic!("historical message retained") + }; + let ContentBlock::Text(placeholder) = &old_user.content[0] else { + panic!("missing historical media becomes text") + }; + assert!(placeholder.contains("safe-hint")); + assert!(!placeholder.contains("private-value")); + } let error = wrapped .invoke(&"tenant-a".into(), request("[IMAGE:latest-missing]")) @@ -825,47 +837,50 @@ async fn historical_text_recovers_each_failed_marker_without_losing_prose_or_val Arc::default(), ) .with_history_fallback(history.clone()); - let mut historical_request = ModelRequest::default(); - historical_request.messages = vec![ - Message::User(UserMessage { - content: vec![ContentBlock::Text( - "Caption [IMAGE:good-image] then [IMAGE:https://example.test/photo?token=secret]" - .into(), - )], - }), - Message::User(UserMessage { - content: vec![ContentBlock::Text("latest text".into())], - }), - ]; + let historical_request = ModelRequest { + messages: vec![ + Message::User(UserMessage { + content: vec![ContentBlock::Text( + "Caption [IMAGE:good-image] then [IMAGE:https://example.test/photo?token=secret]" + .into(), + )], + }), + Message::User(UserMessage { + content: vec![ContentBlock::Text("latest text".into())], + }), + ], + ..ModelRequest::default() + }; wrapped .invoke(&"tenant-a".into(), historical_request) .await .unwrap(); - let captured = model.requests.lock().unwrap(); - let Message::User(old_user) = &captured[0].messages[0] else { - panic!("historical message retained") - }; - let rendered = old_user - .content - .iter() - .filter_map(|block| match block { - ContentBlock::Text(text) => Some(text.as_str()), - _ => None, - }) - .collect::>() - .join("\n"); - assert!(rendered.contains("Caption")); - assert!(rendered.contains("safe-hint")); - assert!(!rendered.contains("token=secret")); - assert!( - old_user + { + let captured = model.requests.lock().unwrap(); + let Message::User(old_user) = &captured[0].messages[0] else { + panic!("historical message retained") + }; + let rendered = old_user .content .iter() - .any(|block| matches!(block, ContentBlock::Image(_))) - ); - drop(captured); + .filter_map(|block| match block { + ContentBlock::Text(text) => Some(text.as_str()), + _ => None, + }) + .collect::>() + .join("\n"); + assert!(rendered.contains("Caption")); + assert!(rendered.contains("safe-hint")); + assert!(!rendered.contains("token=secret")); + assert!( + old_user + .content + .iter() + .any(|block| matches!(block, ContentBlock::Image(_))) + ); + } assert_eq!( history.unavailable.lock().unwrap().as_slice(), &[( From 49a84215e3eafb9ebe3b7c1e4acca98178521467 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:11:33 +0300 Subject: [PATCH 36/38] docs(multimodal): document resolver return type Clarify that a resolver returns ResolvedMedia so host-authored safe path or recovery annotations stay beside native media and in fallbacks. Auto-committed-on: dragonfly --- crates/tinyagents-harness/src/multimodal/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/tinyagents-harness/src/multimodal/README.md b/crates/tinyagents-harness/src/multimodal/README.md index 97f414a61..af86e5547 100644 --- a/crates/tinyagents-harness/src/multimodal/README.md +++ b/crates/tinyagents-harness/src/multimodal/README.md @@ -53,9 +53,11 @@ format contributes a header naming it plus a content hash. text/archive/image fallbacks. The host supplies [`MediaSourceResolver`] for authorized source access, [`MediaHistoryFallback`] for its safe history behavior, and may supply [`MediaImageReadout`] plus selected-model - [`ModelProfile`] facts. Its synchronous trusted-request callback may consume - request metadata or normalize host markers on the disposable copy before - counting; it must not resolve sources or perform other I/O. + [`ModelProfile`] facts. A resolver returns [`ResolvedMedia`] so host-authored + safe path or recovery annotations stay beside native media and in fallbacks. + Its synchronous trusted-request callback may consume request metadata or + normalize host markers on the disposable copy before counting; it must not + resolve sources or perform other I/O. - [`error`] — [`MultimodalError`] and its [`Result`] alias; every variant carries the offending `input` verbatim so a multi-attachment turn's failure is attributable. From cf554d4a35b6b81eb671873edf9e04ea8b8951a4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:12:44 +0300 Subject: [PATCH 37/38] docs: add host adoption map for 482 Document the host adoption map for issue 482, laying out the adoption path and current status so the plan is captured in the repository. Auto-committed-on: dragonfly --- docs/host-adoption/482-host-adoption-map.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 docs/host-adoption/482-host-adoption-map.md diff --git a/docs/host-adoption/482-host-adoption-map.md b/docs/host-adoption/482-host-adoption-map.md new file mode 100644 index 000000000..98c1e1b41 --- /dev/null +++ b/docs/host-adoption/482-host-adoption-map.md @@ -0,0 +1,21 @@ +# OpenHuman adoption map for tinyagents#482 + +This change moves the request-level media loop into +`tinyagents_harness::multimodal::MediaRequestModel`. The host keeps authority, +product selection, and host-specific presentation behind the seams below. + +| Concern | TinyAgents API and OpenHuman replacement | Behavior to preserve in the host | Verification recipe | +| --- | --- | --- | --- | +| Request preparation, scope, limits, and history | Replace the request walk in `crates/openhuman-core/src/agent/attachments/provider.rs::AttachmentModel::prepare` with `MediaRequestModel::new(inner, config, scope_for_request, resolver, extractor)`. In the synchronous `scope_for_request(&State, &mut ModelRequest) -> Result` callback, normalize host `[ATTACHMENT:…]` transcript markers on the disposable copy, then consume `take_request_scope(&mut request.metadata)` and return `AttachmentAccessScope`. The callback runs before counting/history indexing, including requests without a user message, and must not perform source I/O. Configure `max_historical_media_items: 8` and `with_history_fallback` for safe omission and recovery. | Keep `resolve_path`, `secure_open`, channel restrictions, URL/DNS/credential policy, selected-model profile construction, and limits-cache lookup in the host (`attachments/provider.rs`, `attachments/host.rs`, and `attachments/legacy.rs`). Preserve latest-turn count/hard-disable errors; historical-only media under a hard disable makes zero resolver calls and degrades to safe placeholders. The generic wrapper counts each marker and each typed block as a history item, resolves newest items first, retains prose and successful neighboring refs when an older ref fails, and never sends a per-request scope carrier to the inner model. | Use TinyAgents `multimodal::request::tests`: `hard_zero_rejects_before_any_source_read`, `hard_zero_also_blocks_media_only_present_in_history`, `ordinary_count_limit_is_checked_before_resolving_any_marker`, `historical_media_budget_bounds_reads_and_replaces_older_items`, `historical_marker_budget_selects_only_the_newest_references_in_a_text_block`, `historical_text_recovers_each_failed_marker_without_losing_prose_or_valid_media`, `trusted_preparer_normalization_is_counted_before_resolution`, `trusted_preparer_consumes_carrier_when_request_has_no_user_message`, and the concurrent scope/request-metadata tests. Retain host tests for path authorization, channel restrictions, marker normalization, and exact config mapping. | +| Native media, fallback context, and host annotations | Replace `AttachmentModel::prepare`'s native/fallback rendering and `provider_fallback.rs::fallback` block walk with `MediaRequestModel`'s capability-gated media conversion, `TextExtractor`, bounded archive listing, and non-vision fallbacks. The host resolver returns `ResolvedMedia { attachment, annotations }`; it can include safe workspace hints and recovered-source labels, which the wrapper carries before native media and into text fallbacks. Adapt existing image readout selection through `with_image_readout`. | Keep image-readout model selection, caching, document-module availability, extraction timeouts, and any cache policy in host callbacks. A host can wrap `MediaImageReadout` and `TextExtractor` to preserve existing caching. Keep workspace paths and recovery wording host-authored in `ResolvedMedia.annotations`; do not put signed URLs or raw resolver errors there. The wrapper preserves the exact native media payload, emits annotations only after a successful resolution, and applies the existing per-attachment Unicode-safe context cap to image readout, file payload plus archive listing, and annotations. | Use `nonvision_fallback_reads_images_and_uses_the_pdf_extractor`, `nonvision_archive_fallback_lists_entries_without_extracting_them`, `image_and_archive_fallback_context_is_unicode_safe_and_bounded`, `file_fallback_limit_applies_per_attachment_not_across_the_turn`, and `resolver_annotations_survive_next_to_native_media_only_after_success`. Retain host tests for readout-model selection, cache reuse, document feature/config gates, and safe path/recovery annotation rendering. | +| Limits and PDF extraction hook | Map product configuration into `MediaRequestConfig { images, files, max_historical_media_items }`, pass the current host `TextExtractor` implementation to `MediaRequestModel::new`, and remove the duplicate request-level extraction/fallback work from `crates/openhuman-core/src/agent/multimodal.rs`. `FileLimits::effective()` preserves `max_files == 0`; the request wrapper rejects latest-turn media before invoking the resolver and passes real resolved PDF bytes to the extractor for non-vision models. | Keep product config, MIME allowlists, storage/service selection, and the existing document parser implementation in OpenHuman. `TextExtractor` remains a host port, not a second parser framework. `max_files == 0` disables reads for images and files on the latest turn, while older media in prior turns receives safe omission placeholders without reads. | TinyAgents `multimodal::config` hard-zero regression plus `nonvision_fallback_reads_images_and_uses_the_pdf_extractor` verify the sentinel and meaningful fake PDF bytes/result. Keep host config-mapping tests and a parser-enabled integration test that verifies its configured extraction behavior without a real backend. | + +## TinyAgents request-level regression coverage + +Run `cargo test -p tinyagents-harness --features multimodal multimodal::request::tests` +for the request-wrapper suite. It verifies invoke and stream preparation on +disposable requests, errors and cancellation without canonical-history +mutation, concurrent immutable scope isolation, metadata consumption, hard +zero, raw-marker limits, bounded typed and marker history, per-marker recovery, +native/fallback behavior, PDF extraction, safe annotations, archive inspection, +and per-attachment Unicode-safe truncation. From 98124980cedfca35a671405d625d30a7874da453 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 17 Oct 2026 02:14:47 +0300 Subject: [PATCH 38/38] docs(host-adoption): fix attachment provider path in adoption map Correct the path to the attachment provider in the request preparation row so the adoption map points at the actual host file. Auto-committed-on: dragonfly --- docs/host-adoption/482-host-adoption-map.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/host-adoption/482-host-adoption-map.md b/docs/host-adoption/482-host-adoption-map.md index 98c1e1b41..f1f14c983 100644 --- a/docs/host-adoption/482-host-adoption-map.md +++ b/docs/host-adoption/482-host-adoption-map.md @@ -6,7 +6,7 @@ product selection, and host-specific presentation behind the seams below. | Concern | TinyAgents API and OpenHuman replacement | Behavior to preserve in the host | Verification recipe | | --- | --- | --- | --- | -| Request preparation, scope, limits, and history | Replace the request walk in `crates/openhuman-core/src/agent/attachments/provider.rs::AttachmentModel::prepare` with `MediaRequestModel::new(inner, config, scope_for_request, resolver, extractor)`. In the synchronous `scope_for_request(&State, &mut ModelRequest) -> Result` callback, normalize host `[ATTACHMENT:…]` transcript markers on the disposable copy, then consume `take_request_scope(&mut request.metadata)` and return `AttachmentAccessScope`. The callback runs before counting/history indexing, including requests without a user message, and must not perform source I/O. Configure `max_historical_media_items: 8` and `with_history_fallback` for safe omission and recovery. | Keep `resolve_path`, `secure_open`, channel restrictions, URL/DNS/credential policy, selected-model profile construction, and limits-cache lookup in the host (`attachments/provider.rs`, `attachments/host.rs`, and `attachments/legacy.rs`). Preserve latest-turn count/hard-disable errors; historical-only media under a hard disable makes zero resolver calls and degrades to safe placeholders. The generic wrapper counts each marker and each typed block as a history item, resolves newest items first, retains prose and successful neighboring refs when an older ref fails, and never sends a per-request scope carrier to the inner model. | Use TinyAgents `multimodal::request::tests`: `hard_zero_rejects_before_any_source_read`, `hard_zero_also_blocks_media_only_present_in_history`, `ordinary_count_limit_is_checked_before_resolving_any_marker`, `historical_media_budget_bounds_reads_and_replaces_older_items`, `historical_marker_budget_selects_only_the_newest_references_in_a_text_block`, `historical_text_recovers_each_failed_marker_without_losing_prose_or_valid_media`, `trusted_preparer_normalization_is_counted_before_resolution`, `trusted_preparer_consumes_carrier_when_request_has_no_user_message`, and the concurrent scope/request-metadata tests. Retain host tests for path authorization, channel restrictions, marker normalization, and exact config mapping. | +| Request preparation, scope, limits, and history | Replace the request walk in `crates/openhuman-core/src/attachments/provider.rs::AttachmentModel::prepare` with `MediaRequestModel::new(inner, config, scope_for_request, resolver, extractor)`. In the synchronous `scope_for_request(&State, &mut ModelRequest) -> Result` callback, normalize host `[ATTACHMENT:…]` transcript markers on the disposable copy, then consume `take_request_scope(&mut request.metadata)` and return `AttachmentAccessScope`. The callback runs before counting/history indexing, including requests without a user message, and must not perform source I/O. Configure `max_historical_media_items: 8` and `with_history_fallback` for safe omission and recovery. | Keep `resolve_path`, `secure_open`, channel restrictions, URL/DNS/credential policy, selected-model profile construction, and limits-cache lookup in the host (`attachments/provider.rs`, `attachments/host.rs`, and `attachments/legacy.rs`). Preserve latest-turn count/hard-disable errors; historical-only media under a hard disable makes zero resolver calls and degrades to safe placeholders. The generic wrapper counts each marker and each typed block as a history item, resolves newest items first, retains prose and successful neighboring refs when an older ref fails, and never sends a per-request scope carrier to the inner model. | Use TinyAgents `multimodal::request::tests`: `hard_zero_rejects_before_any_source_read`, `hard_zero_also_blocks_media_only_present_in_history`, `ordinary_count_limit_is_checked_before_resolving_any_marker`, `historical_media_budget_bounds_reads_and_replaces_older_items`, `historical_marker_budget_selects_only_the_newest_references_in_a_text_block`, `historical_text_recovers_each_failed_marker_without_losing_prose_or_valid_media`, `trusted_preparer_normalization_is_counted_before_resolution`, `trusted_preparer_consumes_carrier_when_request_has_no_user_message`, and the concurrent scope/request-metadata tests. Retain host tests for path authorization, channel restrictions, marker normalization, and exact config mapping. | | Native media, fallback context, and host annotations | Replace `AttachmentModel::prepare`'s native/fallback rendering and `provider_fallback.rs::fallback` block walk with `MediaRequestModel`'s capability-gated media conversion, `TextExtractor`, bounded archive listing, and non-vision fallbacks. The host resolver returns `ResolvedMedia { attachment, annotations }`; it can include safe workspace hints and recovered-source labels, which the wrapper carries before native media and into text fallbacks. Adapt existing image readout selection through `with_image_readout`. | Keep image-readout model selection, caching, document-module availability, extraction timeouts, and any cache policy in host callbacks. A host can wrap `MediaImageReadout` and `TextExtractor` to preserve existing caching. Keep workspace paths and recovery wording host-authored in `ResolvedMedia.annotations`; do not put signed URLs or raw resolver errors there. The wrapper preserves the exact native media payload, emits annotations only after a successful resolution, and applies the existing per-attachment Unicode-safe context cap to image readout, file payload plus archive listing, and annotations. | Use `nonvision_fallback_reads_images_and_uses_the_pdf_extractor`, `nonvision_archive_fallback_lists_entries_without_extracting_them`, `image_and_archive_fallback_context_is_unicode_safe_and_bounded`, `file_fallback_limit_applies_per_attachment_not_across_the_turn`, and `resolver_annotations_survive_next_to_native_media_only_after_success`. Retain host tests for readout-model selection, cache reuse, document feature/config gates, and safe path/recovery annotation rendering. | | Limits and PDF extraction hook | Map product configuration into `MediaRequestConfig { images, files, max_historical_media_items }`, pass the current host `TextExtractor` implementation to `MediaRequestModel::new`, and remove the duplicate request-level extraction/fallback work from `crates/openhuman-core/src/agent/multimodal.rs`. `FileLimits::effective()` preserves `max_files == 0`; the request wrapper rejects latest-turn media before invoking the resolver and passes real resolved PDF bytes to the extractor for non-vision models. | Keep product config, MIME allowlists, storage/service selection, and the existing document parser implementation in OpenHuman. `TextExtractor` remains a host port, not a second parser framework. `max_files == 0` disables reads for images and files on the latest turn, while older media in prior turns receives safe omission placeholders without reads. | TinyAgents `multimodal::config` hard-zero regression plus `nonvision_fallback_reads_images_and_uses_the_pdf_extractor` verify the sentinel and meaningful fake PDF bytes/result. Keep host config-mapping tests and a parser-enabled integration test that verifies its configured extraction behavior without a real backend. |