From 11211fcab1c27e4779323ff646714dbce475568e Mon Sep 17 00:00:00 2001 From: Soham Rajadhyaksha Date: Thu, 24 Sep 2026 11:25:49 -0700 Subject: [PATCH] Keep the tail when truncating long states for the encoder Embedder sends truncate_prompt_tokens without a side, so vLLM falls back to the tokenizer default. For the pooling runner that is 'right' since vLLM 0.13, which keeps the first N tokens and drops the question that state_text appends last. The training recipe (Recipe.state_ids) keeps the tail, so send truncation_side='left' explicitly. vLLM accepts the field from 0.18 and ignores unknown fields before that. --- src/clm/embedder.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/clm/embedder.py b/src/clm/embedder.py index b19cb70..50ebe0e 100644 --- a/src/clm/embedder.py +++ b/src/clm/embedder.py @@ -40,7 +40,10 @@ def __init__(self, url: str = "http://127.0.0.1:8090/v1/embeddings", model: str def _fetch(self, texts: list[str]) -> tuple[list[np.ndarray], int]: body: dict[str, Any] = {"model": self.model, "input": texts, "encoding_format": "base64"} if self.max_tokens: + # Keep the tail, as the training recipe does: the question sits at the end of a + # state text. vLLM's pooling runner cuts from the right by default since 0.13. body["truncate_prompt_tokens"] = self.max_tokens + body["truncation_side"] = "left" try: r = self.session.post(self.url, json=body, timeout=self.timeout) except requests.RequestException as e: