From 6e770fc2c32bdd0a5cdd66e449f0dee39732199a Mon Sep 17 00:00:00 2001 From: duoduohappy Date: Sat, 7 Nov 2026 23:31:59 +0800 Subject: [PATCH] fix: deserialize into a copy in `from_dict` so the caller's dict stays serialized `default_from_dict` copies `init_parameters` before replacing serialized sub-objects with their instances, so that a second deserialization does not receive already-parsed objects. Several components did their own deserialization into `data["init_parameters"]` before calling it, which defeats that copy: the caller's mapping ended up holding live callables, enums or components, so it could no longer be serialized with `json.dumps` and a second `from_dict` raised. Deserialize into a copy in those components, and hand `default_from_dict` a fresh envelope that carries it. The same defect was previously fixed one component at a time (MetadataRouter #13421, MockChatGenerator #13471, ListJoiner #13563, Tool #13606). This covers the remaining instances found by scanning every `from_dict` implementation in the repository for writes through `data.get("init_parameters", {})` and `data["data"]`. --- haystack/components/agents/agent.py | 8 +- .../builders/chat_prompt_builder.py | 8 +- .../components/converters/output_adapter.py | 8 +- .../embedders/mock_document_embedder.py | 8 +- .../embedders/mock_text_embedder.py | 8 +- .../components/generators/chat/fallback.py | 8 +- haystack/components/generators/chat/llm.py | 8 +- .../preprocessors/document_splitter.py | 8 +- .../retrievers/in_memory/bm25_retriever.py | 8 +- .../in_memory/embedding_retriever.py | 8 +- .../components/writers/document_writer.py | 8 +- haystack/hooks/from_function.py | 8 +- haystack/hooks/human_in_the_loop/hooks.py | 8 +- .../hooks/tool_result_offloading/hooks.py | 8 +- haystack/tools/component_tool.py | 5 +- haystack/tools/pipeline_tool.py | 5 +- ...s-not-mutate-caller-dict-7c1f4a9e2b6d.yaml | 11 +++ test/core/test_from_dict_does_not_mutate.py | 78 +++++++++++++++++++ 18 files changed, 181 insertions(+), 30 deletions(-) create mode 100644 releasenotes/notes/from-dict-does-not-mutate-caller-dict-7c1f4a9e2b6d.yaml create mode 100644 test/core/test_from_dict_does_not_mutate.py diff --git a/haystack/components/agents/agent.py b/haystack/components/agents/agent.py index b6ebb70c58c..9fc4b7b02fc 100644 --- a/haystack/components/agents/agent.py +++ b/haystack/components/agents/agent.py @@ -719,7 +719,11 @@ def from_dict(cls, data: dict[str, Any]) -> "Agent": :param data: Dictionary to deserialize from. :returns: Deserialized agent. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) deserialize_component_inplace(init_params, key="chat_generator") @@ -734,7 +738,7 @@ def from_dict(cls, data: dict[str, Any]) -> "Agent": if init_params.get("hooks") is not None: init_params["hooks"] = _deserialize_hooks_dictionary(init_params["hooks"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def _create_agent_span(self, tools: ToolsType) -> Any: """ diff --git a/haystack/components/builders/chat_prompt_builder.py b/haystack/components/builders/chat_prompt_builder.py index 0d104f4f54f..0e9ed0f66b6 100644 --- a/haystack/components/builders/chat_prompt_builder.py +++ b/haystack/components/builders/chat_prompt_builder.py @@ -343,7 +343,11 @@ def from_dict(cls, data: dict[str, Any]) -> "ChatPromptBuilder": :returns: The deserialized component. """ - init_parameters = data["init_parameters"] + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_parameters = dict(data["init_parameters"]) template = init_parameters.get("template") if template: if isinstance(template, list): @@ -351,4 +355,4 @@ def from_dict(cls, data: dict[str, Any]) -> "ChatPromptBuilder": elif isinstance(template, str): init_parameters["template"] = template - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_parameters}) diff --git a/haystack/components/converters/output_adapter.py b/haystack/components/converters/output_adapter.py index 510f23a33ae..59d7adbeacd 100644 --- a/haystack/components/converters/output_adapter.py +++ b/haystack/components/converters/output_adapter.py @@ -175,7 +175,11 @@ def from_dict(cls, data: dict[str, Any]) -> "OutputAdapter": :returns: The deserialized component. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) # `unsafe=True` swaps the Jinja sandbox for a NativeEnvironment that executes arbitrary code. # Honor it from serialized data only when the whole pipeline is being loaded in unsafe mode; @@ -201,4 +205,4 @@ def from_dict(cls, data: dict[str, Any]) -> "OutputAdapter": name: deserialize_callable(filter_func) if filter_func else None for name, filter_func in custom_filters.items() } - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) diff --git a/haystack/components/embedders/mock_document_embedder.py b/haystack/components/embedders/mock_document_embedder.py index de9c07fac4a..d15d74dddbd 100644 --- a/haystack/components/embedders/mock_document_embedder.py +++ b/haystack/components/embedders/mock_document_embedder.py @@ -122,11 +122,15 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: dict[str, Any]) -> MockDocumentEmbedder: """Deserialize the component from a dictionary.""" - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) embedding_fn = init_params.get("embedding_fn") if embedding_fn: init_params["embedding_fn"] = deserialize_callable(embedding_fn) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def warm_up(self) -> None: """No-op warm up, provided for interface compatibility with real Embedders.""" diff --git a/haystack/components/embedders/mock_text_embedder.py b/haystack/components/embedders/mock_text_embedder.py index 7592cead155..a79f9eabfb8 100644 --- a/haystack/components/embedders/mock_text_embedder.py +++ b/haystack/components/embedders/mock_text_embedder.py @@ -105,11 +105,15 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: dict[str, Any]) -> MockTextEmbedder: """Deserialize the component from a dictionary.""" - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) embedding_fn = init_params.get("embedding_fn") if embedding_fn: init_params["embedding_fn"] = deserialize_callable(embedding_fn) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def warm_up(self) -> None: """No-op warm up, provided for interface compatibility with real Embedders.""" diff --git a/haystack/components/generators/chat/fallback.py b/haystack/components/generators/chat/fallback.py index 8ceff01aa12..a0eb59bad9e 100644 --- a/haystack/components/generators/chat/fallback.py +++ b/haystack/components/generators/chat/fallback.py @@ -74,7 +74,11 @@ def to_dict(self) -> dict[str, Any]: def from_dict(cls, data: dict[str, Any]) -> FallbackChatGenerator: """Rebuild the component from a serialized representation, restoring nested chat generators.""" # Reconstruct nested chat generators from their serialized dicts - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) serialized = init_params.get("chat_generators") or [] deserialized: list[Any] = [] for g in serialized: @@ -84,7 +88,7 @@ def from_dict(cls, data: dict[str, Any]) -> FallbackChatGenerator: deserialized.append(holder["component"]) init_params["chat_generators"] = deserialized data["init_parameters"] = init_params - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def warm_up(self) -> None: """Warm up all underlying chat generators.""" diff --git a/haystack/components/generators/chat/llm.py b/haystack/components/generators/chat/llm.py index 29b706d8aae..3b6038cb07c 100644 --- a/haystack/components/generators/chat/llm.py +++ b/haystack/components/generators/chat/llm.py @@ -115,14 +115,18 @@ def from_dict(cls, data: dict[str, Any]) -> "LLM": :param data: Dictionary to deserialize from. :return: Deserialized LLM instance. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) deserialize_component_inplace(init_params, key="chat_generator") if init_params.get("streaming_callback") is not None: init_params["streaming_callback"] = deserialize_callable(init_params["streaming_callback"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def run( # type: ignore[override] # `messages` is in **kwargs to allow dynamic required/optional status self, diff --git a/haystack/components/preprocessors/document_splitter.py b/haystack/components/preprocessors/document_splitter.py index 553cad3e71b..5de8a83055c 100644 --- a/haystack/components/preprocessors/document_splitter.py +++ b/haystack/components/preprocessors/document_splitter.py @@ -524,13 +524,17 @@ def from_dict(cls, data: dict[str, Any]) -> "DocumentSplitter": """ Deserializes the component from a dictionary. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) splitting_function = init_params.get("splitting_function", None) if splitting_function: init_params["splitting_function"] = deserialize_callable(splitting_function) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) @staticmethod def _concatenate_sentences_based_on_word_amount( diff --git a/haystack/components/retrievers/in_memory/bm25_retriever.py b/haystack/components/retrievers/in_memory/bm25_retriever.py index 412fd084519..4a96dcbb8e1 100644 --- a/haystack/components/retrievers/in_memory/bm25_retriever.py +++ b/haystack/components/retrievers/in_memory/bm25_retriever.py @@ -112,10 +112,14 @@ def from_dict(cls, data: dict[str, Any]) -> "InMemoryBM25Retriever": :returns: The deserialized component. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if "filter_policy" in init_params: init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) @component.output_types(documents=list[Document]) def run( diff --git a/haystack/components/retrievers/in_memory/embedding_retriever.py b/haystack/components/retrievers/in_memory/embedding_retriever.py index ba7daa9c423..1fd14597667 100644 --- a/haystack/components/retrievers/in_memory/embedding_retriever.py +++ b/haystack/components/retrievers/in_memory/embedding_retriever.py @@ -128,10 +128,14 @@ def from_dict(cls, data: dict[str, Any]) -> "InMemoryEmbeddingRetriever": :returns: The deserialized component. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if "filter_policy" in init_params: init_params["filter_policy"] = FilterPolicy.from_str(init_params["filter_policy"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) @component.output_types(documents=list[Document]) def run( diff --git a/haystack/components/writers/document_writer.py b/haystack/components/writers/document_writer.py index 2bb06b73cc3..9ffd21503fd 100644 --- a/haystack/components/writers/document_writer.py +++ b/haystack/components/writers/document_writer.py @@ -72,10 +72,14 @@ def from_dict(cls, data: dict[str, Any]) -> "DocumentWriter": :raises DeserializationError: If the document store is not properly specified in the serialization data or its type cannot be imported. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if "policy" in init_params: init_params["policy"] = DuplicatePolicy[init_params["policy"]] - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) @component.output_types(documents_written=int) def run(self, documents: list[Document], policy: DuplicatePolicy | None = None) -> dict[str, int]: diff --git a/haystack/hooks/from_function.py b/haystack/hooks/from_function.py index ef1fac81ffe..4997f3cd8ee 100644 --- a/haystack/hooks/from_function.py +++ b/haystack/hooks/from_function.py @@ -112,12 +112,16 @@ def from_dict(cls, data: dict[str, Any]) -> "FunctionHook": :param data: The serialized hook dictionary produced by `to_dict`. :returns: The reconstructed `FunctionHook`. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if init_params.get("function") is not None: init_params["function"] = deserialize_callable(init_params["function"]) if init_params.get("async_function") is not None: init_params["async_function"] = deserialize_callable(init_params["async_function"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) def hook(function: Callable[[State], None | Awaitable[None]]) -> FunctionHook: diff --git a/haystack/hooks/human_in_the_loop/hooks.py b/haystack/hooks/human_in_the_loop/hooks.py index ede17b2a803..77ad1680b39 100644 --- a/haystack/hooks/human_in_the_loop/hooks.py +++ b/haystack/hooks/human_in_the_loop/hooks.py @@ -133,9 +133,13 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: dict[str, Any]) -> "ConfirmationHook": """Deserialize the hook, reconstructing its confirmation strategies.""" - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if init_params.get("confirmation_strategies") is not None: init_params["confirmation_strategies"] = _deserialize_confirmation_strategies( init_params["confirmation_strategies"] ) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) diff --git a/haystack/hooks/tool_result_offloading/hooks.py b/haystack/hooks/tool_result_offloading/hooks.py index f81739d2dc8..e22f5f058d7 100644 --- a/haystack/hooks/tool_result_offloading/hooks.py +++ b/haystack/hooks/tool_result_offloading/hooks.py @@ -294,9 +294,13 @@ def from_dict(cls, data: dict[str, Any]) -> "ToolResultOffloadHook": :param data: A dictionary representation produced by `to_dict`. :returns: The deserialized `ToolResultOffloadHook`. """ - init_params = data.get("init_parameters", {}) + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. default_from_dict + # already copies (see its comment); this write happens before that copy. + init_params = dict(data.get("init_parameters", {})) if init_params.get("store") is not None: deserialize_component_inplace(init_params, key="store") if init_params.get("offload_strategies") is not None: init_params["offload_strategies"] = _deserialize_offload_strategies(data=init_params["offload_strategies"]) - return default_from_dict(cls, data) + return default_from_dict(cls, {**data, "init_parameters": init_params}) diff --git a/haystack/tools/component_tool.py b/haystack/tools/component_tool.py index 486d6634cd1..207158c3734 100644 --- a/haystack/tools/component_tool.py +++ b/haystack/tools/component_tool.py @@ -345,7 +345,10 @@ def from_dict(cls, data: dict[str, Any]) -> "ComponentTool": """ Deserializes the ComponentTool from a dictionary. """ - inner_data = data["data"] + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. + inner_data = dict(data["data"]) deserialize_component_inplace(data=inner_data, key="component") if "outputs_to_state" in inner_data and inner_data["outputs_to_state"]: diff --git a/haystack/tools/pipeline_tool.py b/haystack/tools/pipeline_tool.py index 248d8f73c4d..ba08721ccdf 100644 --- a/haystack/tools/pipeline_tool.py +++ b/haystack/tools/pipeline_tool.py @@ -231,7 +231,10 @@ def from_dict(cls, data: dict[str, Any]) -> "PipelineTool": :returns: The deserialized PipelineTool instance. """ - inner_data = data["data"] + # Copy before deserializing: writing into the caller's mapping destroys the + # serialized handles, so the payload can no longer be JSON-serialized and a + # second from_dict() receives already-parsed objects. + inner_data = dict(data["data"]) # `is_pipeline_async` is a legacy key kept only for backward compatibility inner_data.pop("is_pipeline_async", None) pipeline = Pipeline.from_dict(inner_data["pipeline"]) diff --git a/releasenotes/notes/from-dict-does-not-mutate-caller-dict-7c1f4a9e2b6d.yaml b/releasenotes/notes/from-dict-does-not-mutate-caller-dict-7c1f4a9e2b6d.yaml new file mode 100644 index 00000000000..ca8847234cb --- /dev/null +++ b/releasenotes/notes/from-dict-does-not-mutate-caller-dict-7c1f4a9e2b6d.yaml @@ -0,0 +1,11 @@ +--- +fixes: + - | + Fixed ``from_dict`` in several components writing deserialized objects back into the mapping passed in by the caller. + ``DocumentWriter``, ``InMemoryBM25Retriever``, ``InMemoryEmbeddingRetriever``, ``DocumentSplitter``, ``ChatPromptBuilder``, + ``Agent``, ``LLM``, ``FallbackChatGenerator``, ``MockTextEmbedder``, ``MockDocumentEmbedder``, ``OutputAdapter``, + ``FunctionHook``, ``ConfirmationHook``, ``ToolResultOffloadHook``, ``ComponentTool`` and ``PipelineTool`` now deserialize + into a copy. ``default_from_dict`` already copies ``init_parameters`` for that reason, but these components replaced the + serialized handles in the caller's dictionary before calling it, so the copy protected nothing. The caller's mapping was + left holding live callables, enums or components: it could no longer be serialized with ``json.dumps`` and deserializing + it a second time failed. diff --git a/test/core/test_from_dict_does_not_mutate.py b/test/core/test_from_dict_does_not_mutate.py new file mode 100644 index 00000000000..1bd81c58df1 --- /dev/null +++ b/test/core/test_from_dict_does_not_mutate.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: 2022-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +"""`from_dict` must not write deserialized objects back into the caller's mapping. + +`default_from_dict` copies `init_parameters` before replacing serialized +sub-objects (see the comment there), precisely so that a second deserialization +does not receive already-parsed objects. Several components did their own +deserialization into `data["init_parameters"]` *before* calling it, which +defeats that copy: the caller's dict ends up holding live objects, so it can no +longer be JSON-serialized and a second `from_dict` fails. + +Same defect previously fixed one component at a time: MetadataRouter (#13421), +MockChatGenerator (#13471), ListJoiner (#13563), Tool (#13606). This test pins +the invariant for the components fixed alongside it. +""" + +import json +from copy import deepcopy + +import pytest + +from haystack.components.builders import ChatPromptBuilder +from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMemoryEmbeddingRetriever +from haystack.components.writers import DocumentWriter +from haystack.dataclasses import ChatMessage +from haystack.document_stores.in_memory import InMemoryDocumentStore +from haystack.document_stores.types import DuplicatePolicy, FilterPolicy + +CASES = [ + pytest.param( + DocumentWriter, + lambda: DocumentWriter(document_store=InMemoryDocumentStore(), policy=DuplicatePolicy.OVERWRITE), + id="document-writer-enum", + ), + pytest.param( + InMemoryBM25Retriever, + lambda: InMemoryBM25Retriever(document_store=InMemoryDocumentStore(), filter_policy=FilterPolicy.MERGE), + id="bm25-retriever-enum", + ), + pytest.param( + InMemoryEmbeddingRetriever, + lambda: InMemoryEmbeddingRetriever(document_store=InMemoryDocumentStore(), filter_policy=FilterPolicy.MERGE), + id="embedding-retriever-enum", + ), + pytest.param( + ChatPromptBuilder, + lambda: ChatPromptBuilder(template=[ChatMessage.from_user("Hello")]), + id="chat-prompt-builder-nested-message", + ), +] + + +@pytest.mark.parametrize("cls, build", CASES) +class TestFromDictDoesNotMutateCallerDict: + def test_caller_dict_is_unchanged(self, cls, build): + payload = build().to_dict() + snapshot = deepcopy(payload) + + cls.from_dict(payload) + + assert payload == snapshot + + def test_payload_stays_json_serializable(self, cls, build): + payload = build().to_dict() + + cls.from_dict(payload) + + json.dumps(payload) + + def test_second_deserialization_succeeds(self, cls, build): + payload = build().to_dict() + + first = cls.from_dict(payload) + second = cls.from_dict(payload) + + assert type(first) is type(second)