From df39537c99c69639ccb83be08ad1283e386c45ab Mon Sep 17 00:00:00 2001 From: Arik Horodniceanu Date: Fri, 19 Jun 2026 16:34:05 -0700 Subject: [PATCH] Qualcomm AI Engine Direct - [GenAI Pipeline] PR6: Compilation & inference strategy implementations --- .../configs/compilation_input_config.py | 15 +- .../configs/inference_output_config.py | 7 +- .../qualcomm/genai_pipeline/genai_pipeline.py | 30 +++ .../compilation/compiler_adapter.py | 30 ++- .../compilation/default_compiler_adapter.py | 81 ++++-- .../executorch_compilation_strategy.py | 194 +++++++++++++- .../inference/device_runner_adapter.py | 13 +- .../executorch_inference_strategy.py | 137 +++++++++- .../configs/test_compilation_input_config.py | 15 ++ .../test_executorch_compilation_strategy.py | 237 +++++++++++++++++- .../test_executorch_inference_strategy.py | 229 ++++++++++++++++- 11 files changed, 929 insertions(+), 59 deletions(-) diff --git a/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py b/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py index 8066f6c7aa1..43a9ece4584 100644 --- a/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py +++ b/backends/qualcomm/genai_pipeline/configs/compilation_input_config.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional, TYPE_CHECKING +from typing import Any, List, Optional, Tuple, TYPE_CHECKING if TYPE_CHECKING: from executorch.backends.qualcomm.serialization.qc_schema import ( @@ -23,10 +23,22 @@ class CompilationInputConfig: """Input configuration for the compilation stage. + ``model`` and ``example_inputs`` are ``Optional`` only because the + orchestrator builds this from the previous stages' output, which is empty + when those stages are skipped. **Both are required once the compilation + stage executes**, and strategies should validate their presence. + Attributes: soc_model: The target SoC (e.g., QcomChipset.SM8750). Required. backend_type: QNN backend type (HTP, GPU, LPAI, etc.). Required. model: The nn.Module to compile (quantized or original for FP16 mode). + Required when the stage runs. + example_inputs: Positional example inputs for ``torch.export``. Required + when the stage runs. Sourced from the **model** via + ``ModelLoaderAdapter.get_example_inputs``, never from calibration + data: this tuple defines the exported graph's positional signature, + supplies the zero-initialized KV caches a dataset sample does not + carry, and fixes the AR length because HTP has no dynamic shapes. artifact_dir: Directory to store compiled artifacts. compile_specs: QNN compiler specifications for backend delegation. """ @@ -34,5 +46,6 @@ class CompilationInputConfig: soc_model: "QcomChipset" backend_type: "QnnExecuTorchBackendType" model: Optional["nn.Module"] = None + example_inputs: Optional[Tuple[Any, ...]] = None artifact_dir: Path = field(default_factory=lambda: Path(".")) compile_specs: Optional[List["CompileSpec"]] = None diff --git a/backends/qualcomm/genai_pipeline/configs/inference_output_config.py b/backends/qualcomm/genai_pipeline/configs/inference_output_config.py index 22b68b0f636..645ca62d328 100644 --- a/backends/qualcomm/genai_pipeline/configs/inference_output_config.py +++ b/backends/qualcomm/genai_pipeline/configs/inference_output_config.py @@ -18,7 +18,12 @@ class InferenceOutputConfig: """Output produced by the inference stage. Attributes: - inference_results: Generated text output(s) from the model. + inference_results: Generated text output(s) from the model, or the paths + of the pulled result files when the adapter writes results to disk + rather than returning them. Stays ``List[str]`` rather than widening + to ``List[Any]``: ``InferenceResult.output_data`` is already decoded + text by contract, so it needs no conversion, and coercing it in the + strategy would risk stringifying raw token ids. performance_metrics: Performance data (e.g., TTFT, tokens/sec). eval_results: Evaluation metric results (e.g., SQNR, perplexity). etdump: Optional ETDump for debugging. ExecuTorch engine only. diff --git a/backends/qualcomm/genai_pipeline/genai_pipeline.py b/backends/qualcomm/genai_pipeline/genai_pipeline.py index e419d1ac4ec..a11bbcc42d6 100644 --- a/backends/qualcomm/genai_pipeline/genai_pipeline.py +++ b/backends/qualcomm/genai_pipeline/genai_pipeline.py @@ -150,6 +150,34 @@ def _resolve_stage( stage_name: str, skip: set, ): + """Build one stage from the registry, or ``None`` if it is skipped. + + Args: + engine_proxy: Routes each stage to its engine. + stage_name: The stage to resolve. + skip: Stage names to skip entirely. + + Returns: + The constructed stage, or ``None`` when the stage is skipped. + + Raises: + ValueError: If no strategy is registered for the stage's engine. + + .. note:: + Strategies are constructed with **no arguments**, so each gets its + default adapter. That suffices for model preparation and + quantization, but not for compilation or inference: the default + compiler adapter has no body yet, and the inference strategy + requires an injected device runner. A pipeline built purely from the + registry therefore cannot reach a working adapter for those two + stages. + + Injecting adapters today means constructing ``GenAIPipeline`` and + the stages directly rather than going through ``from_proxy``. + Threading adapters through this path lands together with the adapter + bodies, so the hook is shaped by what its callers actually need + rather than guessed at now. + """ if stage_name in skip: return None @@ -241,6 +269,8 @@ def _run_compilation( soc_model=context.soc_model, backend_type=self._engine_proxy.backend_type, model=model, + # Export inputs come from the model, not from calibration_data. + example_inputs=model_prep_output.example_inputs, artifact_dir=Path(context.artifact_dir), ) output = self._compilation_stage.invoke(context, input_config) diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py b/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py index 12740bc47c6..5318365e23d 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/compiler_adapter.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, List, Optional, Protocol, runtime_checkable +from typing import Any, Dict, List, Optional, Protocol, runtime_checkable, Tuple @dataclass @@ -28,8 +28,10 @@ class CompilationResult: class CompilerAdapter(Protocol): """Protocol for compilation operations. - Wraps external compilation APIs (ExportSession, to_edge_transform_and_lower_to_qnn) - behind an injectable interface for testability. + Wraps ``to_edge_transform_and_lower_to_qnn`` behind an injectable interface + for testability. The parameter list of ``compile_model`` deliberately + mirrors that function so that no required lowering input has to travel as a + string-keyed option. .. note:: ``compile_model`` lowers a **single graph**, mirroring the underlying @@ -47,23 +49,43 @@ class CompilerAdapter(Protocol): def compile_model( self, model: Any, + example_inputs: Tuple[Any, ...], compile_specs: Any, artifact_dir: Path, file_name: str, soc_model: Any, backend_type: Any, + constant_methods: Optional[Dict[str, Any]] = None, + dep_table: Optional[Dict] = None, + passes_job: Optional[Any] = None, extra_options: Optional[Dict[str, Any]] = None, ) -> CompilationResult: """Compile the model to on-device .pte artifacts. + The parameter list mirrors ``to_edge_transform_and_lower_to_qnn``: every + argument that lowering genuinely needs is explicit, and + ``extra_options`` is reserved for optional tuning knobs. Passing a + required input as a string-keyed option is deliberately avoided -- it + hides the contract and fails at runtime rather than at the call site. + Args: model: The model to compile (nn.Module or quantized model). + example_inputs: Positional example inputs for ``torch.export``, + sourced from the model itself. compile_specs: QNN compiler specifications for backend delegation. artifact_dir: Directory to store compiled artifacts. file_name: Base name for the output .pte file. soc_model: Target SoC chipset. backend_type: QNN backend type (HTP, GPU, LPAI). - extra_options: Additional compilation options. + constant_methods: Methods returning constants in eager mode. For a + decoder this carries the quantization attributes written during + quantization, so it is only complete after that stage. + dep_table: Per-graph pass dependency table. + passes_job: Per-graph pass configuration. + extra_options: Optional tuning knobs (``skip_node_id_set``, + ``skip_node_op_set``, ``skip_mutable_buffer``, + ``convert_linear_to_conv2d``, ``generate_etrecord``, + ``executorch_backend_config``). Returns: CompilationResult with artifact paths and optional etrecord. May hold diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py b/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py index 222cc3bab9d..e93de929b77 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/default_compiler_adapter.py @@ -6,68 +6,95 @@ from __future__ import annotations +import logging from pathlib import Path -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Tuple from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compiler_adapter import ( CompilationResult, ) +logger = logging.getLogger(__name__) + class DefaultCompilerAdapter: - """Default adapter delegating to the ExportSession recipe-based pipeline. + """Default adapter delegating to ``to_edge_transform_and_lower_to_qnn``. .. note:: - The body is deliberately unimplemented **in this PR only**: this PR - establishes the adapter interfaces, and the compilation strategy that - drives this adapter lands in the following PR, so the implementation - ships alongside its caller rather than ahead of it. + The signature below is the final one -- it mirrors + ``to_edge_transform_and_lower_to_qnn`` argument-for-argument, so the + per-graph inputs (``compile_specs``, ``dep_table``, ``passes_job``, + ``constant_methods``) are explicit parameters rather than + ``extra_options`` keys. + + The **body** is not implemented in this PR. Lowering is implementation + work rather than interface work, and the version this package needs is + the multi-graph one: ``to_edge_transform_and_lower_to_qnn`` accepts + graph-name-keyed dicts for ``module`` / ``inputs`` / ``compiler_specs`` + / ``dep_table`` / ``passes_job``, and a hybrid decoder groups its graphs + into a single multi-method ``.pte`` for weight sharing. Writing a + single-graph body here and then replacing it would mean implementing the + lowering twice, so it lands with the strategy-level fan-out that calls + it. - The APIs it will delegate to -- ``ExportRecipe.get_recipe`` with - ``QNNRecipeType.FP16`` and ``ExportSession`` -- are already available - in-tree; nothing external is blocking it. Until the follow-up lands, - inject a custom ``CompilerAdapter`` implementation. + A recipe-based body (``ExportRecipe.get_recipe(QNNRecipeType.FP16)`` + + ``ExportSession``) was considered and rejected: ``QNNRecipeProvider`` + accepts only ``soc_model`` and the three ``skip_*`` keys and silently + ignores the rest, so ``dep_table``, ``passes_job``, ``constant_methods`` + and ``convert_linear_to_conv2d`` are unreachable through it, and it + hardcodes ``use_fp16=True``. + + Until the body lands, inject a custom ``CompilerAdapter``. """ def compile_model( self, model: Any, + example_inputs: Tuple[Any, ...], compile_specs: Any, artifact_dir: Path, file_name: str, soc_model: Any, backend_type: Any, + constant_methods: Optional[Dict[str, Any]] = None, + dep_table: Optional[Dict] = None, + passes_job: Optional[Any] = None, extra_options: Optional[Dict[str, Any]] = None, ) -> CompilationResult: - """Compile the model using ExportSession. - - Placeholder for the recipe-based compilation flow, which uses - ``ExportRecipe.get_recipe(QNNRecipeType.FP16, ...)`` combined with - ``ExportSession``. Implemented in the compilation-strategy PR. + """Compile the model via ``to_edge_transform_and_lower_to_qnn``. Args: model: The model to compile (nn.Module or quantized model). - compile_specs: QNN compiler specifications (currently unused when - using recipe-based flow; kept for future custom compile spec support). + example_inputs: Positional example inputs for ``torch.export``, + sourced from the model itself. + compile_specs: QNN compiler specifications for backend delegation. artifact_dir: Directory to store compiled .pte artifacts. file_name: Base name for the output .pte file. soc_model: Target SoC chipset enum value. backend_type: QNN backend type (HTP, GPU, LPAI). - extra_options: Additional compilation options. Supported keys: - - ``example_inputs``: Sample inputs for torch.export. - - ``generate_etrecord``: Whether to generate ETRecord. - - ``constant_methods``: Dict of constant methods. + constant_methods: Methods returning constants in eager mode. For a + decoder this carries the quantization attributes written into + ``meta`` during quantization, so it is only complete once that + stage has run. + dep_table: Per-graph pass dependency table. + passes_job: Per-graph pass configuration. + extra_options: Optional tuning knobs forwarded to lowering: + ``skip_node_id_set``, ``skip_node_op_set``, + ``skip_mutable_buffer``, ``convert_linear_to_conv2d``, + ``generate_etrecord``, and ``executorch_backend_config`` to + override the ``to_executorch`` configuration. Returns: CompilationResult with artifact paths and optional etrecord. Raises: - NotImplementedError: Always raised in this PR. The body lands with - the compilation strategy that drives it; inject a custom - CompilerAdapter until then. + NotImplementedError: Always raised in this PR; the body lands with + the multi-graph lowering that calls it. """ raise NotImplementedError( - "DefaultCompilerAdapter is not implemented in this PR: the body " - "lands together with the compilation strategy that drives it. " - "Inject a custom CompilerAdapter implementation until then." + "DefaultCompilerAdapter has no body yet: lowering is implemented " + "together with the strategy-level multi-graph fan-out that calls " + "it, so that to_edge_transform_and_lower_to_qnn is wired up once " + "in its graph-keyed form. Inject a custom CompilerAdapter until " + "then." ) diff --git a/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py b/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py index b2d43a8d008..439c4b753f6 100644 --- a/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py +++ b/backends/qualcomm/genai_pipeline/strategies/compilation/executorch_compilation_strategy.py @@ -4,30 +4,216 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + from executorch.backends.qualcomm.genai_pipeline.configs.compilation_input_config import ( CompilationInputConfig, ) from executorch.backends.qualcomm.genai_pipeline.configs.compilation_output_config import ( CompilationOutputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.exceptions import StageError from executorch.backends.qualcomm.genai_pipeline.pipeline_context import PipelineContext from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compilation_strategy import ( CompilationStrategy, ) +from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compiler_adapter import ( + CompilerAdapter, +) + +logger = logging.getLogger(__name__) + +_STAGE_NAME = "compilation" + +# Optional lowering knobs forwarded from ``context.extra_options``. Required +# lowering inputs are explicit parameters on ``CompilerAdapter.compile_model`` +# and deliberately not accepted here. +_COMPILE_EXTRA_KEYS = ( + "generate_etrecord", + "skip_node_id_set", + "skip_node_op_set", + "skip_mutable_buffer", + "convert_linear_to_conv2d", + "executorch_backend_config", +) class ExecuTorchCompilationStrategy(CompilationStrategy): """ExecuTorch-based compilation using QNN compiler backend. - Reads backend_type from the input config to select the appropriate compiler backend (HTP, GPU, or LPAI). + Delegates to a ``CompilerAdapter`` for all external API calls, + enabling dependency injection for testability. + + The compilation flow: + 1. Validate the input configuration + 2. Delegate to the compiler adapter (wrapping + ``to_edge_transform_and_lower_to_qnn``) + 3. Return artifact paths and optional ETRecord + + .. note:: + **Single-graph only; multi-graph is a follow-up.** This strategy makes + exactly **one** ``compile_model`` call, mirroring the single-graph + contract of the adapter it delegates to. Models exported as several + graphs from the same weights (the hybrid AR-N prefill / AR-1 decode + pair, plus an optional token-embedding graph) need a loop over the + *deployed* graphs, which is why it is deliberately not attempted here: + + * only deployed graphs reach compilation: the full-auto-regressive + calibration graph a hybrid decoder also builds exists purely to + source quantization encodings, is reconciled into the deployed graphs + by the quantization stage and released there, so the loop is over + prefill and decode -- not over every graph the model preparation + stage produced; + * each deployed graph carries its own module, example inputs and + compile specs, so the lowering call takes graph-name-keyed dicts + rather than single values. ExecuTorch's + ``to_edge_transform_and_lower_to_qnn`` already accepts those dicts, + so this needs *using*, not building; + * grouping the graphs into one multi-method ``.pte`` (weight sharing) is + a property of that single lowering call, so it cannot be recovered by + calling this single-graph path repeatedly. + + So the eventual interface is a per-graph map of module to example inputs + and compile specs, which is additive to ``CompilationInputConfig``, so + deferring costs nothing structurally. Per the layering used throughout + this package, the fan-out belongs in this *strategy* -- adapters stay + thin 1:1 wrappers over one graph. ``CompilationOutputConfig.artifact_paths`` + likewise stays a ``List`` here and becomes graph-name-keyed with the same + change. + + Args: + compiler_adapter: Injectable adapter for compilation operations. + Defaults to ``DefaultCompilerAdapter`` if not provided. """ + def __init__( + self, + compiler_adapter: Optional[CompilerAdapter] = None, + ) -> None: + if compiler_adapter is None: + from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.default_compiler_adapter import ( + DefaultCompilerAdapter, + ) + + compiler_adapter = DefaultCompilerAdapter() + self._adapter = compiler_adapter + + @property + def adapter(self) -> CompilerAdapter: + """The compiler adapter used by this strategy.""" + return self._adapter + def invoke( self, context: PipelineContext, input_config: CompilationInputConfig, ) -> CompilationOutputConfig: - raise NotImplementedError( - "ExecuTorchCompilationStrategy.invoke() is a stub. " - "Implementation will be added in a subsequent PR." + """Compile the model to on-device .pte artifacts. + + Args: + context: The pipeline context with global settings. + input_config: The compilation input configuration. + + Returns: + CompilationOutputConfig with artifact paths and optional ETRecord. + + Raises: + StageError: If the model or example inputs are missing, or if + compilation fails. + """ + self._validate_input(input_config) + + logger.info( + "Starting compilation for model '%s' on SoC=%s, backend=%s", + context.model_name, + input_config.soc_model, + input_config.backend_type, ) + + try: + artifact_dir = Path(input_config.artifact_dir) + file_name = context.model_name + + # Pass compilation-relevant options from context.extra_options. + # Only forward keys that the compiler adapter may need, avoiding + # leaking unrelated pipeline options. + compile_extra = { + k: v + for k, v in context.extra_options.items() + if k in _COMPILE_EXTRA_KEYS + } + + # ``example_inputs`` is passed explicitly rather than through + # ``extra_options``: it comes from the model (see + # ``CompilationInputConfig.example_inputs``) and ``torch.export`` + # cannot run without it, so it is part of the adapter's signature. + result = self._adapter.compile_model( + model=input_config.model, + example_inputs=input_config.example_inputs, + compile_specs=input_config.compile_specs, + artifact_dir=artifact_dir, + file_name=file_name, + soc_model=input_config.soc_model, + backend_type=input_config.backend_type, + extra_options=compile_extra if compile_extra else None, + ) + + logger.info( + "Compilation completed: %d artifact(s) produced", + len(result.artifact_paths), + ) + + return CompilationOutputConfig( + artifact_paths=result.artifact_paths, + etrecord=result.etrecord, + ) + + except StageError: + raise + except Exception as e: + raise StageError( + stage_name=_STAGE_NAME, + message="Compilation failed", + original_exception=e, + ) from e + + def _validate_input(self, input_config: CompilationInputConfig) -> None: + """Validate required fields in the input configuration. + + Args: + input_config: The compilation input configuration. + + Raises: + StageError: If required fields are missing. + """ + if input_config.model is None: + raise StageError( + stage_name=_STAGE_NAME, + message="model is required for compilation", + ) + # ``is None`` rather than a truthiness test: an empty tuple is a valid + # export signature for a model taking no positional inputs, and only the + # field's absence means the previous stage produced nothing. Same + # reasoning as ``calibration_data`` in the quantization strategy. + if input_config.example_inputs is None: + raise StageError( + stage_name=_STAGE_NAME, + message=( + "example_inputs is required for compilation; it is produced " + "from the model by ModelLoaderAdapter.get_example_inputs" + ), + ) + if input_config.soc_model is None: + raise StageError( + stage_name=_STAGE_NAME, + message="soc_model is required for compilation", + ) + if input_config.backend_type is None: + raise StageError( + stage_name=_STAGE_NAME, + message="backend_type is required for compilation", + ) diff --git a/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py b/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py index 4877167000b..0c428bdc326 100644 --- a/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py +++ b/backends/qualcomm/genai_pipeline/strategies/inference/device_runner_adapter.py @@ -16,7 +16,12 @@ class InferenceResult: """Result of an on-device inference run. Attributes: - output_data: Raw output data from the model execution. + output_data: Decoded output from the model execution -- generated text, + not raw token ids. The adapter owns decoding, for the same reason it + owns encoding, so callers may forward this to + ``InferenceOutputConfig.inference_results`` unchanged. ``None`` when + the adapter writes results to files instead, to be collected by + ``pull_results``. performance_metrics: Performance data (e.g., TTFT, tokens/sec). etdump: Optional ETDump for debugging. """ @@ -44,7 +49,11 @@ def push_artifacts( Args: artifact_paths: Paths to compiled .pte artifacts. - input_data: Optional input data to push to device. + input_data: Optional pre-encoded input data to push to device. For + adapters that prepare inputs themselves -- turning a prompt into + model inputs needs the tokenizer's chat template, BOS handling, + AR-length padding and KV-cache seeding -- this stays ``None`` + and the adapter sources its own inputs. extra_files: Optional additional files to push. """ ... diff --git a/backends/qualcomm/genai_pipeline/strategies/inference/executorch_inference_strategy.py b/backends/qualcomm/genai_pipeline/strategies/inference/executorch_inference_strategy.py index 7542594ba83..70f281d0d31 100644 --- a/backends/qualcomm/genai_pipeline/strategies/inference/executorch_inference_strategy.py +++ b/backends/qualcomm/genai_pipeline/strategies/inference/executorch_inference_strategy.py @@ -4,27 +4,156 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + from executorch.backends.qualcomm.genai_pipeline.configs.inference_input_config import ( InferenceInputConfig, ) from executorch.backends.qualcomm.genai_pipeline.configs.inference_output_config import ( InferenceOutputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.exceptions import StageError from executorch.backends.qualcomm.genai_pipeline.pipeline_context import PipelineContext +from executorch.backends.qualcomm.genai_pipeline.strategies.inference.device_runner_adapter import ( + DeviceRunnerAdapter, +) from executorch.backends.qualcomm.genai_pipeline.strategies.inference.inference_strategy import ( InferenceStrategy, ) +logger = logging.getLogger(__name__) + +_STAGE_NAME = "inference" + class ExecuTorchInferenceStrategy(InferenceStrategy): - """ExecuTorch-based inference using QNN runtime on device.""" + """ExecuTorch-based inference using QNN runtime on device. + + Delegates to a ``DeviceRunnerAdapter`` for all external API calls, + enabling dependency injection for testability. + + The inference flow: + 1. Validate the input configuration + 2. Push artifacts to device + 3. Execute the model on device + 4. Pull and return results + + Input preparation belongs to the adapter, not here. ``DeviceRunnerAdapter`` + accepts ``input_data`` / ``extra_files`` for adapters that prefer the caller + to supply pre-encoded inputs, but this strategy passes only the artifacts: + turning ``prompt`` into model inputs needs the tokenizer's chat template, + BOS handling, AR-length padding and KV-cache seeding, all of which the + adapter knows and the strategy does not. Decoding is the adapter's for the + same reason, so ``InferenceResult.output_data`` arrives as text and is + forwarded without conversion. + + Args: + device_runner_adapter: Injectable adapter for device inference operations. + Must be provided (no default, since device configuration is required). + """ + + def __init__( + self, + device_runner_adapter: Optional[DeviceRunnerAdapter] = None, + ) -> None: + self._adapter = device_runner_adapter + + @property + def adapter(self) -> Optional[DeviceRunnerAdapter]: + """The device runner adapter used by this strategy.""" + return self._adapter def invoke( self, context: PipelineContext, input_config: InferenceInputConfig, ) -> InferenceOutputConfig: - raise NotImplementedError( - "ExecuTorchInferenceStrategy.invoke() is a stub. " - "Implementation will be added in a subsequent PR." + """Run inference using compiled model artifacts on device. + + Args: + context: The pipeline context with global settings. + input_config: The inference input configuration. + + Returns: + InferenceOutputConfig with inference results and metrics. + + Raises: + StageError: If artifacts are missing, adapter is not configured, + or inference fails. + """ + self._validate_input(input_config) + + logger.info( + "Starting inference for model '%s' on SoC=%s", + context.model_name, + input_config.soc_model, ) + + try: + # Step 1: Push artifacts to device. Only the artifacts: the adapter + # owns prompt -> tokens -> device inputs (see class docstring), so + # input_data/extra_files are deliberately not passed here. + logger.debug("Pushing artifacts to device") + self._adapter.push_artifacts( + artifact_paths=input_config.artifact_paths, + ) + + # Step 2: Execute on device + logger.debug("Executing model on device") + result = self._adapter.execute( + inference_options=input_config.inference_options, + ) + + # Step 3: Pull results from device + output_dir = Path(context.artifact_dir) / "inference_output" + logger.debug("Pulling results to %s", output_dir) + result_files = self._adapter.pull_results(output_dir=output_dir) + + logger.info("Inference completed successfully") + + # ``output_data`` is already decoded text by contract (the adapter + # owns decoding), so it is forwarded as-is; only the pulled-file + # fallback needs converting, because those genuinely are paths. + inference_results = result.output_data + if inference_results is None and result_files: + inference_results = [str(p) for p in result_files] + + return InferenceOutputConfig( + inference_results=inference_results, + performance_metrics=result.performance_metrics, + etdump=result.etdump, + ) + + except StageError: + raise + except Exception as e: + raise StageError( + stage_name=_STAGE_NAME, + message="Inference failed", + original_exception=e, + ) from e + + def _validate_input(self, input_config: InferenceInputConfig) -> None: + """Validate required fields in the input configuration. + + Args: + input_config: The inference input configuration. + + Raises: + StageError: If required fields are missing or adapter is not set. + """ + if self._adapter is None: + raise StageError( + stage_name=_STAGE_NAME, + message="device_runner_adapter is required for inference. " + "Provide a DeviceRunnerAdapter via the constructor.", + ) + if not input_config.artifact_paths: + raise StageError( + stage_name=_STAGE_NAME, + message="artifact_paths is required for inference", + ) diff --git a/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py b/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py index b427115e80e..e8407399183 100644 --- a/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py +++ b/backends/qualcomm/genai_pipeline/tests/configs/test_compilation_input_config.py @@ -23,6 +23,21 @@ def test_required_fields(self): with self.assertRaises(TypeError): CompilationInputConfig() + def test_optional_fields_default_to_none(self): + config = CompilationInputConfig(soc_model=MagicMock(), backend_type=MagicMock()) + self.assertIsNone(config.model) + self.assertIsNone(config.example_inputs) + self.assertIsNone(config.compile_specs) + + def test_example_inputs_carries_export_signature(self): + example_inputs = (MagicMock(name="tokens"), MagicMock(name="attn_mask")) + config = CompilationInputConfig( + soc_model=MagicMock(), + backend_type=MagicMock(), + example_inputs=example_inputs, + ) + self.assertIs(config.example_inputs, example_inputs) + if __name__ == "__main__": unittest.main() diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py index 0e9e400fc8f..20c7a1e9af4 100644 --- a/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py +++ b/backends/qualcomm/genai_pipeline/tests/strategies/compilation/test_executorch_compilation_strategy.py @@ -5,14 +5,22 @@ # LICENSE file in the root directory of this source tree. import unittest -from unittest.mock import MagicMock +from pathlib import Path +from unittest.mock import MagicMock, patch from executorch.backends.qualcomm.genai_pipeline.configs.compilation_input_config import ( CompilationInputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.configs.compilation_output_config import ( + CompilationOutputConfig, +) +from executorch.backends.qualcomm.genai_pipeline.exceptions import StageError from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compilation_strategy import ( CompilationStrategy, ) +from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.compiler_adapter import ( + CompilationResult, +) from executorch.backends.qualcomm.genai_pipeline.strategies.compilation.executorch_compilation_strategy import ( ExecuTorchCompilationStrategy, ) @@ -21,19 +29,230 @@ ) +def _make_mock_adapter(): + """Create a mock compiler adapter returning a valid CompilationResult.""" + adapter = MagicMock() + adapter.compile_model.return_value = CompilationResult( + artifact_paths=[Path("/tmp/test_model.pte")], + etrecord=None, + ) + return adapter + + +def _make_valid_input_config(**overrides): + """Create a valid CompilationInputConfig with defaults.""" + defaults = { + "soc_model": MagicMock(name="SM8750"), + "backend_type": MagicMock(name="kHtpBackend"), + "model": MagicMock(name="test_model"), + "example_inputs": (MagicMock(name="example_input"),), + "artifact_dir": Path("/tmp/artifacts"), + } + defaults.update(overrides) + return CompilationInputConfig(**defaults) + + class TestExecuTorchCompilationStrategy(unittest.TestCase): def test_is_compilation_strategy(self): - strategy = ExecuTorchCompilationStrategy() + """Strategy inherits from CompilationStrategy ABC.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) self.assertIsInstance(strategy, CompilationStrategy) - def test_invoke_raises_not_implemented(self): - strategy = ExecuTorchCompilationStrategy() - with self.assertRaises(NotImplementedError): - strategy.invoke( - make_test_context(), - CompilationInputConfig(soc_model=MagicMock(), backend_type=MagicMock()), - ) + def test_default_adapter_created_when_none_provided(self): + """When no adapter is provided, DefaultCompilerAdapter is created.""" + with patch( + "executorch.backends.qualcomm.genai_pipeline.strategies.compilation." + "default_compiler_adapter.DefaultCompilerAdapter" + ) as mock_cls: + mock_cls.return_value = MagicMock() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=None) + mock_cls.assert_called_once() + self.assertIsNotNone(strategy.adapter) + + def test_custom_adapter_injected(self): + """Custom adapter is used when provided via constructor.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + self.assertIs(strategy.adapter, adapter) + + def test_invoke_happy_path(self): + """Full compilation pipeline runs successfully end-to-end.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + context = make_test_context() + input_config = _make_valid_input_config() + + result = strategy.invoke(context, input_config) + + self.assertIsInstance(result, CompilationOutputConfig) + self.assertEqual(result.artifact_paths, [Path("/tmp/test_model.pte")]) + + def test_invoke_passes_correct_args_to_adapter(self): + """compile_model receives model, specs, artifact_dir, etc.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + model = MagicMock(name="model") + soc = MagicMock(name="soc") + backend = MagicMock(name="backend") + specs = MagicMock(name="specs") + example_inputs = (MagicMock(name="example_input"),) + input_config = _make_valid_input_config( + model=model, + soc_model=soc, + backend_type=backend, + compile_specs=specs, + example_inputs=example_inputs, + artifact_dir=Path("/tmp/out"), + ) + context = make_test_context() + + strategy.invoke(context, input_config) + + # example_inputs is an explicit parameter, not an extra_options key; + # extra_options is None when context has no compilation-relevant keys. + adapter.compile_model.assert_called_once_with( + model=model, + example_inputs=example_inputs, + compile_specs=specs, + artifact_dir=Path("/tmp/out"), + file_name=context.model_name, + soc_model=soc, + backend_type=backend, + extra_options=None, + ) + + def test_invoke_filters_extra_options_for_compilation(self): + """Only compilation-relevant keys from context.extra_options are forwarded.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + skip_ops = {"aten.slice.Tensor"} + context = make_test_context( + extra_options={ + "generate_etrecord": True, + "skip_node_op_set": skip_ops, + # example_inputs travels on the input config, so a stray + # context option of that name must not reach the adapter. + "example_inputs": MagicMock(name="stale_inputs"), + "unrelated_option": "should_not_pass", + } + ) + + strategy.invoke(context, _make_valid_input_config()) + + _, kwargs = adapter.compile_model.call_args + self.assertEqual( + kwargs["extra_options"], + {"generate_etrecord": True, "skip_node_op_set": skip_ops}, + ) + + def test_invoke_missing_example_inputs_raises_stage_error(self): + """StageError raised when example_inputs is None.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + input_config = _make_valid_input_config(example_inputs=None) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("example_inputs", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "compilation") + + def test_invoke_missing_soc_model_raises_stage_error(self): + """StageError raised when soc_model is None.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + input_config = _make_valid_input_config(soc_model=None) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("soc_model", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "compilation") + + def test_invoke_missing_backend_type_raises_stage_error(self): + """StageError raised when backend_type is None.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + input_config = _make_valid_input_config(backend_type=None) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("backend_type", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "compilation") + + def test_invoke_uses_context_model_name_as_file_name(self): + """file_name passed to adapter comes from context.model_name.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + context = make_test_context(model_name="my_model") + + strategy.invoke(context, _make_valid_input_config()) + + _, kwargs = adapter.compile_model.call_args + self.assertEqual(kwargs["file_name"], "my_model") + + def test_invoke_missing_model_raises_stage_error(self): + """StageError raised when model is None.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + input_config = _make_valid_input_config(model=None) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("model", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "compilation") + + def test_invoke_adapter_exception_wrapped_in_stage_error(self): + """Exceptions from the adapter are wrapped in StageError.""" + adapter = _make_mock_adapter() + adapter.compile_model.side_effect = RuntimeError("compile failed") + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), _make_valid_input_config()) + self.assertEqual(cm.exception.stage_name, "compilation") + self.assertIsInstance(cm.exception.original_exception, RuntimeError) + self.assertIn("compile failed", str(cm.exception)) + + def test_invoke_stage_error_not_double_wrapped(self): + """StageError from adapter is re-raised directly.""" + adapter = _make_mock_adapter() + original_error = StageError(stage_name="compilation", message="inner error") + adapter.compile_model.side_effect = original_error + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), _make_valid_input_config()) + self.assertIs(cm.exception, original_error) + + def test_invoke_returns_etrecord_when_present(self): + """etrecord from CompilationResult is forwarded to output config.""" + adapter = _make_mock_adapter() + mock_etrecord = MagicMock(name="etrecord") + adapter.compile_model.return_value = CompilationResult( + artifact_paths=[Path("/tmp/test.pte")], + etrecord=mock_etrecord, + ) + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertIs(result.etrecord, mock_etrecord) + + def test_invoke_multiple_artifacts(self): + """Multiple artifact paths from adapter are forwarded correctly.""" + adapter = _make_mock_adapter() + adapter.compile_model.return_value = CompilationResult( + artifact_paths=[Path("/tmp/prefill.pte"), Path("/tmp/decode.pte")], + etrecord=None, + ) + strategy = ExecuTorchCompilationStrategy(compiler_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertEqual(len(result.artifact_paths), 2) + self.assertEqual(result.artifact_paths[0], Path("/tmp/prefill.pte")) + self.assertEqual(result.artifact_paths[1], Path("/tmp/decode.pte")) if __name__ == "__main__": diff --git a/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py b/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py index 7335ef56a4a..8c9392b0806 100644 --- a/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py +++ b/backends/qualcomm/genai_pipeline/tests/strategies/inference/test_executorch_inference_strategy.py @@ -5,11 +5,19 @@ # LICENSE file in the root directory of this source tree. import unittest +from pathlib import Path from unittest.mock import MagicMock from executorch.backends.qualcomm.genai_pipeline.configs.inference_input_config import ( InferenceInputConfig, ) +from executorch.backends.qualcomm.genai_pipeline.configs.inference_output_config import ( + InferenceOutputConfig, +) +from executorch.backends.qualcomm.genai_pipeline.exceptions import StageError +from executorch.backends.qualcomm.genai_pipeline.strategies.inference.device_runner_adapter import ( + InferenceResult, +) from executorch.backends.qualcomm.genai_pipeline.strategies.inference.executorch_inference_strategy import ( ExecuTorchInferenceStrategy, ) @@ -21,19 +29,226 @@ ) +def _make_mock_adapter(): + """Create a mock device runner adapter with sensible defaults.""" + adapter = MagicMock() + adapter.push_artifacts.return_value = None + adapter.execute.return_value = InferenceResult( + output_data=["Hello, world!"], + performance_metrics={"tokens_per_sec": 42.0}, + etdump=None, + ) + adapter.pull_results.return_value = [Path("/tmp/output/result.bin")] + return adapter + + +def _make_valid_input_config(**overrides): + """Create a valid InferenceInputConfig with defaults.""" + defaults = { + "soc_model": MagicMock(name="SM8750"), + "artifact_paths": [Path("/tmp/test.pte")], + } + defaults.update(overrides) + return InferenceInputConfig(**defaults) + + class TestExecuTorchInferenceStrategy(unittest.TestCase): def test_is_inference_strategy(self): - strategy = ExecuTorchInferenceStrategy() + """Strategy inherits from InferenceStrategy ABC.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) self.assertIsInstance(strategy, InferenceStrategy) - def test_invoke_raises_not_implemented(self): + def test_custom_adapter_injected(self): + """Custom adapter is used when provided via constructor.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + self.assertIs(strategy.adapter, adapter) + + def test_no_adapter_defaults_to_none(self): + """When no adapter is provided, adapter property is None.""" + strategy = ExecuTorchInferenceStrategy() + self.assertIsNone(strategy.adapter) + + def test_invoke_happy_path(self): + """Full inference pipeline runs successfully end-to-end.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + context = make_test_context() + input_config = _make_valid_input_config() + + result = strategy.invoke(context, input_config) + + self.assertIsInstance(result, InferenceOutputConfig) + self.assertEqual(result.inference_results, ["Hello, world!"]) + self.assertEqual(result.performance_metrics["tokens_per_sec"], 42.0) + + def test_invoke_calls_adapter_in_correct_order(self): + """Adapter methods are called in order: push → execute → pull.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + strategy.invoke(make_test_context(), _make_valid_input_config()) + + expected_calls = ["push_artifacts", "execute", "pull_results"] + actual_method_calls = [c[0] for c in adapter.method_calls] + self.assertEqual(actual_method_calls, expected_calls) + + def test_invoke_passes_artifact_paths_to_push(self): + """push_artifacts receives the artifact paths from input config.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + artifact_paths = [Path("/tmp/a.pte"), Path("/tmp/b.pte")] + input_config = _make_valid_input_config(artifact_paths=artifact_paths) + + strategy.invoke(make_test_context(), input_config) + + adapter.push_artifacts.assert_called_once_with( + artifact_paths=artifact_paths, + ) + + def test_invoke_passes_inference_options_to_execute(self): + """execute receives inference_options from input config.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + opts = {"method_index": 1, "iteration": 5} + input_config = _make_valid_input_config(inference_options=opts) + + strategy.invoke(make_test_context(), input_config) + + adapter.execute.assert_called_once_with(inference_options=opts) + + def test_invoke_pull_results_uses_context_artifact_dir(self): + """pull_results output_dir is based on context.artifact_dir.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + context = make_test_context(artifact_dir="/custom/dir") + + strategy.invoke(context, _make_valid_input_config()) + + adapter.pull_results.assert_called_once_with( + output_dir=Path("/custom/dir") / "inference_output", + ) + + def test_invoke_populates_performance_metrics(self): + """Performance metrics from adapter result are in output config.""" + adapter = _make_mock_adapter() + adapter.execute.return_value = InferenceResult( + output_data=None, + performance_metrics={"ttft_ms": 100, "tokens_per_sec": 50}, + ) + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertEqual(result.performance_metrics["ttft_ms"], 100) + self.assertEqual(result.performance_metrics["tokens_per_sec"], 50) + + def test_invoke_populates_etdump(self): + """etdump from adapter result is forwarded to output config.""" + adapter = _make_mock_adapter() + mock_etdump = MagicMock(name="etdump") + adapter.execute.return_value = InferenceResult( + output_data=None, + performance_metrics={}, + etdump=mock_etdump, + ) + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertIs(result.etdump, mock_etdump) + + def test_invoke_no_adapter_raises_stage_error(self): + """StageError raised when no adapter is configured.""" strategy = ExecuTorchInferenceStrategy() - with self.assertRaises(NotImplementedError): - strategy.invoke( - make_test_context(), - InferenceInputConfig(soc_model=MagicMock()), - ) + input_config = _make_valid_input_config() + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("device_runner_adapter", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "inference") + + def test_invoke_missing_artifacts_raises_stage_error(self): + """StageError raised when artifact_paths is empty.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + input_config = _make_valid_input_config(artifact_paths=[]) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("artifact_paths", str(cm.exception)) + self.assertEqual(cm.exception.stage_name, "inference") + + def test_invoke_none_artifacts_raises_stage_error(self): + """StageError raised when artifact_paths is None.""" + adapter = _make_mock_adapter() + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + input_config = _make_valid_input_config(artifact_paths=None) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), input_config) + self.assertIn("artifact_paths", str(cm.exception)) + + def test_invoke_adapter_exception_wrapped_in_stage_error(self): + """Exceptions from the adapter are wrapped in StageError.""" + adapter = _make_mock_adapter() + adapter.push_artifacts.side_effect = RuntimeError("push failed") + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), _make_valid_input_config()) + self.assertEqual(cm.exception.stage_name, "inference") + self.assertIsInstance(cm.exception.original_exception, RuntimeError) + self.assertIn("push failed", str(cm.exception)) + + def test_invoke_stage_error_not_double_wrapped(self): + """StageError from adapter is re-raised directly.""" + adapter = _make_mock_adapter() + original_error = StageError(stage_name="inference", message="inner error") + adapter.execute.side_effect = original_error + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + with self.assertRaises(StageError) as cm: + strategy.invoke(make_test_context(), _make_valid_input_config()) + self.assertIs(cm.exception, original_error) + + def test_invoke_uses_pulled_files_when_output_data_is_none(self): + """When execute returns output_data=None, pulled file paths are used.""" + adapter = _make_mock_adapter() + adapter.execute.return_value = InferenceResult( + output_data=None, + performance_metrics={"tokens_per_sec": 30.0}, + etdump=None, + ) + adapter.pull_results.return_value = [ + Path("/tmp/out/result_0.bin"), + Path("/tmp/out/result_1.bin"), + ] + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertEqual( + result.inference_results, + ["/tmp/out/result_0.bin", "/tmp/out/result_1.bin"], + ) + + def test_invoke_prefers_output_data_over_pulled_files(self): + """When execute returns non-None output_data, pulled files are not used.""" + adapter = _make_mock_adapter() + adapter.execute.return_value = InferenceResult( + output_data=["generated text"], + performance_metrics={}, + etdump=None, + ) + adapter.pull_results.return_value = [Path("/tmp/out/result.bin")] + strategy = ExecuTorchInferenceStrategy(device_runner_adapter=adapter) + + result = strategy.invoke(make_test_context(), _make_valid_input_config()) + + self.assertEqual(result.inference_results, ["generated text"]) if __name__ == "__main__":