feat: adding enrichments after the VLM pipeline

docling/.agents/skills/docling/references/cli.md

@@ -50,9 +50,19 @@ docling report.pdf --pipeline vlm --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model granite_docling --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model smoldocling --output /tmp/
 docling report.pdf --pipeline vlm --vlm-model nemotron_parse_v2 --output /tmp/
+docling chart.png --from image --to dclx --pipeline vlm --enrich-chart-extraction
 docling report.pdf --pipeline native --from pdf --output /tmp/
 ```
 
+VLM picture enrichment runs after conversion. `--enrich-chart-extraction`
+classifies pictures first and adds chart data only where the VLM has not
+already provided it.
+Use `--chart-extraction-preset granite_vision` for the older CSV-only
+`ibm-granite/granite-vision-3.3-2b-chart2csv-preview` checkpoint. The default
+`granite_vision_v4` preset uses Granite Vision 4.1 4B, selecting MLX on
+compatible Apple Silicon systems and Transformers elsewhere. To require MLX,
+use `--chart-extraction-preset granite_vision_v4_mlx` with the same repository.
+
 For PDFs, visible horizontal and vertical rules are used as reading-order
 signals by default. Disable this to compare against rule-free ordering:
 

docling/cli/main.py

@@ -114,6 +114,7 @@
     InputFormat,
     OutputFormat,
 )
+from docling.datamodel.chart_extraction_options import ChartExtractionVlmEngineOptions
 from docling.datamodel.document import ConversionResult, DoclingVersion
 from docling.datamodel.pipeline_options import (
     AsrPipelineOptions,
@@ -292,6 +293,7 @@ def _expand_from_formats(from_formats: list[str] | None) -> list[InputFormat]:
 
 # Get available VLM presets from the registry
 vlm_preset_ids = VlmConvertOptions.list_preset_ids()
+chart_extraction_preset_ids = ChartExtractionVlmEngineOptions.list_preset_ids()
 
 DOCLING_ASCII_ART = r"""
                              ████ ██████
@@ -1070,6 +1072,16 @@ def convert(  # noqa: C901
             ..., help="Enable chart data extraction from bar, pie, and line charts."
         ),
     ] = False,
+    chart_extraction_preset: Annotated[
+        str,
+        typer.Option(
+            "--chart-extraction-preset",
+            help=(
+                "Choose the chart extraction preset. Available presets: "
+                f"{', '.join(chart_extraction_preset_ids)}"
+            ),
+        ),
+    ] = "granite_vision_v4",
     artifacts_path: Annotated[
         Path | None,
         typer.Option(..., help="If provided, the location of the model artifacts."),
@@ -1483,6 +1495,17 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
             table_structure_factory.create_options(kind=table_structure_engine)
         )
 
+        try:
+            chart_extraction_options = ChartExtractionVlmEngineOptions.from_preset(
+                chart_extraction_preset
+            )
+        except KeyError as exc:
+            raise typer.BadParameter(
+                f"Unknown chart extraction preset {chart_extraction_preset!r}. "
+                f"Available presets: {', '.join(chart_extraction_preset_ids)}",
+                param_hint="--chart-extraction-preset",
+            ) from exc
+
         if pipeline in {ProcessingPipeline.STANDARD, ProcessingPipeline.LEGACY}:
             pipeline_cls = (
                 LegacyStandardPdfPipeline
@@ -1504,6 +1527,7 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
                 do_picture_description=enrich_picture_description,
                 do_picture_classification=enrich_picture_classes,
                 do_chart_extraction=enrich_chart_extraction,
+                chart_extraction_options=chart_extraction_options,
                 document_timeout=document_timeout,
             )
             if isinstance(
@@ -1541,6 +1565,7 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
                 do_picture_description=enrich_picture_description,
                 do_picture_classification=enrich_picture_classes,
                 do_chart_extraction=enrich_chart_extraction,
+                chart_extraction_options=chart_extraction_options,
             )
             if artifacts_path is not None:
                 simple_format_option.artifacts_path = artifacts_path
@@ -1637,6 +1662,7 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
                 do_picture_description=enrich_picture_description,
                 do_picture_classification=enrich_picture_classes,
                 do_chart_extraction=enrich_chart_extraction,
+                chart_extraction_options=chart_extraction_options,
                 document_timeout=document_timeout,
             )
             if parser_threads is not None:
@@ -1671,7 +1697,15 @@ def _resolve_pdf_backend() -> tuple[type[PdfDocumentBackend], PdfBackendOptions]
             pipeline_options = VlmPipelineOptions(
                 accelerator_options=accelerator_options,
                 enable_remote_services=enable_remote_services,
+                do_picture_classification=enrich_picture_classes,
+                do_picture_description=enrich_picture_description,
+                do_chart_extraction=enrich_chart_extraction,
+                chart_extraction_options=chart_extraction_options,
             )
+            if picture_description_max_new_tokens is not None:
+                pipeline_options.picture_description_options.generation_config[
+                    "max_new_tokens"
+                ] = picture_description_max_new_tokens
             if _should_generate_export_images(image_export_mode, to_formats):
                 pipeline_options.generate_page_images = True
                 pipeline_options.generate_picture_images = True

docling/datamodel/chart_extraction_options.py

@@ -33,6 +33,7 @@ class ChartExtractionOutputFormat(str, Enum):
     """
 
     GRANITE_VISION_CHARTS = "granite_vision_charts"
+    GRANITE_VISION_CHART2CSV = "granite_vision_chart2csv"
 
 
 class ChartExtractionVlmEngineOptions(StagePresetMixin, VlmEngineOptionsMixin):
@@ -47,14 +48,13 @@ class ChartExtractionVlmEngineOptions(StagePresetMixin, VlmEngineOptionsMixin):
     * ``chart2summary`` — generate a natural-language description (default: False)
     * ``chart2code``    — generate Python code that recreates the chart (default: False)
 
-    .. note::
-        The ``granite_vision`` V1 preset (ibm-granite/granite-vision-3.3-2b-chart2csv-preview)
-        was removed in this release. Use ``granite_vision_v4`` (the default) instead.
-        The last release supporting V1 was 2.x (see the changelog for migration guidance).
+    The ``granite_vision`` preset uses the older CSV-only Chart2CSV model and
+    its recommended plain-language prompt. ``granite_vision_v4`` remains the
+    default and additionally supports summaries and code.
 
     Examples::
 
-        # Default preset (granite_vision_v4, Transformers engine)
+        # Default preset (granite_vision_v4, automatic local engine selection)
         options = ChartExtractionVlmEngineOptions.from_preset("granite_vision_v4")
 
         # Override engine at preset time
@@ -127,6 +127,11 @@ def _at_least_one_output(self) -> Self:
             raise ValueError(
                 "At least one of chart2csv, chart2summary, or chart2code must be True."
             )
+        if self.output_format == ChartExtractionOutputFormat.GRANITE_VISION_CHART2CSV:
+            if not self.chart2csv or self.chart2summary or self.chart2code:
+                raise ValueError(
+                    "The granite_vision Chart2CSV preset supports CSV output only."
+                )
         return self
 
     def active_prompts(self) -> list[str]:
@@ -148,9 +153,15 @@ def active_prompts(self) -> list[str]:
 
 from docling.datamodel import stage_model_specs as _stage_model_specs  # noqa: E402
 
+ChartExtractionVlmEngineOptions.register_preset(
+    _stage_model_specs.CHART_EXTRACTION_GRANITE_VISION
+)
 ChartExtractionVlmEngineOptions.register_preset(
     _stage_model_specs.CHART_EXTRACTION_GRANITE_VISION_V4
 )
+ChartExtractionVlmEngineOptions.register_preset(
+    _stage_model_specs.CHART_EXTRACTION_GRANITE_VISION_V4_MLX
+)
 
 
 # ---------------------------------------------------------------------------
@@ -181,10 +192,7 @@ class ChartExtractionModelKind(metaclass=_ChartExtractionModelKindMeta):
         Use :meth:`ChartExtractionVlmEngineOptions.from_preset` with
         ``'granite_vision_v4'`` instead.
 
-    .. note::
-        ``GRANITE_VISION`` (V1) support has been removed. References to
-        ``ChartExtractionModelKind.GRANITE_VISION`` will resolve to
-        ``'granite-vision-v4'`` with a deprecation warning.
+    ``GRANITE_VISION`` selects the older CSV-only Chart2CSV model.
     """
 
     GRANITE_VISION = "granite-vision"
@@ -214,9 +222,9 @@ def __hash__(self) -> int:
 
     _members: ClassVar[Dict[str, "_ChartExtractionModelKindMeta"]] = {}
 
-    # Map old enum values to new preset IDs (V1 → V4 with deprecation)
+    # Map legacy enum values to the registered preset IDs.
     _PRESET_MAP: ClassVar[Dict[str, str]] = {
-        "granite-vision": "granite_vision_v4",
+        "granite-vision": "granite_vision",
         "granite-vision-v4": "granite_vision_v4",
     }
 
@@ -236,9 +244,8 @@ class ChartExtractionModelOptions(ChartExtractionVlmEngineOptions):
 
     For backwards compatibility, instantiating this class emits a
     ``DeprecationWarning`` and returns a fully functional
-    ``ChartExtractionVlmEngineOptions`` configured from the ``granite_vision_v4``
-    preset. Passing ``model=ChartExtractionModelKind.GRANITE_VISION`` (V1) is
-    accepted but silently upgraded to V4 with an additional warning.
+    ``ChartExtractionVlmEngineOptions`` configured from the requested preset.
+    The default remains ``granite_vision_v4``.
     """
 
     kind: ClassVar[Literal["chart_extraction"]] = "chart_extraction"  # type: ignore[assignment]
@@ -268,18 +275,12 @@ def __init__(self, **data: Any) -> None:
                     f"Unknown model {model_str!r}. "
                     f"Valid values: {list(ChartExtractionModelKind._PRESET_MAP)}"
                 )
-            if model_str == ChartExtractionModelKind.GRANITE_VISION:
-                warnings.warn(
-                    "ChartExtractionModelKind.GRANITE_VISION (V1) is no longer supported "
-                    "and has been upgraded to granite_vision_v4.",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
-
         # Bootstrap from the preset so model_spec and engine_options are populated,
         # then allow the caller's remaining kwargs (chart2csv, etc.) to override.
         preset_instance = ChartExtractionVlmEngineOptions.from_preset(
-            "granite_vision_v4"
+            ChartExtractionModelKind._PRESET_MAP.get(
+                str(model_val), "granite_vision_v4"
+            )
         )
         merged = {**preset_instance.model_dump(), **data}
         super().__init__(**merged)

docling/datamodel/pipeline_options.py

@@ -1245,7 +1245,7 @@ class CodeFormulaVlmOptions(StagePresetMixin, VlmEngineOptionsMixin, BaseModel):
 _default_chart_extraction_options = ChartExtractionVlmEngineOptions.from_preset(
     "granite_vision_v4"
 )
-"""Default chart extraction options using granite_vision_v4 preset with Transformers runtime."""
+"""Default chart extraction options using granite_vision_v4 with automatic runtime selection."""
 
 
 # Define an enum for the backend options
@@ -1474,7 +1474,7 @@ class ConvertPipelineOptions(PipelineOptions):
             description=(
                 "Configuration for the chart extraction stage. "
                 "Use ChartExtractionVlmEngineOptions.from_preset('granite_vision_v4') "
-                "(default) or from_preset('granite_vision') for the V1 model. "
+                "(default) or from_preset('granite_vision') for the CSV-only model. "
                 "Controls which output formats are generated (chart2csv, chart2summary, chart2code)."
             )
         ),

docling/datamodel/service/options.py

@@ -654,9 +654,14 @@ class ConvertDocumentsOptions(BaseModel):
             description=(
                 "Preset ID for chart extraction. "
                 'Use "default" for the admin-controlled default, or a specific preset '
-                'such as "granite_vision_v4" or "granite_vision".'
+                'such as "granite_vision_v4", "granite_vision_v4_mlx", or "granite_vision".'
             ),
-            examples=["default", "granite_vision_v4", "granite_vision"],
+            examples=[
+                "default",
+                "granite_vision_v4",
+                "granite_vision_v4_mlx",
+                "granite_vision",
+            ],
         ),
     ] = None
 

docling/datamodel/stage_model_specs.py

@@ -1959,7 +1959,7 @@ def from_preset(
     ),
     scale=2.0,
     default_engine_type=VlmEngineType.TRANSFORMERS,
-    stage_options={"output_format": "granite_vision_charts"},
+    stage_options={"output_format": "granite_vision_chart2csv"},
 )
 
 CHART_EXTRACTION_GRANITE_VISION_V4 = StageModelPreset(
@@ -1978,6 +1978,7 @@ def from_preset(
         trust_remote_code=True,
         supported_engines={
             VlmEngineType.TRANSFORMERS,
+            VlmEngineType.MLX,
             VlmEngineType.API_LMSTUDIO,
             VlmEngineType.API_OLLAMA,
             VlmEngineType.API_OPENAI,
@@ -1989,6 +1990,7 @@ def from_preset(
                     "transformers_model_type": TransformersModelType.AUTOMODEL_IMAGETEXTTOTEXT,
                 },
             ),
+            VlmEngineType.MLX: EngineModelConfig(min_engine_version="0.7.0"),
         },
         api_overrides={
             VlmEngineType.API_LMSTUDIO: ApiModelConfig(
@@ -2003,6 +2005,16 @@ def from_preset(
         },
     ),
     scale=2.0,
-    default_engine_type=VlmEngineType.TRANSFORMERS,
+    default_engine_type=VlmEngineType.AUTO_INLINE,
     stage_options={"output_format": "granite_vision_charts"},
 )
+
+CHART_EXTRACTION_GRANITE_VISION_V4_MLX = StageModelPreset(
+    preset_id="granite_vision_v4_mlx",
+    name="Granite-Vision-4.1-4B (MLX)",
+    description="IBM Granite Vision 4.1-4B chart extraction on Apple Silicon",
+    model_spec=CHART_EXTRACTION_GRANITE_VISION_V4.model_spec,
+    scale=CHART_EXTRACTION_GRANITE_VISION_V4.scale,
+    default_engine_type=VlmEngineType.MLX,
+    stage_options=CHART_EXTRACTION_GRANITE_VISION_V4.stage_options,
+)

docling/models/picture_description_base_model.py

@@ -58,7 +58,11 @@ def __init__(
         self.images_scale = options.scale
 
     def is_processable(self, doc: DoclingDocument, element: NodeItem) -> bool:
-        return self.enabled and isinstance(element, PictureItem)
+        return (
+            self.enabled
+            and isinstance(element, PictureItem)
+            and (element.meta is None or element.meta.description is None)
+        )
 
     def _annotate_images(
         self, images: Iterable[Image.Image]