feat(file_processors): route docling VLM processing through stack inference API

docs/docs/providers/file_processors/inline_docling.mdx

@@ -11,6 +11,8 @@ description: |
   - **Layout preservation** — tables, lists, and nested structures are converted to Markdown
   - **Multi-format support** — PDF, DOCX, PPTX, HTML, and images
   - **Better RAG quality** — structured chunks with heading metadata produce more relevant retrieval results
+  - **VLM-based processing** — optionally route Vision Language Model inference through the stack's model-serving
+    infrastructure for richer document understanding (layout analysis, OCR via vision models)
 
   ## Usage
 
@@ -31,6 +33,24 @@ description: |
       config: {}
   ```
 
+  ### Enabling VLM Processing
+
+  To enable VLM-based document processing, set `vlm_model` to a vision model registered with the
+  stack's inference API. The VLM pipeline routes inference through the stack's model-serving
+  infrastructure — no separate GPU resources are needed for document processing.
+
+  ```yaml
+  file_processors:
+    - provider_id: docling
+      provider_type: inline::docling
+      config:
+        vlm_model: granite-docling-258M
+        vlm_preset: granite_docling
+  ```
+
+  When `vlm_model` is not set or no inference provider is available, the processor gracefully
+  degrades to the standard non-VLM pipeline.
+
   ## Installation
 
   ```bash
@@ -60,6 +80,8 @@ preserves semantic boundaries. It supports PDF, DOCX, PPTX, HTML, and images.
 - **Layout preservation** — tables, lists, and nested structures are converted to Markdown
 - **Multi-format support** — PDF, DOCX, PPTX, HTML, and images
 - **Better RAG quality** — structured chunks with heading metadata produce more relevant retrieval results
+- **VLM-based processing** — optionally route Vision Language Model inference through the stack's model-serving
+  infrastructure for richer document understanding (layout analysis, OCR via vision models)
 
 ## Usage
 
@@ -80,6 +102,24 @@ file_processors:
     config: {}
 ```
 
+### Enabling VLM Processing
+
+To enable VLM-based document processing, set `vlm_model` to a vision model registered with the
+stack's inference API. The VLM pipeline routes inference through the stack's model-serving
+infrastructure — no separate GPU resources are needed for document processing.
+
+```yaml
+file_processors:
+  - provider_id: docling
+    provider_type: inline::docling
+    config:
+      vlm_model: granite-docling-258M
+      vlm_preset: granite_docling
+```
+
+When `vlm_model` is not set or no inference provider is available, the processor gracefully
+degrades to the standard non-VLM pipeline.
+
 ## Installation
 
 ```bash
@@ -97,12 +137,16 @@ See [Docling's documentation](https://docling-project.github.io/docling/) for mo
 |-------|------|----------|---------|-------------|
 | `default_chunk_size_tokens` | `int` | No | 800 | Default chunk size in tokens when chunking_strategy type is 'auto' |
 | `default_chunk_overlap_tokens` | `int` | No | 400 | Default chunk overlap in tokens when chunking_strategy type is 'auto' |
-| `do_ocr` | `bool` | No | True | Enable OCR for scanned documents. Set to False for digital PDFs (with embedded text) to improve processing speed by ~3x for non-scanned PDFs. Note: Setting to False on scanned PDFs will result in minimal text extraction. |
+| `do_ocr` | `bool` | No | True | Enable OCR for scanned documents. Set to False for digital PDFs (with embedded text) to improve processing speed by ~3x for non-scanned PDFs. Note: Setting to False on scanned PDFs will result in minimal text extraction. Ignored when vlm_model is set (VLM pipeline handles text extraction). |
+| `vlm_model` | `str \| None` | No |  | Model identifier for VLM-based document processing. When set and an inference provider is available, enables the VLM pipeline for richer document understanding (layout analysis, OCR via vision models). The model must be registered with the stack's inference API. When None (default), uses the standard non-VLM pipeline. |
+| `vlm_preset` | `str` | No | granite_docling | Docling VLM preset controlling prompt template and response format. Must be a valid preset name registered with VlmConvertOptions. The default 'granite_docling' is the recommended preset for production use. |
 
 ## Sample Configuration
 
 ```yaml
 default_chunk_size_tokens: 800
 default_chunk_overlap_tokens: 400
 do_ocr: true
+vlm_model: null
+vlm_preset: granite_docling
 ```

src/ogx/providers/inline/file_processor/docling/__init__.py

@@ -18,8 +18,9 @@ async def get_provider_impl(config: DoclingFileProcessorConfig, deps: dict[Api,
     assert isinstance(config, DoclingFileProcessorConfig), f"Unexpected config type: {type(config)}"
 
     files_api = deps.get(Api.files)
+    inference_api = deps.get(Api.inference)
 
-    impl = DoclingFileProcessor(config, files_api)
+    impl = DoclingFileProcessor(config, files_api, inference_api)
     return impl
 
 

src/ogx/providers/inline/file_processor/docling/config.py

@@ -32,7 +32,27 @@ class DoclingFileProcessorConfig(BaseModel):
         description=(
             "Enable OCR for scanned documents. Set to False for digital PDFs "
             "(with embedded text) to improve processing speed by ~3x for non-scanned PDFs. "
-            "Note: Setting to False on scanned PDFs will result in minimal text extraction."
+            "Note: Setting to False on scanned PDFs will result in minimal text extraction. "
+            "Ignored when vlm_model is set (VLM pipeline handles text extraction)."
+        ),
+    )
+
+    vlm_model: str | None = Field(
+        default=None,
+        description=(
+            "Model identifier for VLM-based document processing. When set and an inference "
+            "provider is available, enables the VLM pipeline for richer document understanding "
+            "(layout analysis, OCR via vision models). The model must be registered with the "
+            "stack's inference API. When None (default), uses the standard non-VLM pipeline."
+        ),
+    )
+
+    vlm_preset: str = Field(
+        default="granite_docling",
+        description=(
+            "Docling VLM preset controlling prompt template and response format. "
+            "Must be a valid preset name registered with VlmConvertOptions. "
+            "The default 'granite_docling' is the recommended preset for production use."
         ),
     )
 
@@ -41,5 +61,7 @@ def sample_run_config(cls, **kwargs: Any) -> dict[str, Any]:
         return {
             "default_chunk_size_tokens": 800,
             "default_chunk_overlap_tokens": 400,
-            "do_ocr": True,  # default option for scanned pdfs
+            "do_ocr": True,
+            "vlm_model": None,
+            "vlm_preset": "granite_docling",
         }

src/ogx/providers/inline/file_processor/docling/docling.py

@@ -14,12 +14,15 @@
 
 from docling.chunking import HybridChunker
 from docling.datamodel.base_models import InputFormat
-from docling.datamodel.pipeline_options import PdfPipelineOptions
+from docling.datamodel.pipeline_options import PdfPipelineOptions, VlmConvertOptions, VlmPipelineOptions
+from docling.datamodel.vlm_engine_options import ApiVlmEngineOptions, VlmEngineType
 from docling.document_converter import DocumentConverter, PdfFormatOption
+from docling.pipeline.vlm_pipeline import VlmPipeline
 from docling_core.transforms.chunker.tokenizer.huggingface import HuggingFaceTokenizer
 from fastapi import UploadFile
 
 from ogx.log import get_logger
+from ogx.providers.inline.file_processor.docling.vlm_engine import OgxInferenceVlmEngine
 from ogx.providers.inline.file_processor.zip_utils import validate_zip_content
 from ogx.providers.utils.files.response import response_body_bytes
 from ogx.providers.utils.vector_io.vector_utils import generate_chunk_id
@@ -55,22 +58,81 @@ class DoclingFileProcessor:
     Supports multiple file formats via docling's DocumentConverter (PDF, DOCX, PPTX, HTML, images, etc.).
     """
 
-    def __init__(self, config: DoclingFileProcessorConfig, files_api=None) -> None:
+    def __init__(self, config: DoclingFileProcessorConfig, files_api=None, inference_api=None) -> None:
         self.config = config
         self.files_api = files_api
+        self.inference_api = inference_api
+        self._vlm_enabled = False
 
-        pipeline_options = PdfPipelineOptions(
-            do_ocr=config.do_ocr,
-        )
-
-        self.converter = DocumentConverter(
-            format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)}
-        )
+        self.converter = self._build_converter()
         self._converter_lock = threading.Lock()
 
     def supported_mime_types(self) -> set[str] | None:
         return DOCLING_MIME_TYPES
 
+    def _build_converter(self) -> DocumentConverter:
+        if self.config.vlm_model and self.inference_api:
+            return self._build_vlm_converter()
+
+        if self.config.vlm_model and not self.inference_api:
+            log.warning(
+                "vlm_model is configured but no inference provider is available, falling back to standard pipeline",
+                vlm_model=self.config.vlm_model,
+            )
+
+        pipeline_options = PdfPipelineOptions(do_ocr=self.config.do_ocr)
+        return DocumentConverter(format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)})
+
+    def _build_vlm_converter(self) -> DocumentConverter:
+        assert self.config.vlm_model is not None
+
+        available_presets = list(VlmConvertOptions._presets.keys())
+        if self.config.vlm_preset not in available_presets:
+            raise ValueError(f"Invalid vlm_preset '{self.config.vlm_preset}'. Available presets: {available_presets}")
+
+        vlm_options = VlmConvertOptions.from_preset(
+            self.config.vlm_preset,
+            engine_options=ApiVlmEngineOptions(engine_type=VlmEngineType.API_OPENAI),
+        )
+
+        vlm_pipeline_options = VlmPipelineOptions(
+            vlm_options=vlm_options,
+            enable_remote_services=True,
+        )
+
+        converter = DocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_cls=VlmPipeline,
+                    pipeline_options=vlm_pipeline_options,
+                )
+            }
+        )
+
+        converter.initialize_pipeline(InputFormat.PDF)
+
+        event_loop = asyncio.get_event_loop()
+        ogx_engine = OgxInferenceVlmEngine(
+            inference_api=self.inference_api,
+            model=self.config.vlm_model,
+            event_loop=event_loop,
+        )
+
+        for pipeline in converter.initialized_pipelines.values():
+            for stage in getattr(pipeline, "build_pipe", []):
+                if hasattr(stage, "engine"):
+                    stage.engine = ogx_engine
+                    break
+
+        self._vlm_enabled = True
+        log.info(
+            "VLM pipeline enabled",
+            vlm_model=self.config.vlm_model,
+            vlm_preset=self.config.vlm_preset,
+        )
+
+        return converter
+
     async def process_file(
         self,
         request: ProcessFileRequest,
@@ -134,13 +196,17 @@ def _process_content(
 
         processing_time_ms = int((time.time() - start_time) * 1000)
 
+        extraction_method = "docling-vlm" if self._vlm_enabled else "docling"
         response_metadata: dict[str, Any] = {
             "processor": "docling",
             "processing_time_ms": processing_time_ms,
             "page_count": page_count,
-            "extraction_method": "docling",
+            "extraction_method": extraction_method,
             "file_size_bytes": len(content),
         }
+        if self._vlm_enabled:
+            response_metadata["vlm_model"] = self.config.vlm_model
+            response_metadata["vlm_preset"] = self.config.vlm_preset
 
         # Create chunks
         chunks = self._create_chunks(doc, document_id, chunking_strategy, document_metadata)

src/ogx/providers/inline/file_processor/docling/vlm_engine.py

@@ -0,0 +1,131 @@
+# Copyright (c) The OGX Contributors.
+# All rights reserved.
+#
+# This source code is licensed under the terms described in the LICENSE file in
+# the root directory of this source tree.
+
+import asyncio
+import base64
+import time
+from io import BytesIO
+
+from docling.models.inference_engines.vlm.base import (
+    BaseVlmEngine,
+    VlmEngineInput,
+    VlmEngineOutput,
+)
+
+from ogx.log import get_logger
+from ogx_api.inference.models import (
+    OpenAIChatCompletionContentPartImageParam,
+    OpenAIChatCompletionContentPartTextParam,
+    OpenAIChatCompletionRequestWithExtraBody,
+    OpenAIImageURL,
+    OpenAIUserMessageParam,
+)
+
+log = get_logger(name=__name__, category="providers::file_processors")
+
+
+class OgxInferenceVlmEngine(BaseVlmEngine):
+    """VLM engine that routes inference through the stack's model-serving API.
+
+    Instead of making HTTP requests to an external endpoint, this engine calls
+    the stack's Inference API directly in-process via dependency injection.
+    """
+
+    def __init__(self, inference_api, model: str, event_loop: asyncio.AbstractEventLoop) -> None:
+        self.inference_api = inference_api
+        self.model = model
+        self.event_loop = event_loop
+        self._initialized = True
+
+    def initialize(self) -> None:
+        pass
+
+    def predict_batch(self, input_batch: list[VlmEngineInput]) -> list[VlmEngineOutput]:
+        if not input_batch:
+            return []
+
+        log.info(
+            "Processing VLM batch through stack inference API",
+            model=self.model,
+            batch_size=len(input_batch),
+        )
+
+        start_time = time.time()
+        outputs = [self._predict_single(inp) for inp in input_batch]
+        total_time = time.time() - start_time
+
+        log.info(
+            "VLM batch complete",
+            batch_size=len(input_batch),
+            total_time_s=round(total_time, 2),
+            per_image_s=round(total_time / len(input_batch), 2),
+        )
+
+        return outputs
+
+    def _predict_single(self, input_data: VlmEngineInput) -> VlmEngineOutput:
+        request_start = time.time()
+
+        try:
+            image = input_data.image.copy().convert("RGBA")
+            img_io = BytesIO()
+            image.save(img_io, "PNG")
+            image_base64 = base64.b64encode(img_io.getvalue()).decode("utf-8")
+        except Exception:
+            log.exception("Failed to encode image for VLM inference")
+            return VlmEngineOutput(text="", stop_reason="error")
+
+        message = OpenAIUserMessageParam(
+            role="user",
+            content=[
+                OpenAIChatCompletionContentPartImageParam(
+                    image_url=OpenAIImageURL(url=f"data:image/png;base64,{image_base64}"),
+                ),
+                OpenAIChatCompletionContentPartTextParam(text=input_data.prompt),
+            ],
+        )
+
+        try:
+            request = OpenAIChatCompletionRequestWithExtraBody(
+                model=self.model,
+                messages=[message],
+                temperature=input_data.temperature,
+                max_tokens=input_data.max_new_tokens or None,
+                stop=input_data.stop_strings or None,
+                stream=False,
+            )
+            future = asyncio.run_coroutine_threadsafe(
+                self.inference_api.openai_chat_completion(request),
+                self.event_loop,
+            )
+            response = future.result(timeout=120)
+
+            generated_text = response.choices[0].message.content or ""
+            finish_reason = response.choices[0].finish_reason or "stop"
+
+            stop_reason = "end_of_sequence"
+            if finish_reason == "length":
+                stop_reason = "length"
+            elif finish_reason == "content_filter":
+                log.warning("VLM response was filtered due to content safety policy")
+                stop_reason = "content_filtered"
+
+            generation_time = time.time() - request_start
+
+            return VlmEngineOutput(
+                text=generated_text.strip(),
+                stop_reason=stop_reason,
+                metadata={
+                    "generation_time": generation_time,
+                    "model": self.model,
+                },
+            )
+        except Exception:
+            log.exception("Failed to process VLM inference request")
+            return VlmEngineOutput(text="", stop_reason="error")
+
+    def cleanup(self) -> None:
+        pass