fix(pdf): keep PDF URI links in ScrapingResult.links (#2374)

crawl4ai/processors/pdf/__init__.py

@@ -3,8 +3,10 @@
 from dataclasses import asdict
 from crawl4ai.async_logger import AsyncLogger
 from crawl4ai.async_crawler_strategy import AsyncCrawlerStrategy
-from crawl4ai.models import AsyncCrawlResponse, ScrapingResult 
+from urllib.parse import urljoin
+from crawl4ai.models import AsyncCrawlResponse, Link, Links, ScrapingResult
 from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
+from crawl4ai.utils import get_base_domain, is_external_url
 from .processor import NaivePDFProcessorStrategy  # Assuming your current PDF code is in pdf_processor.py
 
 # Default resource bounds for remote PDF fetches. These cap what an untrusted
@@ -158,28 +160,20 @@ def scrap(self, url: str, html: str, **params) -> ScrapingResult:
         </html>
         """
 
-            # Accumulate media and links with page numbers
+            # Accumulate media with page numbers
             media = {"images": []}
-            links = {"urls": []}
 
             for page in result.pages:
                 # Add page number to each image
                 for img in page.images:
                     img["page"] = page.page_number
                     media["images"].append(img)
-
-                # Add page number to each link
-                for link in page.links:
-                    links["urls"].append({
-                        "url": link,
-                        "page": page.page_number
-                    })
 
             return ScrapingResult(
                 cleaned_html=cleaned_html,
                 success=True,
                 media=media,
-                links=links,
+                links=self._build_links(url, result.pages),
                 metadata=asdict(result.metadata)
             )
         finally:
@@ -193,6 +187,38 @@ def scrap(self, url: str, html: str, **params) -> ScrapingResult:
                     if self.logger:
                         self.logger.warning(f"Failed to cleanup temp file {pdf_path}: {e}")
 
+    @staticmethod
+    def _build_links(url: str, pages) -> Links:
+        """Turn the URI annotations of every page into ``Links``.
+
+        ``Links`` only has ``internal`` and ``external`` lists of ``Link``
+        items, so the links are sorted against the PDF's own domain the way
+        the HTML scraper does it. A local PDF has no domain, so all of its
+        absolute links are external. A target that appears on several pages
+        is reported once, at its first occurrence.
+        """
+        is_remote = url.startswith(("http://", "https://"))
+        base_domain = get_base_domain(url) if is_remote else ""
+        internal = {}
+        external = {}
+
+        for page in pages:
+            for raw_href in page.links:
+                href = str(raw_href).strip()
+                if not href:
+                    continue
+                if is_remote:
+                    href = urljoin(url, href)
+                if href in internal or href in external:
+                    continue
+
+                if base_domain and not is_external_url(href, base_domain):
+                    internal[href] = Link(href=href, base_domain=base_domain)
+                else:
+                    external[href] = Link(href=href, base_domain=get_base_domain(href))
+
+        return Links(internal=list(internal.values()), external=list(external.values()))
+
     async def ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
         # For simple cases, you can use the sync version
         return await asyncio.to_thread(self.scrap, url, html, **kwargs)

docs/md_v2/advanced/pdf-parsing.md

@@ -112,7 +112,7 @@ When initializing `PDFContentScrapingStrategy`, you can configure its behavior u
     -   The extracted information is compiled into a `ScrapingResult` object:
         -   `cleaned_html`: Contains an HTML-like representation of the PDF, where each page's content is often wrapped in a `<div>` with page number information.
         -   `media`: A dictionary where `media["images"]` will contain information about extracted images if `extract_images` was `True`.
-        -   `links`: A dictionary where `links["urls"]` can contain URLs found within the PDF content.
+        -   `links`: The URI link annotations found in the PDF, as `Link` items in `links.internal` (same domain as the PDF URL) and `links.external` (other domains, or every absolute link of a local PDF). Each target is listed once.
         -   `metadata`: A dictionary holding PDF metadata (e.g., title, author, num_pages).
 -   **`async ascrap(self, url: str, html: str, **kwargs) -> ScrapingResult`**:
     -   The asynchronous version of `scrap`. Under the hood, it typically runs the synchronous `scrap` method in a separate thread using `asyncio.to_thread` to avoid blocking the event loop.

tests/unit/test_pdf_scraping_links_2374.py

@@ -0,0 +1,88 @@
+"""Regression tests for unclecode/crawl4ai#2374.
+
+`PDFContentScrapingStrategy.scrap()` used to build ``links = {"urls": [...]}``
+with ``{"url", "page"}`` items and hand it to ``ScrapingResult``. ``Links`` only
+has ``internal`` and ``external`` lists of ``Link`` items (keyed by ``href``),
+so pydantic dropped the unknown ``urls`` key and every URI annotation the PDF
+processor had extracted was lost, with ``success=True``.
+
+The links now land in ``internal`` / ``external`` as ``Link`` items, classified
+against the PDF's own domain the same way the HTML scraper does it.
+"""
+
+import pytest
+
+pypdf = pytest.importorskip("pypdf")
+
+from pypdf import PdfWriter
+from pypdf.annotations import Link as LinkAnnotation
+
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+
+def _write_pdf(path, pages):
+    """Write a PDF with one blank page per entry; each entry lists its URIs."""
+    writer = PdfWriter()
+    for page_index, urls in enumerate(pages):
+        writer.add_blank_page(width=612, height=792)
+        for i, url in enumerate(urls):
+            y = 20 + i * 30
+            writer.add_annotation(
+                page_index,
+                LinkAnnotation(rect=(20, y, 200, y + 20), url=url),
+            )
+    writer.write(path)
+    return path
+
+
+def _hrefs(items):
+    return [item.href for item in items]
+
+
+def test_local_pdf_uri_links_survive_scraping(tmp_path):
+    pdf = _write_pdf(tmp_path / "linked.pdf", [["https://example.org/reference"]])
+
+    result = PDFContentScrapingStrategy(extract_images=False).scrap(str(pdf), "")
+
+    assert result.success is True
+    # A local file has no domain of its own, so every absolute link is external.
+    assert _hrefs(result.links.external) == ["https://example.org/reference"]
+    assert result.links.internal == []
+    assert result.links.external[0].base_domain == "example.org"
+
+
+def test_remote_pdf_links_split_into_internal_and_external(tmp_path, monkeypatch):
+    pdf = _write_pdf(
+        tmp_path / "remote.pdf",
+        [
+            ["https://docs.example.org/guide", "https://other.net/paper"],
+            # The same target on a later page is reported once, like HTML links.
+            ["https://docs.example.org/guide", "mailto:team@example.org"],
+        ],
+    )
+    strategy = PDFContentScrapingStrategy(extract_images=False)
+    # Skip the download: serve the local file for the remote URL.
+    monkeypatch.setattr(strategy, "_get_pdf_path", lambda url: str(pdf))
+
+    result = strategy.scrap("https://www.example.org/files/report.pdf", "")
+
+    assert result.success is True
+    assert _hrefs(result.links.internal) == ["https://docs.example.org/guide"]
+    assert _hrefs(result.links.external) == [
+        "https://other.net/paper",
+        "mailto:team@example.org",
+    ]
+    assert result.links.internal[0].base_domain == "example.org"
+    assert result.links.external[0].base_domain == "other.net"
+
+
+def test_links_reach_the_model_dump_used_by_crawl_results(tmp_path):
+    pdf = _write_pdf(tmp_path / "dump.pdf", [["https://example.org/a"]])
+
+    dumped = (
+        PDFContentScrapingStrategy(extract_images=False)
+        .scrap(str(pdf), "")
+        .links.model_dump()
+    )
+
+    assert [link["href"] for link in dumped["external"]] == ["https://example.org/a"]