From 19354df37cd7d29010cc340f5ee571b123ab5088 Mon Sep 17 00:00:00 2001
From: zhongxiao-chang <313725528+zhongxiao-chang@users.noreply.github.com>
Date: Mon, 19 Oct 2026 19:21:31 +0800
Subject: [PATCH] fix(scraping): keep text after removed
 style/link/meta/noscript tags (#2416)

`LXMLWebScrapingStrategy._scrap` removed <style>, <link>, <meta> and
<noscript> elements with `element.getparent().remove(element)`. In lxml the
text that follows an element inside its parent lives in the element's
`.tail`, and `remove()` discards the tail with the element -- so a paragraph
that contains one of those tags lost everything after it, while the crawl
still reported success.

Wikipedia emits `<link rel="mw:PageProp/Category">` inside paragraphs, so
this silently truncated ordinary article text.

Switch the loop to `drop_tree()`, which moves the tail onto the previous
sibling (or the parent when there is none). The <script> loop directly below
already re-attaches `.tail` by hand, so the intent of the surrounding code is
unchanged; this loop was the one site that still dropped it.

Adds tests/unit/test_removed_tags_keep_tail.py: five parametrised shapes
(link/meta/style/noscript inside a sentence, tag as first child) plus
neighbouring-inline cases, running fully offline through
LXMLWebScrapingStrategy -> DefaultMarkdownGenerator. 6 of the 7 cases fail on
develop without this change and all 7 pass with it.
---
 crawl4ai/content_scraping_strategy.py     |  7 +-
 tests/unit/test_removed_tags_keep_tail.py | 94 +++++++++++++++++++++++
 2 files changed, 99 insertions(+), 2 deletions(-)
 create mode 100644 tests/unit/test_removed_tags_keep_tail.py

diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py
index ab7bc3efb..0e986c494 100644
--- a/crawl4ai/content_scraping_strategy.py
+++ b/crawl4ai/content_scraping_strategy.py
@@ -823,11 +823,14 @@ def _scrap(
                 except Exception:
                     pass
 
-            # Remove script and style tags
+            # Remove style/link/meta/noscript tags, but keep the text that follows
+            # them: lxml stores that text as the element's .tail, and remove()
+            # discards the tail along with the element (see issue #2416).
+            # drop_tree() moves the tail onto the previous sibling or the parent.
             for tag in ["style", "link", "meta", "noscript"]:
                 for element in body.xpath(f".//{tag}"):
                     if element.getparent() is not None:
-                        element.getparent().remove(element)
+                        element.drop_tree()
                         
             # Handle script separately
             for element in body.xpath(f".//script"):
diff --git a/tests/unit/test_removed_tags_keep_tail.py b/tests/unit/test_removed_tags_keep_tail.py
new file mode 100644
index 000000000..bc129c539
--- /dev/null
+++ b/tests/unit/test_removed_tags_keep_tail.py
@@ -0,0 +1,94 @@
+"""Text after removed <style>/<link>/<meta>/<noscript> elements must survive.
+
+Regression tests for issue #2416: the scraping strategy removed those elements
+with ``element.getparent().remove(element)``, which also drops the element's
+``.tail`` -- the text that follows it inside the same parent.  Wikipedia puts
+``<link rel="mw:PageProp/Category">`` inside paragraphs, so a kept paragraph
+lost everything after such a link, silently (the crawl still reports success).
+
+The tests run fully offline: the strategy and the markdown generator only need
+an HTML string and a base URL.
+"""
+
+import pytest
+
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+BASE_URL = "https://example.com/article"
+
+
+def scrape_and_markdown(fragment):
+    result = LXMLWebScrapingStrategy().scrap(
+        BASE_URL,
+        "<html><body>" + fragment + "</body></html>",
+        word_count_threshold=0,
+    )
+    assert result.success
+    markdown = DefaultMarkdownGenerator().generate_markdown(
+        result.cleaned_html, base_url=BASE_URL
+    )
+    return result.cleaned_html, markdown.raw_markdown
+
+
+@pytest.mark.parametrize(
+    "fragment,expected",
+    [
+        pytest.param(
+            '<p>The port grew quickly, and as of 2020<link rel="mw:PageProp/Category" href="/x">,'
+            " it handles more cargo than any other port.</p>",
+            "The port grew quickly, and as of 2020, it handles more cargo than any other port.",
+            id="link-inside-sentence",
+        ),
+        pytest.param(
+            '<p>The old warehouses stood on the quay<meta property="mw:PageProp/toc">'
+            " until the great fire destroyed them.</p>",
+            "The old warehouses stood on the quay until the great fire destroyed them.",
+            id="meta-inside-sentence",
+        ),
+        pytest.param(
+            "<p>Styled text<style>.a{color:red}</style> continues after a style element.</p>",
+            "Styled text continues after a style element.",
+            id="style-inside-sentence",
+        ),
+        pytest.param(
+            "<p>Plain text<noscript>Enable JavaScript</noscript> continues after a noscript element.</p>",
+            "Plain text continues after a noscript element.",
+            id="noscript-inside-sentence",
+        ),
+        pytest.param(
+            '<p><link rel="mw:PageProp/Category" href="/x">Text at the start of the paragraph.</p>',
+            "Text at the start of the paragraph.",
+            id="element-is-first-child",
+        ),
+    ],
+)
+def test_text_after_removed_element_is_kept(fragment, expected):
+    cleaned_html, markdown = scrape_and_markdown(fragment)
+
+    assert expected in markdown
+    assert expected.replace("&", "&amp;") in cleaned_html
+    # The element itself is still gone.
+    for tag in ("<style", "<link", "<meta", "<noscript"):
+        assert tag not in cleaned_html
+
+
+def test_removed_element_between_two_sentences():
+    fragment = (
+        '<p>First sentence.<link rel="mw:PageProp/Category" href="/x">'
+        "Second sentence.</p>"
+    )
+    cleaned_html, markdown = scrape_and_markdown(fragment)
+
+    assert "First sentence.Second sentence." in markdown
+
+
+def test_neighbouring_inline_markup_is_kept():
+    fragment = (
+        "<p>Before <strong>bold</strong><style>.a{color:red}</style>"
+        " <em>after</em> the style element.</p>"
+    )
+    _, markdown = scrape_and_markdown(fragment)
+
+    assert "Before **bold**" in markdown
+    assert "_after_ the style element." in markdown