Curate scraped HTML for vector-DB ingest: tighten URL filter to stop scope-bleed into sister projects, save an empty stub when a configured selector misses, strip embedded TOC/nav/footer/svg/script/copyright cruft from inside extracted content, and reassign ~20 scrape entries to the right scraper class for their site's theme.

Author: BBC-Esq

core/constants.py

@@ -1599,11 +1599,13 @@
     },
     "cffi": {
         "URL": "https://cffi.readthedocs.io/en/stable/",
-        "folder": "cffi"
+        "folder": "cffi",
+        "scraper_class": "DivClassDocumentScraper"
     },
     "chardet": {
         "URL": "https://chardet.readthedocs.io/en/stable/",
-        "folder": "chardet"
+        "folder": "chardet",
+        "scraper_class": "FuroThemeScraper"
     },
     "charset-normalizer": {
         "URL": "https://charset-normalizer.readthedocs.io/en/stable/",
@@ -1612,11 +1614,13 @@
     },
     "click": {
         "URL": "https://click.palletsprojects.com/en/stable/",
-        "folder": "click"
+        "folder": "click",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "coloredlogs": {
         "URL": "https://coloredlogs.readthedocs.io/en/latest/",
-        "folder": "coloredlogs"
+        "folder": "coloredlogs",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "contourpy": {
         "URL": "https://contourpy.readthedocs.io/en/stable/",
@@ -1626,16 +1630,17 @@
     "cryptography": {
         "URL": "https://cryptography.io/en/stable/",
         "folder": "cryptography",
-        "scraper_class": "ReadthedocsScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "CTranslate2": {
         "URL": "https://opennmt.net/CTranslate2/",
         "folder": "ctranslate2",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "cycler": {
         "URL": "https://matplotlib.org/cycler/",
-        "folder": "cycler"
+        "folder": "cycler",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "dataclasses-json": {
         "URL": "https://lidatong.github.io/dataclasses-json/",
@@ -1649,11 +1654,13 @@
     },
     "deepdiff 8.6.1": {
         "URL": "https://zepworks.com/deepdiff/8.6.1/",
-        "folder": "deepdiff_861"
+        "folder": "deepdiff_861",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "Deprecated": {
         "URL": "https://deprecated.readthedocs.io/en/latest/",
-        "folder": "deprecated"
+        "folder": "deprecated",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "deprecation": {
         "URL": "https://deprecation.readthedocs.io/en/latest/",
@@ -1672,7 +1679,8 @@
     },
     "distro": {
         "URL": "https://distro.readthedocs.io/en/stable/",
-        "folder": "distro"
+        "folder": "distro",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "einops": {
         "URL": "https://einops.rocks/",
@@ -1697,7 +1705,7 @@
     "fonttools": {
         "URL": "https://fonttools.readthedocs.io/en/stable/",
         "folder": "fonttools",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "fsspec": {
         "URL": "https://filesystem-spec.readthedocs.io/en/stable/",
@@ -1764,11 +1772,6 @@
         "folder": "kiwisolver",
         "scraper_class": "ReadthedocsScraper"
     },
-    "Langchain": {
-        "URL": "https://python.langchain.com/api_reference/",
-        "folder": "langchain",
-        "scraper_class": "PydataThemeScraper"
-    },
     "llvmlite": {
         "URL": "https://llvmlite.readthedocs.io/en/stable/",
         "folder": "llvmlite",
@@ -1782,7 +1785,7 @@
     "markdown-it-py": {
         "URL": "https://markdown-it-py.readthedocs.io/en/stable/",
         "folder": "markdown_it_py",
-        "scraper_class": "MainIdMainContentRoleMainScraper"
+        "scraper_class": "PydataThemeScraper"
     },
     "markupsafe": {
         "URL": "https://markupsafe.palletsprojects.com/en/stable/",
@@ -1887,7 +1890,7 @@
     "pi-heif": {
         "URL": "https://pillow-heif.readthedocs.io/en/latest/",
         "folder": "piheif",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "pikepdf": {
         "URL": "https://pikepdf.readthedocs.io/en/stable/",
@@ -1952,6 +1955,7 @@
     "PyOpenGL": {
         "URL": "https://mcfletch.github.io/pyopengl/documentation/manual/",
         "folder": "pyopengl",
+        "scraper_class": "MainScraper"
     },
     "PyPDF": {
         "URL": "https://pypdf.readthedocs.io/en/stable/",
@@ -1966,7 +1970,7 @@
     "python-dateutil": {
         "URL": "https://dateutil.readthedocs.io/en/stable/",
         "folder": "python_dateutil",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "python-dotenv": {
         "URL": "https://saurabh-kumar.com/python-dotenv/",
@@ -2016,7 +2020,7 @@
     "Rich": {
         "URL": "https://rich.readthedocs.io/en/stable/",
         "folder": "rich",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "rpds-py": {
         "URL": "https://rpds.readthedocs.io/en/stable/",
@@ -2035,7 +2039,8 @@
     },
     "scikit-learn": {
         "URL": "https://scikit-learn.org/stable/",
-        "folder": "scikit_learn"
+        "folder": "scikit_learn",
+        "scraper_class": "PydataThemeScraper"
     },
     "SciPy 1.16.2": {
         "URL": "https://docs.scipy.org/doc/scipy-1.16.2/",
@@ -2050,17 +2055,17 @@
     "Six": {
         "URL": "https://six.readthedocs.io/",
         "folder": "six",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "sniffio": {
         "URL": "https://sniffio.readthedocs.io/en/stable/",
         "folder": "sniffio",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "SoundFile 0.13.1": {
         "URL": "https://python-soundfile.readthedocs.io/en/0.13.1/",
         "folder": "soundfile_0131",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "sounddevice 0.5.3": {
         "URL": "https://python-sounddevice.readthedocs.io/en/0.5.3/",
@@ -2075,11 +2080,12 @@
     "SpeechBrain (latest)": {
         "URL": "https://speechbrain.readthedocs.io/en/latest/",
         "folder": "speechbrain_latest",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "SQLAlchemy 20": {
         "URL": "https://docs.sqlalchemy.org/en/20/",
-        "folder": "sqlalchemy_20"
+        "folder": "sqlalchemy_20",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "sympy": {
         "URL": "https://docs.sympy.org/latest/",
@@ -2089,12 +2095,12 @@
     "tenacity": {
         "URL": "https://tenacity.readthedocs.io/en/stable/",
         "folder": "tenacity",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "Tile DB": {
         "URL": "https://tiledb-inc-tiledb.readthedocs-hosted.com/projects/tiledb-py/en/stable/",
         "folder": "tiledb",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "tiledb-vector-search": {
         "URL": "https://tiledb-inc.github.io/TileDB-Vector-Search/documentation/",
@@ -2104,6 +2110,7 @@
     "tiledb-cloud": {
         "URL": "https://tiledb-inc.github.io/TileDB-Cloud-Py/",
         "folder": "tiledb_cloud",
+        "scraper_class": "FastcoreScraper"
     },
     "Timm 1.0.20": {
         "URL": "https://huggingface.co/docs/timm/v1.0.20/en/",
@@ -2172,6 +2179,7 @@
     "webencodings": {
         "URL": "https://pythonhosted.org/webencodings/",
         "folder": "webencodings",
+        "scraper_class": "BodyRoleMainScraper"
     },
     "Wrapt": {
         "URL": "https://wrapt.readthedocs.io/en/master/",
@@ -2181,7 +2189,7 @@
     "xlrd": {
         "URL": "https://xlrd.readthedocs.io/en/stable/",
         "folder": "xlrd",
-        "scraper_class": "RstContentScraper"
+        "scraper_class": "DivClassDocumentScraper"
     },
     "yarl": {
         "URL": "https://yarl.aio-libs.org/en/stable/",

modules/scraper.py

@@ -1,4 +1,5 @@
 import os
+import re
 import asyncio
 import aiofiles
 from bs4 import BeautifulSoup
@@ -12,6 +13,85 @@
 from core.constants import PROJECT_ROOT
 
 
+_VERSION_SUFFIX_RE = re.compile(r"^v?\d+(\.\d+)*$")
+
+# Cruft commonly embedded INSIDE the main article element by Sphinx-style themes.
+# Stripped post-extraction so the saved HTML is closer to "vector-DB-ready" content.
+_CRUFT_TAGS = ("script", "style", "nav", "footer", "svg")
+_CRUFT_CLASSES = (
+    "toctree-wrapper",      # Sphinx project TOC tree, often dumped at bottom of index pages
+    "related",              # Sphinx prev/next bar
+    "sphinxsidebar",        # classic Sphinx sidebar, when scoped inside the content
+    "footer",               # Sphinx classic theme div.footer (copyright/build info)
+    "edit-this-page",       # MkDocs Material "Edit this page"
+    "md-source-file",       # MkDocs Material source-file metadata
+    "prev-next-area",       # Pydata-theme prev/next nav
+    "prev-next-bottom",
+    "prev-next-top",
+    "bd-toc",               # Pydata "On this page" sidebar
+    "bd-sidebar-secondary",
+    "feedback-widget",      # Pydantic.dev "Was this page helpful?"
+    "pagination-links",     # Pydantic.dev / generic prev-next pagination
+    "footer-version",       # PyMuPDF "This documentation covers all versions..."
+    "try_examples_button_container",  # SciPy "Try it in your browser! / Open in Tab"
+    "try_examples_outer_iframe",      # SciPy interactive-examples sandbox iframe
+)
+_CRUFT_IDS = (
+    "indices-and-tables",   # Sphinx auto-generated bottom-of-index "Index/ModIndex/Search" stub
+    "search",
+    "footerDisclaimer",     # PyMuPDF "This software is provided AS-IS..."
+)
+
+
+def _strip_trailing_version(path: str) -> str:
+    """Strip a trailing '-vX.Y.Z' / '-1.2.3' from a URL path.
+
+    Used by is_valid_url so that, e.g., a seed of /foo-v1.2/ still matches /foo-v1.3/.
+    Does NOT strip non-version suffixes (so /array-api-compat does not become /array-api).
+    """
+    parts = path.rsplit("-", 1)
+    if len(parts) == 2 and _VERSION_SUFFIX_RE.match(parts[1]):
+        return parts[0]
+    return path
+
+
+_CRUFT_HEADINGS = {"copyright", "copyrights", "license", "licenses"}
+
+
+def _strip_embedded_cruft(content):
+    """Remove TOC trees, nav, scripts, etc. that sit INSIDE the extracted main element."""
+    for tag_name in _CRUFT_TAGS:
+        for el in content.find_all(tag_name):
+            el.decompose()
+    for cls in _CRUFT_CLASSES:
+        for el in content.find_all(class_=cls):
+            el.decompose()
+    for cruft_id in _CRUFT_IDS:
+        for el in content.find_all(id=cruft_id):
+            el.decompose()
+    # Strip <section> whose first heading is a boilerplate heading like "Copyright" or "License".
+    for section in content.find_all("section"):
+        h = section.find(["h1", "h2", "h3", "h4", "h5", "h6"])
+        if h and h.get_text(strip=True).lower() in _CRUFT_HEADINGS:
+            section.decompose()
+    # Strip <p> elements whose only meaningful content is Prev/Next pagination anchors
+    # (e.g. ruamel.yaml's "<p><a>Prev</a> <a>Next</a></p>" with no class/id to target).
+    _NAV_LABELS = {"prev", "previous", "next"}
+    for p in content.find_all("p"):
+        anchors = p.find_all("a")
+        if not anchors:
+            continue
+        anchor_text = " ".join(a.get_text(strip=True).lower() for a in anchors).split()
+        if anchor_text and all(w in _NAV_LABELS for w in anchor_text):
+            # Confirm there's no extra non-anchor text in the paragraph.
+            non_anchor_text = "".join(
+                str(c) for c in p.contents if getattr(c, "name", None) != "a"
+            ).strip()
+            if not non_anchor_text:
+                p.decompose()
+    return content
+
+
 class BaseScraper:
     def __init__(self, url, folder):
         self.url = url
@@ -25,10 +105,18 @@ def __init__(self, url, folder):
     def process_html(self, soup):
         main_content = self.extract_main_content(soup)
         if main_content:
+            cleaned = _strip_embedded_cruft(deepcopy(main_content))
             new_soup = BeautifulSoup("<html><body></body></html>", "lxml")
-            new_soup.body.append(deepcopy(main_content))
+            new_soup.body.append(cleaned)
             return new_soup
-        return soup
+        # Fallback differs based on whether the user configured a scraper_class:
+        #   - BaseScraper directly (no scraper_class set) => preserve full page
+        #   - any subclass (selector configured but missed) => save empty stub
+        #     so a misconfigured selector is visible as a tiny file instead of
+        #     silently saving the full untrimmed page with TOC/nav cruft.
+        if type(self) is BaseScraper:
+            return soup
+        return BeautifulSoup("<html><body></body></html>", "lxml")
 
     def extract_main_content(self, soup):
         return None
@@ -383,7 +471,7 @@ def strip_www(netloc: str) -> str:
             return False
 
         if acceptable_domain_extension:
-            base_no_version = acceptable_domain_extension.rsplit('-', 1)[0]
+            base_no_version = _strip_trailing_version(acceptable_domain_extension)
             return (
                 parsed.path.startswith(acceptable_domain_extension) or
                 parsed.path.startswith(base_no_version)