Allowed search results for Django code terms which contain stop words.

Author: sarahboyce

docs/models.py

@@ -2,6 +2,7 @@
 import html
 import json
 import operator
+import re
 from functools import reduce
 from pathlib import Path
 
@@ -187,6 +188,10 @@ def sync_to_db(self, decoded_documents):
             document_path = _clean_document_path(document["current_page_name"])
             document["slug"] = Path(document_path).parts[-1]
             document["parents"] = " ".join(Path(document_path).parts[:-1])
+            document["code_references"] = _generate_code_references(document["body"])
+            document["code_references_search"] = " ".join(
+                document["code_references"].keys()
+            )
             Document.objects.create(
                 release=self,
                 path=document_path,
@@ -213,6 +218,52 @@ def _clean_document_path(path):
     return path
 
 
+def _generate_code_references(body):
+    """
+    Django documents classes with the syntax `.. class::`.
+    This results in the following HTML:
+    <dl class="py class">
+        <dt class="sig sig-object py" id="django.db.models.ManyToManyField">
+        ...
+        </dt>
+    </dl>
+    This is similar for attributes (`.. attribute::`), methods etc.
+    """
+    # Collect all <dt> HTML tag ids into a list, e.g:
+    # [
+    #    'django.db.models.Index',
+    #    'django.db.models.Index.expressions',
+    #    'django.db.models.Index.fields',
+    #    ...
+    # ]
+    code_references = list(re.findall(r'<dt[^>]+id="([^"]+)"', body))
+    # As the search term can be "expressions", "Index.expressions" etc. create a mapping
+    # between potential code search terms and their HTML id.
+    # {
+    #   'django.db.models.Index': 'django.db.models.Index',
+    #   'Index': 'django.db.models.Index',
+    #   'models.Index': 'django.db.models.Index',
+    #   'db.models.Index': 'django.db.models.Index',
+    #   'django.db.models.Index.expressions': 'django.db.models.Index.expressions',
+    #   'expressions': 'django.db.models.Index.expressions',
+    #   'Index.expressions': 'django.db.models.Index.expressions',
+    #   'models.Index.expressions': 'django.db.models.Index.expressions',
+    #   'db.models.Index.expressions': 'django.db.models.Index.expressions',
+    #   'django.db.models.Index.fields': 'django.db.models.Index.fields',
+    #   'fields': 'django.db.models.Index.fields',
+    #   'Index.fields': 'django.db.models.Index.fields',
+    #   'models.Index.fields': 'django.db.models.Index.fields',
+    #   'db.models.Index.fields': 'django.db.models.Index.fields',
+    #   ...
+    # }
+    code_paths = {}
+    for reference in code_references:
+        code_path = reference.split(".")
+        for i in range(len(code_path)):
+            code_paths[".".join(code_path[-i:])] = reference
+    return code_paths
+
+
 def document_url(doc):
     if doc.path:
         kwargs = {
@@ -251,7 +302,10 @@ def search(self, query_text, release):
             search_query = SearchQuery(
                 query_text, config=models.F("config"), search_type="websearch"
             )
-            search_rank = SearchRank(models.F("search"), search_query)
+            search_query_simple = SearchQuery(
+                query_text, config="simple", search_type="websearch"
+            )
+            search_rank = SearchRank(models.F("search"), search_query_simple)
             base_qs = (
                 self.prefetch_related(
                     Prefetch(
@@ -287,7 +341,7 @@ def search(self, query_text, release):
             )
             vector_qs = (
                 base_qs.alias(rank=search_rank)
-                .filter(search=search_query)
+                .filter(search=search_query_simple)
                 .order_by("-rank")
             )
             if not vector_qs:

docs/search.py

@@ -43,6 +43,11 @@
     SearchVector("title", weight="A", config=F("config"))
     + SearchVector(KeyTextTransform("slug", "metadata"), weight="A", config=F("config"))
     + SearchVector(KeyTextTransform("toc", "metadata"), weight="B", config=F("config"))
+    + SearchVector(
+        KeyTextTransform("code_references_search", "metadata"),
+        weight="B",
+        config="simple",
+    )
     + SearchVector(KeyTextTransform("body", "metadata"), weight="C", config=F("config"))
     + SearchVector(
         KeyTextTransform("parents", "metadata"), weight="D", config=F("config")