DOC-13760 produce markdown per page (WIP)

antora-playbook-staging-chatbot.diff.yml

@@ -1,6 +1,7 @@
 antora:
   extensions:
   - ./lib/report-tree.js
+  - ./lib/markdown-for-llm.js
 
 site:
   title: Couchbase Docs Staging Chatbot
@@ -94,9 +95,9 @@ content:
   - url: https://github.com/couchbaselabs/docs-ai
     branches: [ai-services-ga]
 
-# ui:
-  # bundle:
-    # url: https://github.com/couchbase/docs-ui/releases/download/prod-216/ui-bundle.zip
+ui:
+  bundle:
+    url: https://github.com/couchbase/docs-ui/releases/download/DOC-13760-markdown-alt-1/ui-bundle.zip
 
 asciidoc:
   attributes:

antora-playbook-staging-chatbot.yml

@@ -13,9 +13,6 @@ git:
 urls:
   latest_version_segment_strategy: redirect:to
   latest_version_segment: current
-ui:
-  bundle:
-    url: https://github.com/couchbase/docs-ui/releases/download/prod-218/ui-bundle.zip
 output:
   dir: ./public
 runtime:
@@ -26,6 +23,7 @@ antora:
     - ./lib/embargo.js
     - ./lib/antora-component-version-rank.js
     - ./lib/report-tree.js
+    - ./lib/markdown-for-llm.js
 site:
   title: Couchbase Docs Staging Chatbot
   url: https://chatbot-staging.docs-test.couchbase.com
@@ -58,6 +56,9 @@ site:
         { "title": "Contribute", "components": ["home", "styleguide", "ui-ux", "pendo"] }
       ]
     google_analytics: null
+ui:
+  bundle:
+    url: https://github.com/couchbase/docs-ui/releases/download/DOC-13760-markdown-alt-1/ui-bundle.zip
 asciidoc:
   extensions:
     - ./lib/source-url-include-processor.js
@@ -183,6 +184,7 @@ content:
         - master
     - url: https://github.com/couchbaselabs/docs-enterprise-analytics
       branches:
+        - release/2.1
         - release/2.0
     - url: https://github.com/couchbase/docs-analytics
       branches:
@@ -240,9 +242,9 @@ content:
         - release/1.0
     - url: https://github.com/couchbase/docs-sdk-java
       branches:
+        - release/3.10
         - release/3.9
         - release/3.8
-        - temp/3.7
     - url: https://github.com/couchbase/docs-quarkus-extension
       branches:
         - release/1.2
@@ -257,9 +259,9 @@ content:
         - release/1.2
     - url: https://github.com/couchbase/docs-sdk-scala
       branches:
+        - release/3.10
         - release/3.9
         - release/1.8
-        - release/1.7
     - url: https://github.com/couchbase/docs-sdk-nodejs
       branches:
         - temp/4.6

antora-playbook.preview.yml

@@ -23,6 +23,7 @@ antora:
     - ./lib/embargo.js
     - ./lib/antora-component-version-rank.js
     - ./lib/preview.js
+    - ./lib/markdown-for-llm.js
     - ./lib/report-tree.js
 site:
   url: https://docs.couchbase.com
@@ -146,6 +147,7 @@ content:
     - url: https://github.com/couchbase/docs-connectors-talend
     - url: https://github.com/couchbaselabs/docs-enterprise-analytics
       branches:
+        - release/2.1
         - release/2.0
     - url: https://github.com/couchbase/docs-analytics
       branches:

home/preview/DOC-13760-produce-markdown-per-page.yml

@@ -0,0 +1,4 @@
+override:
+  ui:
+    bundle:
+      url: https://github.com/couchbase/docs-ui/releases/download/DOC-13760-markdown-alt-1/ui-bundle.zip

lib/markdown-for-llm.js

@@ -0,0 +1,78 @@
+'use strict'
+
+const {convertHtmlToMarkdown} = require('dom-to-semantic-markdown')
+const {JSDOM} = require('jsdom')
+
+function overrideElementProcessing (element) {
+
+  if (element.tagName?.toLowerCase() === 'a' 
+   && element.className === 'anchor' ) {
+      return [{type: 'custom', blank: true}]
+  }
+
+  if (element.classList?.contains("admonitionblock")) {
+    element.classList.remove('admonitionblock')
+    const admonition = element.className.toUpperCase()
+    const content = toMarkdown(
+      element.querySelector("td.content").innerHTML)
+
+    return [{
+      type: 'custom', 
+      admonition,
+      content
+    }]
+  }
+}
+
+function renderCustomNode (node) {
+  if (node.blank) {
+    return ''
+  }
+  if (node.admonition) {
+    const body = node.content.split('\n').map(line => `> ${line}`).join('\n')
+    return `\n> [!${node.admonition}]\n${body}\n\n`
+  }
+}
+
+function toMarkdown (html) {
+  const dom = new JSDOM(html)
+  const markdown = convertHtmlToMarkdown(
+    html,
+    {
+      overrideDOMParser: new dom.window.DOMParser(),
+      overrideElementProcessing,
+      renderCustomNode
+    }
+  )
+  return markdown
+}
+
+function markdownify(page, siteCatalog) {
+    const html = page.contents.toString()
+    const markdown = `# ${page.asciidoc.doctitle}\n\n` + toMarkdown(html)
+
+    const path = page.out.path.replace(/\.html$/, '.md')
+
+    // tell docs-ui to output <link rel="alternate" ...> for the markdown page.
+    page.asciidoc.attributes["page-markdown-alt"] = `${page.out.rootPath}/${path}`
+
+    siteCatalog.addFile({
+      contents: Buffer.from(markdown),
+      out: { path }
+    })
+}
+
+module.exports.register = function ({ playbook, config }) {
+    const logger = this.getLogger('markdown-for-llm')
+
+    this.on('navigationBuilt', ({ playbook, siteAsciiDocConfig, siteCatalog, uiCatalog, contentCatalog }) => {
+
+        logger.info('Compiling Markdown summaries')
+
+        for (const page of contentCatalog.getPages()) {
+          if (page.pub) {
+            markdownify(page, siteCatalog)
+          }
+        }
+    })
+}