feat: create metadata entries generator

bin/commands/generate.mjs

@@ -1,6 +1,5 @@
 import { cpus } from 'node:os';
 import { resolve } from 'node:path';
-import process from 'node:process';
 
 import { coerce } from 'semver';
 
@@ -12,7 +11,6 @@ import createGenerator from '../../src/generators.mjs';
 import { publicGenerators } from '../../src/generators/index.mjs';
 import createNodeReleases from '../../src/releases.mjs';
 import { loadAndParse } from '../utils.mjs';
-import { runLint } from './lint.mjs';
 
 const availableGenerators = Object.keys(publicGenerators);
 
@@ -125,20 +123,23 @@ export default {
   async action(opts) {
     const docs = await loadAndParse(opts.input, opts.ignore);
 
-    if (!opts.skipLint && !runLint(docs)) {
-      console.error('Lint failed; aborting generation.');
-      process.exit(1);
-    }
+    // if (!opts.skipLint && !runLint(docs)) {
+    //   console.error('Lint failed; aborting generation.');
+    //   process.exit(1);
+    // }
 
-    const { runGenerators } = createGenerator(docs);
     const { getAllMajors } = createNodeReleases(opts.changelog);
 
+    const releases = await getAllMajors();
+
+    const { runGenerators } = createGenerator(docs);
+
     await runGenerators({
       generators: opts.target,
       input: opts.input,
       output: opts.output && resolve(opts.output),
       version: coerce(opts.version),
-      releases: await getAllMajors(),
+      releases,
       gitRef: opts.gitRef,
       threads: parseInt(opts.threads, 10),
     });

bin/utils.mjs

@@ -23,7 +23,7 @@ const parser = lazy(createMarkdownParser);
  * Load and parse markdown API docs.
  * @param {string[]} input - Glob patterns for input files.
  * @param {string[]} [ignore] - Glob patterns to ignore.
- * @returns {Promise<ApiDocMetadataEntry[]>} - Parsed documentation objects.
+ * @returns {Promise<ParserOutput<import('mdast').Root>[]>}
  */
 export async function loadAndParse(input, ignore) {
   const files = await loader().loadFiles(input, ignore);

src/generators.mjs

@@ -4,11 +4,6 @@ import { allGenerators } from './generators/index.mjs';
 import WorkerPool from './threading/index.mjs';
 
 /**
- * @typedef {{ ast: GeneratorMetadata<ApiDocMetadataEntry, ApiDocMetadataEntry>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
- * @typedef {AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
- * @param markdownInput
- * @param jsInput
- *
  * This method creates a system that allows you to register generators
  * and then execute them in a specific order, keeping track of the
  * generation process, and handling errors that may occur from the
@@ -21,18 +16,20 @@ import WorkerPool from './threading/index.mjs';
  * Generators can also write to files. These would usually be considered
  * the final generators in the chain.
  *
- * @param {ApiDocMetadataEntry} markdownInput The parsed API doc metadata entries
- * @param {Array<import('acorn').Program>} parsedJsFiles
+ * @typedef {{ ast: GeneratorMetadata<ParserOutput, ParserOutput>}} AstGenerator The AST "generator" is a facade for the AST tree and it isn't really a generator
+ * @typedef {AvailableGenerators & AstGenerator} AllGenerators A complete set of the available generators, including the AST one
+ *
+ * @param {ParserOutput} input The API doc AST tree
  */
-const createGenerator = markdownInput => {
+const createGenerator = input => {
   /**
    * We store all the registered generators to be processed
    * within a Record, so we can access their results at any time whenever needed
    * (we store the Promises of the generator outputs)
    *
    * @type {{ [K in keyof AllGenerators]: ReturnType<AllGenerators[K]['generate']> }}
    */
-  const cachedGenerators = { ast: Promise.resolve(markdownInput) };
+  const cachedGenerators = { ast: Promise.resolve(input) };
 
   const threadPool = new WorkerPool();
 

src/generators/addon-verify/index.mjs

@@ -30,7 +30,7 @@ export default {
   description:
     'Generates a file list from code blocks extracted from `doc/api/addons.md` to facilitate C++ compilation and JavaScript runtime validations',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a file list from code blocks.

src/generators/ast-js/index.mjs

@@ -20,7 +20,7 @@ export default {
 
   description: 'Parses Javascript source files passed into the input.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * @param {Input} _

src/generators/index.mjs

@@ -11,8 +11,10 @@ import apiLinks from './api-links/index.mjs';
 import oramaDb from './orama-db/index.mjs';
 import astJs from './ast-js/index.mjs';
 import llmsTxt from './llms-txt/index.mjs';
+import metadata from './metadata/index.mjs';
 
 export const publicGenerators = {
+  metadata: metadata,
   'json-simple': jsonSimple,
   'legacy-html': legacyHtml,
   'legacy-html-all': legacyHtmlAll,

src/generators/json-simple/index.mjs

@@ -26,7 +26,7 @@ export default {
   description:
     'Generates the simple JSON version of the API docs, and returns it as a string',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the simplified JSON version of the API docs

src/generators/legacy-html/index.mjs

@@ -41,7 +41,7 @@ export default {
   description:
     'Generates the legacy version of the API docs in HTML, with the assets and styles included as files',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the legacy version of the API docs in HTML

src/generators/legacy-json/index.mjs

@@ -25,7 +25,7 @@ export default {
 
   description: 'Generates the legacy version of the JSON API docs.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a legacy JSON file.

src/generators/llms-txt/index.mjs

@@ -19,7 +19,7 @@ export default {
   description:
     'Generates a llms.txt file to provide information to LLMs at inference time',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates a llms.txt file

src/generators/man-page/index.mjs

@@ -25,7 +25,7 @@ export default {
 
   description: 'Generates the Node.js man-page.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the Node.js man-page

src/generators/metadata/index.mjs

@@ -0,0 +1,28 @@
+'use strict';
+
+import { parseApiDoc } from './utils/parse.mjs';
+
+/**
+ * This generator generates a flattened list of metadata entries from a API doc
+ *
+ * @typedef {ParserOutput<import('mdast').Root>[]} Input
+ *
+ * @type {GeneratorMetadata<Input, ApiDocMetadataEntry[]>}
+ */
+export default {
+  name: 'metadata',
+
+  version: '1.0.0',
+
+  description: 'generates a flattened list of API doc metadata entries',
+
+  dependsOn: 'ast',
+
+  /**
+   * @param {Input} inputs
+   * @returns {Promise<ApiDocMetadataEntry[]>}
+   */
+  async generate(inputs) {
+    return inputs.flatMap(input => parseApiDoc(input));
+  },
+};

src/generators/metadata/utils/parse.mjs

@@ -0,0 +1,148 @@
+'use strict';
+
+import { u as createTree } from 'unist-builder';
+import { findAfter } from 'unist-util-find-after';
+import { remove } from 'unist-util-remove';
+import { selectAll } from 'unist-util-select';
+import { SKIP, visit } from 'unist-util-visit';
+
+import createQueries from '../../../utils/queries/index.mjs';
+import { getRemark } from '../../../utils/remark.mjs';
+import { createNodeSlugger } from '../../../utils/slugger/index.mjs';
+import createMetadata from '../../../metadata.mjs';
+
+/**
+ * This generator generates a flattened list of metadata entries from a API doc
+ *
+ * @param {ParserOutput<import('mdast').Root>} input
+ * @returns {Promise<ApiDocMetadataEntry[]>}
+ */
+export const parseApiDoc = ({ file, tree }) => {
+  /**
+   * This holds references to all the Metadata entries for a given file
+   * this is used so we can traverse the AST tree and keep mutating things
+   * and then stringify the whole api doc file at once without creating sub traversals
+   *
+   * Then once we have the whole file parsed, we can split the resulting string into sections
+   * and seal the Metadata Entries (`.create()`) and return the result to the caller of parae.
+   *
+   * @type {Array<ApiDocMetadataEntry>}
+   */
+  const metadataCollection = [];
+
+  const {
+    setHeadingMetadata,
+    addYAMLMetadata,
+    updateMarkdownLink,
+    updateTypeReference,
+    updateLinkReference,
+    addStabilityMetadata,
+  } = createQueries();
+
+  // Creates an instance of the Remark processor with GFM support
+  // which is used for stringifying the AST tree back to Markdown
+  const remarkProcessor = getRemark();
+
+  // Creates a new Slugger instance for the current API doc file
+  const nodeSlugger = createNodeSlugger();
+
+  // Get all Markdown Footnote definitions from the tree
+  const markdownDefinitions = selectAll('definition', tree);
+
+  // Get all Markdown Heading entries from the tree
+  const headingNodes = selectAll('heading', tree);
+
+  // Handles Markdown link references and updates them to be plain links
+  visit(tree, createQueries.UNIST.isLinkReference, node =>
+    updateLinkReference(node, markdownDefinitions)
+  );
+
+  // Removes all the original definitions from the tree as they are not needed
+  // anymore, since all link references got updated to be plain links
+  remove(tree, markdownDefinitions);
+
+  // Handles the normalisation URLs that reference to API doc files with .md extension
+  // to replace the .md into .html, since the API doc files get eventually compiled as HTML
+  visit(tree, createQueries.UNIST.isMarkdownUrl, node =>
+    updateMarkdownLink(node)
+  );
+
+  // If the document has no headings but it has content, we add a fake heading to the top
+  // so that our parsing logic can work correctly, and generate content for the whole file
+  if (headingNodes.length === 0 && tree.children.length > 0) {
+    tree.children.unshift(createTree('heading', { depth: 1 }, []));
+  }
+
+  // Handles iterating the tree and creating subtrees for each API doc entry
+  // where an API doc entry is defined by a Heading Node
+  // (so all elements after a Heading until the next Heading)
+  // and then it creates and updates a Metadata entry for each API doc entry
+  // and then generates the final content for each API doc entry and pushes it to the collection
+  visit(tree, createQueries.UNIST.isHeading, (headingNode, index) => {
+    // Creates a new Metadata entry for the current API doc file
+    const apiEntryMetadata = createMetadata(nodeSlugger);
+
+    // Adds the Metadata of the current Heading Node to the Metadata entry
+    setHeadingMetadata(headingNode, apiEntryMetadata);
+
+    // We retrieve the immediate next Heading if it exists
+    // This is used for ensuring that we don't include items that would
+    // belong only to the next heading to the current Heading metadata
+    // Note that if there is no next heading, we use the current node as the next one
+    const nextHeadingNode =
+      findAfter(tree, index, createQueries.UNIST.isHeading) ?? headingNode;
+
+    // This is the cutover index of the subtree that we should get
+    // of all the Nodes within the AST tree that belong to this section
+    // If `next` is equals the current heading, it means there's no next heading
+    // and we are reaching the end of the document, hence the cutover should be the end of
+    // the document itself.
+    const stop =
+      headingNode === nextHeadingNode
+        ? tree.children.length
+        : tree.children.indexOf(nextHeadingNode);
+
+    // Retrieves all the nodes that should belong to the current API docs section
+    // `index + 1` is used to skip the current Heading Node
+    const subTree = createTree('root', tree.children.slice(index, stop));
+
+    // Visits all Stability Index nodes from the current subtree if there's any
+    // and then apply the Stability Index metadata to the current metadata entry
+    visit(subTree, createQueries.UNIST.isStabilityNode, node =>
+      addStabilityMetadata(node, apiEntryMetadata)
+    );
+
+    // Visits all HTML nodes from the current subtree and if there's any that matches
+    // our YAML metadata structure, it transforms into YAML metadata
+    // and then apply the YAML Metadata to the current Metadata entry
+    visit(subTree, createQueries.UNIST.isYamlNode, node => {
+      // TODO: Is there always only one YAML node?
+      apiEntryMetadata.setYamlPosition(node.position);
+      addYAMLMetadata(node, apiEntryMetadata);
+    });
+
+    // Visits all Text nodes from the current subtree and if there's any that matches
+    // any API doc type reference and then updates the type reference to be a Markdown link
+    visit(subTree, createQueries.UNIST.isTextWithType, (node, _, parent) =>
+      updateTypeReference(node, parent)
+    );
+
+    // Removes already parsed items from the subtree so that they aren't included in the final content
+    remove(subTree, [createQueries.UNIST.isYamlNode]);
+
+    // Applies the AST transformations to the subtree based on the API doc entry Metadata
+    // Note that running the transformation on the subtree isn't costly as it is a reduced tree
+    // and the GFM transformations aren't that heavy
+    const parsedSubTree = remarkProcessor.runSync(subTree);
+
+    // We seal and create the API doc entry Metadata and push them to the collection
+    const parsedApiEntryMetadata = apiEntryMetadata.create(file, parsedSubTree);
+
+    // We push the parsed API doc entry Metadata to the collection
+    metadataCollection.push(parsedApiEntryMetadata);
+
+    return SKIP;
+  });
+
+  return metadataCollection;
+};

src/generators/orama-db/index.mjs

@@ -21,7 +21,7 @@ export default {
 
   description: 'Generates the Orama database for the API docs.',
 
-  dependsOn: 'ast',
+  dependsOn: 'metadata',
 
   /**
    * Generates the Orama database.

src/linter/tests/fixtures/issues.mjs

@@ -1,5 +1,3 @@
-// @ts-check
-
 /**
  * @type {import('../../types').LintIssue}
  */

src/loaders/markdown.mjs

@@ -15,8 +15,8 @@ const createLoader = () => {
   /**
    * Loads API Doc files and transforms it into VFiles
    *
-   * @param {string} searchPath A glob/path for API docs to be loaded
-   * @param {string | undefined} ignorePath A glob/path of files to ignore
+   * @param {string[]} searchPath A glob/path for API docs to be loaded
+   * @param {string[] | undefined} ignorePath A glob/path of files to ignore
    * The input string can be a simple path (relative or absolute)
    * The input string can also be any allowed glob string
    *

src/metadata.mjs

@@ -109,7 +109,7 @@ const createMetadata = slugger => {
      * The Navigation entries has a dedicated separate method for retrieval
      * as it can be manipulated outside of the scope of the generation of the content
      *
-     * @param {import('vfile').VFile} apiDoc The API doc file being parsed
+     * @param {{stem?: string, basename?: string}} apiDoc The API doc file being parsed
      * @param {ApiDocMetadataEntry['content']} section An AST tree containing the Nodes of the API doc entry section
      * @returns {ApiDocMetadataEntry} The locally created Metadata entries
      */