Exclude unix packages

micropython-lib.yaml

@@ -647,165 +647,3 @@ packages:
     index: https://micropython.org/pi/v2
     author: MicroPython
     description: Compression and decompression using the deflate algorithm
-  - name: cgi
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/cgi
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.charset
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.charset
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.encoders
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.encoders
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.errors
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.errors
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.feedparser
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.feedparser
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.header
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.header
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.internal
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.internal
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.message
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.message
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.parser
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.parser
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: email.utils
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.utils
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: fcntl
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/fcntl
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: ffilib
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/ffilib
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-    description: >-
-      MicroPython FFI helper module to easily interface with underlying shared
-      libraries
-  - name: getopt
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/getopt
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: gettext
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/gettext
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: glob
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/glob
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: html.entities
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/html.entities
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: html.parser
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/html.parser
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: http.client
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/http.client
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: json
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/json
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: machine
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/machine
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: multiprocessing
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/multiprocessing
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: os
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/os
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: pwd
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/pwd
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: pyusb
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/pyusb
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: re
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/re
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: select
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/select
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: signal
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/signal
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: socket
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/socket
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: sqlite3
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/sqlite3
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: test.support
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/test.support
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: time
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/time
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: timeit
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/timeit
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: tty
-    docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/tty
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: ucurses
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/ucurses
-    index: https://micropython.org/pi/v2
-    author: MicroPython
-  - name: urllib.parse
-    docs: >-
-      https://github.com/micropython/micropython-lib/tree/master/unix-ffi/urllib.parse
-    index: https://micropython.org/pi/v2
-    author: MicroPython

tools/create-index.mjs

@@ -3,6 +3,12 @@ import path from 'path';
 import { execSync } from 'child_process';
 import yaml from 'js-yaml';
 
+/**
+ * Clones a Git repository to a specified directory
+ * @param {string} url The URL of the Git repository
+ * @param {string} directory The target directory to clone the repository to
+ * @returns 
+ */
 const cloneRepository = (url, directory) => {
   try {
     // Clone the repository
@@ -14,6 +20,12 @@ const cloneRepository = (url, directory) => {
   }
 };
 
+/**
+ * Gets the GitHub URL of a Git repository from the local filesystem
+ * @param {string} rootPath The root path of the Git repository
+ * @returns {string} The GitHub URL of the repository
+ * @returns {null} If an error occurs
+ */
 const getGitHubUrl = (rootPath) => {
   try {
     // Get the GitHub repository URL
@@ -29,6 +41,12 @@ const getGitHubUrl = (rootPath) => {
   }
 };
 
+/**
+ * Gets the current Git branch of a local repository
+ * @param {string} rootPath The root path of the Git repository
+ * @returns {string} The name of the current branch
+ * @returns {null} If an error occurs
+ */
 const getCurrentBranch = (rootPath) => {
   try {
     // Get the current branch name
@@ -40,42 +58,95 @@ const getCurrentBranch = (rootPath) => {
   }
 };
 
-const getRepositoryRoot = (rootPath) => {
+/**
+ * Gets the root folder of a Git repository (the one containing the .git folder)
+ * @param {string} aPath The path to a file or directory within the repository
+ * @returns {string} The root folder of the repository
+ * @returns {null} If an error occurs
+ */
+const getRepositoryRoot = (aPath) => {
   try {
     // Get the root folder of the repository
-    const repositoryRoot = execSync('git rev-parse --show-toplevel', { cwd: rootPath, encoding: 'utf-8' }).trim();
+    const repositoryRoot = execSync('git rev-parse --show-toplevel', { cwd: aPath, encoding: 'utf-8' }).trim();
     return repositoryRoot;
   } catch (error) {
     console.error(`Error getting repository root: ${error.message}`);
     return null;
   }
 };
 
-const searchPackages = (directory, outputFilename, indexUrl) => {
+/**
+ * Constructs a GitHub URL for a specific directory in a Git repository.
+ * @param {string} baseUrl The base URL of the GitHub repository
+ * @param {string} branch The current branch of the repository
+ * @param {string} repositoryRoot The root folder of the repository
+ * @param {string} dirPath The path to the directory within the repository
+ * @returns {string} The GitHub URL for the directory
+ */
+const constructGitHubUrl = (baseUrl, branch, repositoryRoot, dirPath) => {
+  const relativePath = path.relative(repositoryRoot, dirPath);
+  const normalizedPath = relativePath.replace(/\\/g, '/'); // Normalize path separators for Windows
+
+  return `${baseUrl}/tree/${branch}/${normalizedPath}`;
+};
+
+/**
+ * Extracts the description from a manifest.py file
+ * @param {string} filePath The path to the manifest.py file
+ * @returns {string} The description extracted from the file
+ * @returns {null} If an error occurs
+ */
+function extractDescription(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf8');
+    const descriptionMatch = /description="(.*?)"/.exec(content);
+
+    if (descriptionMatch && descriptionMatch[1]) {
+      return descriptionMatch[1];
+    }
+  } catch (error) {
+    console.error(`Error reading ${filePath}: ${error.message}`);
+    return null;
+  }
+}
+
+/**
+ * Generates a list of packages from directories containing a package.json file
+ * @param {string} directory The starting directory to search for packages
+ * @param {string} indexUrl The URL of the package index to be assigned to each package
+ * @param {RegExp} excludePattern A regular expression to exclude certain directories
+ * @returns {string} A YAML representation of the package list as a string
+ */
+const generatePackageList = (directory, indexUrl, excludePattern) => {
   const result = { packages: [] };
 
   const repositoryRoot = getRepositoryRoot(directory);
+  console.log(`Repository root: ${repositoryRoot} from ${directory}`);
   const gitHubUrl = getGitHubUrl(repositoryRoot);
   const currentBranch = getCurrentBranch(repositoryRoot);
 
   if (!repositoryRoot || !gitHubUrl || !currentBranch) {
     return;
   }
 
-  const search = (dir, rootPath) => {
+  const collectPackages = (dir, rootPath) => {
     const files = fs.readdirSync(dir);
 
     for (const file of files) {
       const filePath = path.join(dir, file);
       const isDirectory = fs.statSync(filePath).isDirectory();
 
       if (isDirectory) {
-        search(filePath, rootPath);
+        collectPackages(filePath, rootPath);
       } else {
         const isPackageJson = file === 'package.json';
         const isManifestPy = file === 'manifest.py';
         const packageName = path.basename(dir);
 
+        if (excludePattern && excludePattern.test(dir)) {
+          continue; // Skip excluded packages
+        }
+
         if(packageName.startsWith("_")) {
           continue; // Skip "private" packages
         }
@@ -89,16 +160,8 @@ const searchPackages = (directory, outputFilename, indexUrl) => {
           };
 
           if (isManifestPy) {
-            try {
-              const content = fs.readFileSync(filePath, 'utf8');
-              const descriptionMatch = /description="(.*?)"/.exec(content);
-
-              if (descriptionMatch && descriptionMatch[1]) {
-                packageInfo.description = descriptionMatch[1];
-              }
-            } catch (error) {
-              console.error(`Error reading ${file}: ${error.message}`);
-            }
+            const description = extractDescription(filePath, packageInfo, file);
+            packageInfo.description = description;
           }
 
           result.packages.push(packageInfo);
@@ -107,22 +170,8 @@ const searchPackages = (directory, outputFilename, indexUrl) => {
     }
   };
 
-  const constructGitHubUrl = (baseUrl, branch, repositoryRoot, dirPath) => {
-    const relativePath = path.relative(repositoryRoot, dirPath);
-    const normalizedPath = relativePath.replace(/\\/g, '/'); // Normalize path separators for Windows
-
-    return `${baseUrl}/tree/${branch}/${normalizedPath}`;
-  };
-
-  search(directory, repositoryRoot);
-
-  try {
-    const yamlData = yaml.dump(result);
-    fs.writeFileSync(outputFilename, `---\n${yamlData}`);
-    console.log(`YAML file saved to ${outputFilename}`);
-  } catch (error) {
-    console.error(`Error writing YAML file: ${error.message}`);
-  }
+  collectPackages(directory, repositoryRoot);
+  return yaml.dump(result);
 };
 
 // Check if command line arguments are provided
@@ -142,6 +191,13 @@ if (process.argv.length < 3) {
   const directory = "build/micropython-lib";
   const indexUrl = "https://micropython.org/pi/v2";
   const outputFilename = process.argv[2];
+  const excludePattern = /\/unix-ffi\//; // Skip Unix-specific packages
 
-  searchPackages(directory, outputFilename, indexUrl);
+  const packageList = generatePackageList(directory, indexUrl, excludePattern);
+  try {
+    fs.writeFileSync(outputFilename, `---\n${packageList}`);
+    console.log(`YAML file saved to ${outputFilename}`);
+  } catch (error) {
+    console.error(`Error writing YAML file: ${error.message}`);
+  }
 }