From a7bedd17e6041b8b9f7053b7f44ce69611716285 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Nov 2024 12:50:08 +0100 Subject: [PATCH 1/2] Add documentation to index creation tool --- tools/create-index.mjs | 120 ++++++++++++++++++++++++++++++----------- 1 file changed, 88 insertions(+), 32 deletions(-) diff --git a/tools/create-index.mjs b/tools/create-index.mjs index 91d9351..1016e74 100644 --- a/tools/create-index.mjs +++ b/tools/create-index.mjs @@ -3,6 +3,12 @@ import path from 'path'; import { execSync } from 'child_process'; import yaml from 'js-yaml'; +/** + * Clones a Git repository to a specified directory + * @param {string} url The URL of the Git repository + * @param {string} directory The target directory to clone the repository to + * @returns + */ const cloneRepository = (url, directory) => { try { // Clone the repository @@ -14,6 +20,12 @@ const cloneRepository = (url, directory) => { } }; +/** + * Gets the GitHub URL of a Git repository from the local filesystem + * @param {string} rootPath The root path of the Git repository + * @returns {string} The GitHub URL of the repository + * @returns {null} If an error occurs + */ const getGitHubUrl = (rootPath) => { try { // Get the GitHub repository URL @@ -29,6 +41,12 @@ const getGitHubUrl = (rootPath) => { } }; +/** + * Gets the current Git branch of a local repository + * @param {string} rootPath The root path of the Git repository + * @returns {string} The name of the current branch + * @returns {null} If an error occurs + */ const getCurrentBranch = (rootPath) => { try { // Get the current branch name @@ -40,10 +58,16 @@ const getCurrentBranch = (rootPath) => { } }; -const getRepositoryRoot = (rootPath) => { +/** + * Gets the root folder of a Git repository (the one containing the .git folder) + * @param {string} aPath The path to a file or directory within the repository + * @returns {string} The root folder of the repository + * @returns {null} If an error occurs + */ +const getRepositoryRoot = (aPath) => { try { // Get the root folder of the repository - const repositoryRoot = execSync('git rev-parse --show-toplevel', { cwd: rootPath, encoding: 'utf-8' }).trim(); + const repositoryRoot = execSync('git rev-parse --show-toplevel', { cwd: aPath, encoding: 'utf-8' }).trim(); return repositoryRoot; } catch (error) { console.error(`Error getting repository root: ${error.message}`); @@ -51,10 +75,53 @@ const getRepositoryRoot = (rootPath) => { } }; -const searchPackages = (directory, outputFilename, indexUrl) => { +/** + * Constructs a GitHub URL for a specific directory in a Git repository. + * @param {string} baseUrl The base URL of the GitHub repository + * @param {string} branch The current branch of the repository + * @param {string} repositoryRoot The root folder of the repository + * @param {string} dirPath The path to the directory within the repository + * @returns {string} The GitHub URL for the directory + */ +const constructGitHubUrl = (baseUrl, branch, repositoryRoot, dirPath) => { + const relativePath = path.relative(repositoryRoot, dirPath); + const normalizedPath = relativePath.replace(/\\/g, '/'); // Normalize path separators for Windows + + return `${baseUrl}/tree/${branch}/${normalizedPath}`; +}; + +/** + * Extracts the description from a manifest.py file + * @param {string} filePath The path to the manifest.py file + * @returns {string} The description extracted from the file + * @returns {null} If an error occurs + */ +function extractDescription(filePath) { + try { + const content = fs.readFileSync(filePath, 'utf8'); + const descriptionMatch = /description="(.*?)"/.exec(content); + + if (descriptionMatch && descriptionMatch[1]) { + return descriptionMatch[1]; + } + } catch (error) { + console.error(`Error reading ${filePath}: ${error.message}`); + return null; + } +} + +/** + * Generates a list of packages from directories containing a package.json file + * @param {string} directory The starting directory to search for packages + * @param {string} indexUrl The URL of the package index to be assigned to each package + * @param {RegExp} excludePattern A regular expression to exclude certain directories + * @returns {string} A YAML representation of the package list as a string + */ +const generatePackageList = (directory, indexUrl, excludePattern) => { const result = { packages: [] }; const repositoryRoot = getRepositoryRoot(directory); + console.log(`Repository root: ${repositoryRoot} from ${directory}`); const gitHubUrl = getGitHubUrl(repositoryRoot); const currentBranch = getCurrentBranch(repositoryRoot); @@ -62,7 +129,7 @@ const searchPackages = (directory, outputFilename, indexUrl) => { return; } - const search = (dir, rootPath) => { + const collectPackages = (dir, rootPath) => { const files = fs.readdirSync(dir); for (const file of files) { @@ -70,12 +137,16 @@ const searchPackages = (directory, outputFilename, indexUrl) => { const isDirectory = fs.statSync(filePath).isDirectory(); if (isDirectory) { - search(filePath, rootPath); + collectPackages(filePath, rootPath); } else { const isPackageJson = file === 'package.json'; const isManifestPy = file === 'manifest.py'; const packageName = path.basename(dir); + if (excludePattern && excludePattern.test(dir)) { + continue; // Skip excluded packages + } + if(packageName.startsWith("_")) { continue; // Skip "private" packages } @@ -89,16 +160,8 @@ const searchPackages = (directory, outputFilename, indexUrl) => { }; if (isManifestPy) { - try { - const content = fs.readFileSync(filePath, 'utf8'); - const descriptionMatch = /description="(.*?)"/.exec(content); - - if (descriptionMatch && descriptionMatch[1]) { - packageInfo.description = descriptionMatch[1]; - } - } catch (error) { - console.error(`Error reading ${file}: ${error.message}`); - } + const description = extractDescription(filePath, packageInfo, file); + packageInfo.description = description; } result.packages.push(packageInfo); @@ -107,22 +170,8 @@ const searchPackages = (directory, outputFilename, indexUrl) => { } }; - const constructGitHubUrl = (baseUrl, branch, repositoryRoot, dirPath) => { - const relativePath = path.relative(repositoryRoot, dirPath); - const normalizedPath = relativePath.replace(/\\/g, '/'); // Normalize path separators for Windows - - return `${baseUrl}/tree/${branch}/${normalizedPath}`; - }; - - search(directory, repositoryRoot); - - try { - const yamlData = yaml.dump(result); - fs.writeFileSync(outputFilename, `---\n${yamlData}`); - console.log(`YAML file saved to ${outputFilename}`); - } catch (error) { - console.error(`Error writing YAML file: ${error.message}`); - } + collectPackages(directory, repositoryRoot); + return yaml.dump(result); }; // Check if command line arguments are provided @@ -142,6 +191,13 @@ if (process.argv.length < 3) { const directory = "build/micropython-lib"; const indexUrl = "https://micropython.org/pi/v2"; const outputFilename = process.argv[2]; + const excludePattern = /\/unix-ffi\//; // Skip Unix-specific packages - searchPackages(directory, outputFilename, indexUrl); + const packageList = generatePackageList(directory, indexUrl, excludePattern); + try { + fs.writeFileSync(outputFilename, `---\n${packageList}`); + console.log(`YAML file saved to ${outputFilename}`); + } catch (error) { + console.error(`Error writing YAML file: ${error.message}`); + } } From 44c988f90180d6e39b0dc2b94d448323379b4c73 Mon Sep 17 00:00:00 2001 From: Sebastian Romero Date: Fri, 29 Nov 2024 12:50:17 +0100 Subject: [PATCH 2/2] Remove unix only packages --- micropython-lib.yaml | 162 ------------------------------------------- 1 file changed, 162 deletions(-) diff --git a/micropython-lib.yaml b/micropython-lib.yaml index 1cc7c6b..bb6989c 100644 --- a/micropython-lib.yaml +++ b/micropython-lib.yaml @@ -647,165 +647,3 @@ packages: index: https://micropython.org/pi/v2 author: MicroPython description: Compression and decompression using the deflate algorithm - - name: cgi - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/cgi - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.charset - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.charset - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.encoders - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.encoders - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.errors - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.errors - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.feedparser - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.feedparser - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.header - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.header - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.internal - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.internal - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.message - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.message - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.parser - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.parser - index: https://micropython.org/pi/v2 - author: MicroPython - - name: email.utils - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/email.utils - index: https://micropython.org/pi/v2 - author: MicroPython - - name: fcntl - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/fcntl - index: https://micropython.org/pi/v2 - author: MicroPython - - name: ffilib - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/ffilib - index: https://micropython.org/pi/v2 - author: MicroPython - description: >- - MicroPython FFI helper module to easily interface with underlying shared - libraries - - name: getopt - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/getopt - index: https://micropython.org/pi/v2 - author: MicroPython - - name: gettext - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/gettext - index: https://micropython.org/pi/v2 - author: MicroPython - - name: glob - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/glob - index: https://micropython.org/pi/v2 - author: MicroPython - - name: html.entities - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/html.entities - index: https://micropython.org/pi/v2 - author: MicroPython - - name: html.parser - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/html.parser - index: https://micropython.org/pi/v2 - author: MicroPython - - name: http.client - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/http.client - index: https://micropython.org/pi/v2 - author: MicroPython - - name: json - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/json - index: https://micropython.org/pi/v2 - author: MicroPython - - name: machine - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/machine - index: https://micropython.org/pi/v2 - author: MicroPython - - name: multiprocessing - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/multiprocessing - index: https://micropython.org/pi/v2 - author: MicroPython - - name: os - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/os - index: https://micropython.org/pi/v2 - author: MicroPython - - name: pwd - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/pwd - index: https://micropython.org/pi/v2 - author: MicroPython - - name: pyusb - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/pyusb - index: https://micropython.org/pi/v2 - author: MicroPython - - name: re - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/re - index: https://micropython.org/pi/v2 - author: MicroPython - - name: select - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/select - index: https://micropython.org/pi/v2 - author: MicroPython - - name: signal - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/signal - index: https://micropython.org/pi/v2 - author: MicroPython - - name: socket - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/socket - index: https://micropython.org/pi/v2 - author: MicroPython - - name: sqlite3 - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/sqlite3 - index: https://micropython.org/pi/v2 - author: MicroPython - - name: test.support - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/test.support - index: https://micropython.org/pi/v2 - author: MicroPython - - name: time - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/time - index: https://micropython.org/pi/v2 - author: MicroPython - - name: timeit - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/timeit - index: https://micropython.org/pi/v2 - author: MicroPython - - name: tty - docs: https://github.com/micropython/micropython-lib/tree/master/unix-ffi/tty - index: https://micropython.org/pi/v2 - author: MicroPython - - name: ucurses - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/ucurses - index: https://micropython.org/pi/v2 - author: MicroPython - - name: urllib.parse - docs: >- - https://github.com/micropython/micropython-lib/tree/master/unix-ffi/urllib.parse - index: https://micropython.org/pi/v2 - author: MicroPython