19 extend the ci to the other repo of the organization (#23)

* feat: made the github action generic for more than one repositories

Author: TechRufy

.github/workflows/ci.yaml

@@ -40,11 +40,6 @@ jobs:
         with:
           ssh-private-key: ${{ secrets.PRIVATE_SSH_KEY }}
 
-      - name: Clone private repo
-        run: |
-          ssh-keyscan -H github.com >> /root/.ssh/known_hosts
-          git clone git@github.com:ubm-driverless/ubm-f1tenth.git
-
       - name: Cache pip dependencies
         uses: actions/cache@v3
         with:
@@ -56,8 +51,53 @@ jobs:
       - name: Install tools
         run: |
           mkdir -p .pip-cache
-          pip install --cache-dir .pip-cache --upgrade pip
-          pip install --cache-dir .pip-cache -r requirements.txt
+          python3 -m pip install --cache-dir .pip-cache --upgrade pip
+          python3 -m pip install --cache-dir .pip-cache -r requirements.txt
+
+      - name: Clone configured repos
+        env:
+          ORG_READ_TOKEN: ${{ secrets.ACTION_TOKEN }}
+        run: |
+          mkdir -p /root/.ssh
+          ssh-keyscan -H github.com >> /root/.ssh/known_hosts
+          python3 - <<'PY'
+          from pathlib import Path
+          import os
+          import subprocess
+
+          import yaml
+
+          config_path = Path("repositories.yaml")
+          config = yaml.safe_load(config_path.read_text()) or {}
+          repositories = config.get("repositories", [])
+
+          if not repositories:
+              raise SystemExit("No repositories configured in repositories.yaml")
+
+          repos_root = Path("repos")
+          repos_root.mkdir(exist_ok=True)
+
+          for repository in repositories:
+              owner = repository.get("owner", "ubm-driverless")
+              name = repository["name"]
+              ref = repository.get("ref")
+              target = repos_root / name
+              token = os.environ.get("ORG_READ_TOKEN", "").strip()
+              if token:
+                remote = f"https://x-access-token:{token}@github.com/{owner}/{name}.git"
+              else:
+                remote = f"git@github.com:{owner}/{name}.git"
+
+              clone = subprocess.run(["git", "clone", remote, str(target)], check=False)
+              if clone.returncode != 0:
+                print(f"[WARN] Could not clone {owner}/{name}. Skipping this repository.")
+                continue
+
+              if ref:
+                checkout = subprocess.run(["git", "-C", str(target), "checkout", ref], check=False)
+                if checkout.returncode != 0:
+                  print(f"[WARN] Ref '{ref}' not found for {owner}/{name}. Using repository default branch.")
+          PY
 
       - name: Generate docs
         id: generate_docs
@@ -70,35 +110,33 @@ jobs:
           # Configurable Paths (mkdocs.yml needs to be updated accordingly)
           ROSDOC2_OUTPUT="./rosdoc2_generated"
           SRC_FOLDER="./src"
+          REPOS_ROOT="./repos"
           mkdir -p "$ROSDOC2_OUTPUT" "$SRC_FOLDER"
-          CPP_PACKAGES_MD="./docs/CPP_PACKAGES.md"
           echo "rosdoc2_output=$ROSDOC2_OUTPUT" >> $GITHUB_OUTPUT
 
           # Build C++ packages docs with rosdoc2
           echo "Building docs of C++ packages..."
           source /home/ubm/rosdoc2/bin/activate
           # rosdoc2 pulls sphinx which requires docutils < 0.22.
-          python -m pip install "docutils>=0.20,<0.22"
+          python3 -m pip install "docutils>=0.20,<0.22"
           source /opt/ros/foxy/setup.bash
-          echo "# C++ Packages" > "$CPP_PACKAGES_MD"
-          echo "" >> "$CPP_PACKAGES_MD"
-          for f in $(find ./ubm-f1tenth -type f -iname "package.xml"); do
+          find "$REPOS_ROOT" -type f -iname "package.xml" -print0 | while IFS= read -r -d '' f; do
             package_dir="${f%/package.xml}"
+            repo_dir="${package_dir#${REPOS_ROOT}/}"
+            repo_name="${repo_dir%%/*}"
             package_name="${package_dir##*/}"
-            package_doc_path="$ROSDOC2_OUTPUT/$package_name/index.html"
             if grep -q "rosdoc2" "$f"; then
-              echo "Building $package_name with rosdoc2"
-              rosdoc2 build -p "$package_dir" -o "$ROSDOC2_OUTPUT/"
-              if [ -f "$package_doc_path" ]; then
-                echo "- [$package_name](../$ROSDOC2_OUTPUT/$package_name/)" >> "$CPP_PACKAGES_MD"
+              echo "Building $repo_name/$package_name with rosdoc2"
+              if ! rosdoc2 build -p "$package_dir" -o "$ROSDOC2_OUTPUT/$repo_name/"; then
+                echo "[WARN] rosdoc2 failed for $repo_name/$package_name. Skipping package."
               fi
             fi
           done
           deactivate
            
           # Preparing Python packages for mkdocs
           echo "Preparing Python packages for mkdocs..."
-          for f in $(find ./ubm-f1tenth -type f -iname "package.xml"); do
+          find "$REPOS_ROOT" -type f -iname "package.xml" -print0 | while IFS= read -r -d '' f; do
             package_dir="${f%/package.xml}"
             package_name="${package_dir##*/}"
             if grep -q "ament_python" "$f"; then
@@ -107,6 +145,10 @@ jobs:
             fi
           done
 
+      - name: Generate repository pages
+        run: |
+          python3 scripts/gen_repository_pages.py
+
       - name: Build documentation
         run: |
           export PYTHONPATH="$PYTHONPATH:$(pwd)/src"

.gitignore

@@ -2,4 +2,8 @@ site/
 src/
 venv/
 .venv/
-.vscode/
+.vscode/
+repos/
+cross_reference/
+docs_build/
+rosdoc2_generated/

docs/guides/ADD_DOCUMENTATION.md

@@ -38,23 +38,54 @@ To contribute new documentation to the website, follow these steps:
         python -m venv venv
         ```
 
-    2. Enable the virtual environment (for bash shell)
+    2. Enable the virtual environment
 
         ```bash
         source ./venv/bin/activate
         ```
+
+        Windows PowerShell:
+
+        ```powershell
+        .\venv\Scripts\Activate.ps1
+        ```
+
+        Windows cmd:
+
+        ```bat
+        venv\Scripts\activate.bat
+        ```
     
     4. Install the requirements.
 
         ```bash
         pip install -r requirements.txt
         ```
 
-    3. Modify the `PYTHONPATH` environment variable for the current terminal session and serve locally the website with mkdocs
+    3. Run the local helper script to prepare generated docs content and start the preview
+
+        ```bash
+        python scripts/local_docs.py serve
+        ```
+
+    4. (Optional) Generate a static build locally
+
+        ```bash
+        python scripts/local_docs.py build
+        ```
+
+    5. (Optional) Generate C++ rosdoc2 output locally and include it in the static site
+
+        ```bash
+        python scripts/local_docs.py build-cpp
+        ```
+
+        Note: this command requires Docker Desktop (or another running Docker daemon).
+
+    6. (Optional) Serve the built static site including rosdoc2 output (recommended to test C++ links)
 
         ```bash
-        export PYTHONPATH="$PYTHONPATH:$(pwd)/src"
-        python3 -m mkdocs serve
+        python scripts/local_docs.py serve-cpp
         ```
 
 !!! tip

docs/setup/DOCS_GENERATION.md

@@ -2,7 +2,7 @@
 
 ## Introduction
 
-This guide explains how documentation generation is configured and works in the `ubm-f1tenth` repository.
+This guide explains how documentation generation is configured and works across the repositories listed in `repositories.yaml`.
 
 ## Overview
 
@@ -16,29 +16,35 @@ The whole process is automatized with the use of GitHub Actions.
 
 ### Deployment with GitHub Actions
 
-The CI/CD workflow automates the process of building and deploying documentation to GitHub Pages. The key steps in `.github/workflows/ci.yml` are:
+The CI/CD workflow automates the process of building and deploying documentation to GitHub Pages. The key steps in `.github/workflows/ci.yaml` are:
 
-1. **Clone ubm-f1tenth Repository**
+1. **Clone configured repositories**
+    - The workflow reads `repositories.yaml` and clones each listed repository into `./repos/`.
+    - Add or remove repositories from that file to control which repositories are documented.
 
-2. **Generate C++ Documentation**:
-    - Uses rosdoc2 to generate documentation for C++ packages found in the repository.
-    - The script searches for package.xml files in the ubm-f1tenth directory and identifies packages that use rosdoc2.
-    - For each identified package, it runs rosdoc2 build to generate HTML documentation, which is stored in a specified output directory.
-    - A markdown file (CPP_PACKAGES.md) is created listing the packages with links to their generated documentation.
+2. **Generate repository pages**:
+    - `scripts/gen_repository_pages.py` creates a page for each repository listed in `repositories.yaml`.
+    - Each repository page contains two subsections: one for Python packages and one for C++ packages.
+    - The C++ subsection links to the rosdoc2 output generated for that repository.
 
-3. **Move Python packages to src**:
-    - Copies Python packages found within the repository into the `src/` folder to prepare them for MkDocs documentation generation. This is done by checking for the presence of ament_python in each package.xml file.
+3. **Generate C++ Documentation**:
+    - Uses rosdoc2 to generate documentation for C++ packages found in the configured repositories.
+    - The script searches for package.xml files under `./repos/` and identifies packages that use rosdoc2.
+    - For each identified package, it runs rosdoc2 build to generate HTML documentation, which is stored under a repository-specific output directory.
 
-4. **Build Documentation**:
+4. **Move Python packages to src**:
+    - Copies Python packages found within the configured repositories into the `src/` folder to prepare them for MkDocs documentation generation. This is done by checking for the presence of ament_python in each package.xml file.
+
+5. **Build Documentation**:
     - Runs MkDocs to generate static documentation from the source files.
     - All the configs are set in the `mkdocs.yaml` file.
     - `mkdocs build` clears the content of the `site/` directory, builds the content inside the `docs/` directory and processes with mkdocstrings the source code present in `src/` directory. The output will be in the `site` directory.
     - Custom navigation is generated dynamically based on Python module structure by the `gen_ref_pages.py` present in the script/ folder.
 
-5. **Add Additional Files to Site**:
+6. **Add Additional Files to Site**:
     - Copies the generated C++ documentation to the `site` directory for deployment.
 
-6. **Deploy with ghp-import**:
+7. **Deploy with ghp-import**:
     - Deploys the content of `site/` to GitHub Pages using the `ghp-import` tool.
 
     !!! danger

mkdocs.yaml

@@ -9,8 +9,7 @@ site_url: https://ubm-driverless.github.io/ubm-docs/
 nav:
   - Setup: setup/
   - Guides: guides/
-  - Python Packages: PYTHON_PACKAGES/
-  - C++ Packages: CPP_PACKAGES/
+  - Repositories: repositories.md
 # - NAME_OF_THE_SECTION: FOLDER_NAME_INSIDE_DOCS/ # To set up another entry
 
 
@@ -50,6 +49,7 @@ plugins:
   - gen-files:
       scripts:
         - scripts/gen_ref_pages.py
+        - scripts/gen_repository_pages.py
   - literate-nav:
       nav_file: SUMMARY.md
   - section-index

repositories.yaml

@@ -0,0 +1,7 @@
+repositories:
+  - owner: ubm-driverless
+    name: ubm-f1tenth
+    ref: main
+  - owner: ubm-driverless
+    name: ubm-fsae
+    ref: main

requirements.txt

@@ -7,3 +7,4 @@ mkdocs-section-index
 mkdocs-github-admonitions-plugin
 mkdocs-site-urls
 ghp-import
+PyYAML