improve built repository action to be usable from filesystem directly

timfel · timfel · commit 03d80518fd6c · 2023-12-08T14:05:00.000+01:00
diff --git a/.github/workflows/build-repository.yml b/.github/workflows/build-repository.yml
@@ -4,32 +4,54 @@ jobs:
   build-repo:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
       - name: Download artifacts for linux-amd64
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
+        continue-on-error: true
         with:
-          workflow: build-linux-amd64-wheels
+          workflow: build-linux-amd64-wheels.yml
+          workflow_conclusion: ''
+          if_no_artifact_found: warn
+          allow_forks: 'false'
       - name: Download artifacts for linux-aarch64
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
+        continue-on-error: true
         with:
-          workflow: build-linux-aarch64-wheels
+          workflow: build-linux-aarch64-wheels.yml
+          workflow_conclusion: ''
+          if_no_artifact_found: warn
+          allow_forks: 'false'
       - name: Download artifacts for macos-amd64
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
+        continue-on-error: true
         with:
-          workflow: build-macos-amd64-wheels
+          workflow: build-macos-amd64-wheels.yml
+          workflow_conclusion: ''
+          if_no_artifact_found: warn
+          allow_forks: 'false'
       - name: Download artifacts for macos-aarch64
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
+        continue-on-error: true
         with:
-          workflow: build-macos-aarch64-wheels
+          workflow: build-macos-aarch64-wheels.yml
+          workflow_conclusion: ''
+          if_no_artifact_found: warn
+          allow_forks: 'false'
       - name: Download artifacts for windows-amd64
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e
+        continue-on-error: true
         with:
-          workflow: build-windows-amd64-wheels
+          workflow: build-windows-amd64-wheels.yml
+          workflow_conclusion: ''
+          if_no_artifact_found: warn
+          allow_forks: 'false'
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
       - name: Create repository
-        run: python ${GITHUB_WORKSPACE}/generate_repository.py
+        run: python ${GITHUB_WORKSPACE}/scripts/wheelbuilder/generate_repository.py
       - name: Store repository
         uses: umutozd/upload-artifact@5c459179e7745e2c730c50b10a6459da0b6f25db
         with:
diff --git a/scripts/wheelbuilder/generate_repository.py b/scripts/wheelbuilder/generate_repository.py
@@ -48,6 +48,7 @@
 import re
 import shutil
 import sys
+import html
 
 
 def normalize(name):
@@ -67,28 +68,49 @@ def normalize(name):
     wheels = glob.glob(os.path.join(workspace, "**/*.whl")) or glob.glob(
         os.path.join(workspace, "**/**/*.whl.gz__")
     )
-    repo = os.path.join(workspace, "repository")
+    repo = os.path.join(workspace, "repository", "simple")
     os.makedirs(repo, exist_ok=True)
+    pkg_index = [
+        "<html><head><title>Simple Index</title>",
+        "<meta name='api-version' value='2' /></head><body>",
+    ]
+    processed_wheels = set()
     for wheel in wheels:
         print("Processing", wheel)
-        parts = os.path.basename(wheel).split("-")
+        basename = os.path.basename(wheel)
+        parts = basename.split("-")
         for idx, part in enumerate(parts):
-            if part.startswith("graalpy3") or part.startswith("py3"):
+            if part.startswith("graalpy3") or part.startswith("py2") or part.startswith("py3"):
                 version_idx = idx - 1
                 break
+        else:
+            continue
         wheel_name = normalize("-".join(parts[:version_idx]))
         target_dir = os.path.join(repo, wheel_name)
         os.makedirs(target_dir, exist_ok=True)
         if wheel.endswith(".gz__"):
             with gzip.open(wheel, "rb") as f_in:
                 with open(
                     os.path.join(
-                        target_dir, os.path.basename(wheel).replace(".gz__", "")
+                        target_dir, basename.replace(".gz__", "")
                     ),
                     "wb",
                 ) as f_out:
                     shutil.copyfileobj(f_in, f_out)
         else:
             shutil.copy(wheel, target_dir)
 
+        if wheel_name not in processed_wheels:
+            processed_wheels.add(wheel_name)
+            wheel_name = html.escape(wheel_name)
+            pkg_index.append(f"<a href='{wheel_name}/'>{wheel_name}</a><br />")
+
+        with open(os.path.join(target_dir, "index.html"), "a") as f:
+            basename = html.escape(basename)
+            f.write(f"<a href='{basename}'>{basename}</a><br />\n")
+
+    pkg_index.append("</body></html>")
+    with open(os.path.join(repo, "index.html"), "w") as f:
+        f.write("\n".join(pkg_index))
+
     shutil.make_archive(f"{workspace}/repository", "zip", repo)
diff --git a/scripts/wheelbuilder/wheelbuilder/generator.py b/scripts/wheelbuilder/wheelbuilder/generator.py
@@ -117,11 +117,21 @@ def generate(workflow_directory):
             "build-repo": {
                 "runs-on": "ubuntu-latest",
                 "steps": [
+                    {
+                        "name": "Checkout",
+                        "uses": "actions/checkout@v4",
+                    },
+                ]
+                + [
                     {
                         "name": f"Download artifacts for {p.name}-{p.arch}",
-                        "uses": "dawidd6/action-download-artifact@v2",
+                        "uses": "dawidd6/action-download-artifact@268677152d06ba59fcec7a7f0b5d961b6ccd7e1e",
+                        "continue-on-error": True,
                         "with": {
-                            "workflow": f"build-{p.name}-{p.arch}-wheels",
+                            "workflow": f"build-{p.name}-{p.arch}-wheels.yml",
+                            "workflow_conclusion": "",
+                            "if_no_artifact_found": "warn",
+                            "allow_forks": "false",
                         },
                     }
                     for p in PLATFORMS
@@ -136,7 +146,7 @@ def generate(workflow_directory):
                     },
                     {
                         "name": "Create repository",
-                        "run": "python ${GITHUB_WORKSPACE}/generate_repository.py",
+                        "run": "python ${GITHUB_WORKSPACE}/scripts/wheelbuilder/generate_repository.py",
                     },
                     {
                         "name": "Store repository",
