[WIP] tools: Add scripts and supporting files to generate os packge repo

.github/workflows/generate-osrepo-site.yml

@@ -0,0 +1,59 @@
+name: generate-site
+on:
+  workflow_dispatch:
+
+jobs:
+  gen-site:
+    runs-on: ubuntu-latest
+
+    name: Generate index and site assets
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Install RPM tools
+        run: sudo apt install -y rpm gpg createrepo-c dpkg-dev reprepro
+
+      - name: Setup requirements
+        working-directory: tools/packaging/osrepos
+        run: pip install -r requirements.txt
+
+      - name: Download packages
+        working-directory: tools/packaging/osrepos
+        run: python scripts/fetch-releases.py
+
+      - name: Import GPG key
+        id: gpg-import
+        uses: crazy-max/ghaction-import-gpg@v6
+        with:
+          gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+
+      - name: Sign RPMs
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: sh scripts/sign-rpms.sh ${{ steps.gpg-import.outputs.fingerprint }}
+
+      - name: Create YUM repository
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: createrepo_c -v _site/rpm
+
+      - name: Sign YUM repository
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: gpg --armor --detach-sign _site/rpm/repodata/repomd.xml
+
+      - name: Create APT repository
+        shell: sh
+        working-directory: tools/packaging/osrepos
+        run: sh -x scripts/generate-apt-repo.sh
+
+      - name: Prepare assets
+        working-directory: tools/packaging/osrepos
+        run: |
+          cp -aRv dragonfly.repo pgp-key.public dragonfly.sources _site/
+          rm -rf _site/deb/conf
+
+      - name: Generate Directory Listings
+        working-directory: tools/packaging/osrepos
+        run: python scripts/generate-index.py

tools/packaging/osrepos/README.md

@@ -0,0 +1,74 @@
+# Package repositories for rpm and debian packages
+
+This directory contains scripts and definitions for setting up YUM and apt repositories for Linux users to install
+dragonfly packages.
+
+The repositories are served as static websites. The generate-site workflow is used to set up and deploy the sites using
+scripts and definitions included here.
+
+The workflow does the following tasks:
+
+* Download the latest 5 releases from dragonfly releases page, specifically deb and rpm assets
+    * for deb files, only the latest package is downloaded and present (see note below)
+* Set up a directory structure separating deb and rpm files into version specific paths
+* Sign the packages (see note on GPG)
+* Deploy the assets prepared, along with the public GPG key and repo definitions for apt and rpm tooling
+
+## Using the YUM repository
+
+Add the repository using:
+
+```shell
+sudo dnf config-manager addrepo --from-repofile=https://{placeholder}/dragonfly.repo
+```
+
+Then install dragonfly as usual, or a specific version:
+
+```shell
+sudo dnf -y install dragonfly-0:v1.33.1-1.fc30.x86_64
+```
+
+## Using the APT repository
+
+First download the public GPG key to an appropriate location:
+
+```shell
+sudo curl -Lo /usr/share/keyrings/dragonfly-keyring.public https://{placeholder}/pgp-key.public
+```
+
+Then add the sources file:
+
+```shell
+sudo curl -Lo /etc/apt/sources.list.d/dragonfly.sources https://{placeholder}/dragonfly.sources
+```
+
+Finally install dragonfly using apt
+
+```shell
+sudo apt update && sudo apt install dragonfly
+```
+
+#### Versions in APT repository
+
+Unlike the yum repo, the apt repo only has the latest version. The reason for this is the tool, `reprepro` supplied by
+debian to build repositories only supports multiple
+versions in version 5.4 onwards, and the github runner using ubuntu-latest does not have this version.
+
+Another option would be to use the components feature of apt repositories in the sources file we ask users to install,
+but then the versions would need
+to be hardcoded in the sources file and the user would have
+to update the file with each new release whcih makes for a bad user experience. As of now users wanting older packages
+should download them directly.
+
+### Signing packages
+
+The packages are signed using the GPG key imported from the secret GPG_PRIVATE_KEY in this repository.
+
+The corresponding public key is served with site assets, so the apt/yum/dnf based tooling can consume the public key to
+verify package integrity.
+
+### TODO
+
+- [X] debian packages signing (not required? release file is signed)
+- [X] debian repo metadata setup
+- [ ] tests asserting that packages are installable?

tools/packaging/osrepos/dragonfly.repo

@@ -0,0 +1,6 @@
+[dragonfly]
+name=Dragonfly Packages
+baseurl=https://{placeholder}/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://{placeholder}/pgp-key.public

tools/packaging/osrepos/dragonfly.sources

@@ -0,0 +1,5 @@
+Types: deb
+URIs: http://{placeholder}/deb
+Suites: noble
+Components: main
+Signed-By: /usr/share/keyrings/dragonfly-keyring.public

tools/packaging/osrepos/pgp-key.public

@@ -0,0 +1,29 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBGjkpygBEADuvzXdOXChr/e4Uh2UBne60NPjmuhpjmArfMfqySeRezJ1Nuvd
+AvKNuYRyCw+zsh0Zc/sSANpIdAeKPqrfZJgfEIJI0f8WVjfqsCKi+yWB7Bx0GjQ9
+y/xoFLKkT7p0P/F4yRlb8kQq2KVP9UvcZBETJY96TpQIJM4N3XoG+8DsELW5HYF2
+6sbhgmaNUsxm9oH5UqHcBc7TTgUp10GmZFR4dTeB1IffD/eLMVDMQ8ygzmVxkJPQ
+zEKfpFFzseTVyreQlZ5U4GDR8FiB0mY4gZxbCywNqZRycyMM7v4EHuUO0fOgRHdl
+5dseF+H1aEG/00JRo6zjiIbgMga0x9wYmVWvTU4wLnGoomukEMCkEQxlil1QjUlK
+XI0EltU03DuGki5uhYc9dSS1h74ku2xWePaMsvmxrTphRo1WQBDutzVXSIZ6NBc3
+BN+VBHcumVvif9aRrsfsj2CXhnOB61AW+VWk3fk0evW9cceXZDA0NgGdyeTfS7EI
+pioaWtmE3Uv3AfHTlNbMytxG7d7k7oAT2xV6z2IygyQZ5LI1tvSJJ+I5kZHKeruj
+k2bFp6H9FGi+g4kA+z9QWgkt+0UXYbjKZAs5Es1uGrRk6o1rAyVTKBKz62F0YQbK
+j8Q49Z6iSobaKeQG8naCVkALSM49i4Zpw3x1jUpd7k8/KhpJObq3rewqIQARAQAB
+tCREcmFnb25mbHkgPHBhY2thZ2luZ0BkcmFnb25mbHlkYi5pbz6JAlIEEwEKADwW
+IQRgvYPC7oTdikxvMGcSMEAYvD0qugUCaOSnKAMbLwQFCwkIBwICIgIGFQoJCAsC
+BBYCAwECHgcCF4AACgkQEjBAGLw9KrpGbw//VH2zUjaoSh7SnKGdDOA7A95o2EET
+ZvChxImyb6xNKfUoMajPnKcJFg514aPFKLuJl4qJmikxdqBF/bYkznCQSJcLQhsT
+pvkqanUh/XwBqbJye1QjBq1o0qXLgeY/Ciz2nqupwLQdzvGHO6+2Yk04T89pnZEo
+CDSoZKkacu8TpalStqzqDlumryXZzdZ35hAu9OT0fVc2wtcMiY3pznLG1iawNk8I
+bzme0ezGA/fk7xEptEbGlb1OtUV5+iG/SFEVvic8GTNf1yLQNCVK3QzD1ciL3MzR
+OTH8a04ov2bMxjl8bIefKE/dFBeCSKbvkfTSMAEgqUAuRp7gvoO7uHO05A5AHU2i
+y4agskGkgQR9u1yqUXyYIM9kkpuUqqAkwRqg1pw55LG686Xe35QYH4zbpgvr45/Q
+JRPFjCbLzR1ZcNyrecHgrq2M9WNlk6dtdWBSJuc7L0M8KJqfrPxQmMpMm/KR43Ey
+um0FCgb2J+ceO2W4GrE/DHHoNTt2iio2gMcmRXM7XTmVupsigbYk7AqGncLIQ60B
+94jtv16ggXIeA5sPqmyssARXtweTM+EzLLs4K79be4K5j/yyg3CxxvZcq5CZNwoi
+fbQgGVNb4SS+nv2r1mVe9XNSonmVVrAqSIFpptH5ahqgaRDUnmy0Lzk7qiHv02OW
+PjbSiwQGHDHwq98=
+=SOT5
+-----END PGP PUBLIC KEY BLOCK-----

tools/packaging/osrepos/reprepro-config/distributions

@@ -0,0 +1,8 @@
+Codename: noble
+Suite: stable
+Architectures: amd64 arm64
+Components: main
+Origin: Dragonfly
+Label: Dragonfly
+Description: Dragonfly APT repository
+SignWith: 60BD83C2EE84DD8A4C6F306712304018BC3D2ABA

tools/packaging/osrepos/reprepro-config/options

@@ -0,0 +1 @@
+verbose

tools/packaging/osrepos/requirements.txt

@@ -0,0 +1,5 @@
+certifi>=2025.10.5
+charset-normalizer>=3.4.3
+idna>=3.10
+requests>=2.32.5
+urllib3>=2.5.0

tools/packaging/osrepos/scripts/fetch-releases.py

@@ -0,0 +1,92 @@
+import dataclasses
+import enum
+import os.path
+import time
+
+import requests
+
+RELEASE_URL = "https://api.github.com/repos/dragonflydb/dragonfly/releases"
+
+
+class AssetKind(enum.Enum):
+    RPM = 1
+    DEB = 2
+
+
+@dataclasses.dataclass
+class Package:
+    kind: AssetKind
+    download_url: str
+    version: str
+    filename: str
+    arch: str
+
+    @staticmethod
+    def from_url(url: str) -> "Package":
+        tokens = url.split("/")
+        filename = tokens[-1]
+        kind = AssetKind.RPM if filename.endswith(".rpm") else AssetKind.DEB
+        if kind == AssetKind.DEB:
+            arch = filename.split(".")[0].split("_")[1]
+        else:
+            arch = filename.split(".")[1]
+        return Package(
+            kind=kind, download_url=url, version=tokens[-2], filename=filename, arch=arch
+        )
+
+    def storage_path(self, root: str) -> str:
+        match self.kind:
+            case AssetKind.RPM:
+                return os.path.join(root, "rpm", self.version)
+            case AssetKind.DEB:
+                return os.path.join("deb_tmp", self.arch, self.version)
+
+
+def collect_download_urls() -> list[Package]:
+    packages = []
+    # TODO retry logic
+    response = requests.get(RELEASE_URL)
+    releases = response.json()
+    for release in releases[:5]:
+        for asset in release["assets"]:
+            if asset["name"].endswith(".rpm") or asset["name"].endswith(".deb"):
+                packages.append(Package.from_url(asset["browser_download_url"]))
+    return packages
+
+
+def download_packages(root: str, packages: list[Package]):
+    # The debian repository building tool, reprepo, only supports a single package per version by default.
+    # The ability to support multiple versions has been added but is not present in ubuntu-latest on
+    # github action runners yet. So we only download one package, the latest, for ubuntu.
+    # The rest of the scripts work on a set of packages, so that when the Limit parameter is supported,
+    # we can remove this flag and start hosting more than the latest versions.
+    # Another alternative would be to use the components feature of reprepo, but it would involve updating
+    # the repository definition itself for each release, which is a bad experience for end users.
+    deb_done = False
+    for package in packages:
+        if package.kind == AssetKind.DEB and deb_done:
+            continue
+
+        print(f"Downloading {package.download_url}")
+        path = package.storage_path(root)
+        if not os.path.exists(path):
+            os.makedirs(path)
+
+        target = os.path.join(path, package.filename)
+        # TODO retry logic
+        response = requests.get(package.download_url)
+        with open(target, "wb") as f:
+            f.write(response.content)
+        print(f"Downloaded {package.download_url}")
+        time.sleep(0.5)
+        if package.kind == AssetKind.DEB:
+            deb_done = True
+
+
+def main():
+    packages = collect_download_urls()
+    download_packages("_site", packages)
+
+
+if __name__ == "__main__":
+    main()

tools/packaging/osrepos/scripts/generate-apt-repo.sh

@@ -0,0 +1,15 @@
+set -e
+
+METADATA_ROOT=_site/deb
+mkdir -pv ${METADATA_ROOT}/conf
+
+cp -av reprepro-config/* ${METADATA_ROOT}/conf
+
+reprepro -b ${METADATA_ROOT} createsymlinks
+reprepro -b ${METADATA_ROOT} export
+
+for file in $(find deb_tmp -type f -name "*.deb"); do
+  reprepro -b ${METADATA_ROOT} includedeb noble "${file}"
+done
+
+rm -rf deb_tmp

tools/packaging/osrepos/scripts/generate-index.py

@@ -0,0 +1,32 @@
+import os.path
+
+HEADER = """<!DOCTYPE html>
+<html>
+<body>
+"""
+
+FOOTER = """</body>
+</html>
+"""
+
+
+def build_index(dirpath):
+    print(f"building index.html for {dirpath}")
+    target = os.path.join(dirpath, "index.html")
+    with open(target, "w") as f:
+        f.write(HEADER.format(dir=dirpath))
+        for item in sorted(os.listdir(dirpath)):
+            if item == "index.html":
+                continue
+            name = item + "/" if os.path.isdir(os.path.join(dirpath, item)) else item
+            f.write(f"""<a href="{item}">{name}</a><br>\n""")
+        f.write(FOOTER)
+
+
+def recurse_dir(root):
+    for root, dirs, _ in os.walk(root):
+        build_index(root)
+
+
+if __name__ == "__main__":
+    recurse_dir("_site")

tools/packaging/osrepos/scripts/sign-rpms.sh

@@ -0,0 +1,13 @@
+set -e
+
+# GPG key must have been imported
+
+echo "Signing RPMs with key id ${1}"
+
+GPG_TTY=""
+export GPG_TTY
+
+for file in $(find _site/rpm -type f -name "*.rpm"); do
+  echo "Signing ${file}"
+  rpm --define "__gpg /usr/bin/gpg" --define "%_signature gpg" --define "%_gpg_name ${1}" --addsign "${file}"
+done