diff --git a/.github/workflows/generate-osrepo-site.yml b/.github/workflows/generate-osrepo-site.yml new file mode 100644 index 000000000000..02e1700e2da9 --- /dev/null +++ b/.github/workflows/generate-osrepo-site.yml @@ -0,0 +1,59 @@ +name: generate-site +on: + workflow_dispatch: + +jobs: + gen-site: + runs-on: ubuntu-latest + + name: Generate index and site assets + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Install RPM tools + run: sudo apt install -y rpm gpg createrepo-c dpkg-dev reprepro + + - name: Setup requirements + working-directory: tools/packaging/osrepos + run: pip install -r requirements.txt + + - name: Download packages + working-directory: tools/packaging/osrepos + run: python scripts/fetch-releases.py + + - name: Import GPG key + id: gpg-import + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + + - name: Sign RPMs + shell: sh + working-directory: tools/packaging/osrepos + run: sh scripts/sign-rpms.sh ${{ steps.gpg-import.outputs.fingerprint }} + + - name: Create YUM repository + shell: sh + working-directory: tools/packaging/osrepos + run: createrepo_c -v _site/rpm + + - name: Sign YUM repository + shell: sh + working-directory: tools/packaging/osrepos + run: gpg --armor --detach-sign _site/rpm/repodata/repomd.xml + + - name: Create APT repository + shell: sh + working-directory: tools/packaging/osrepos + run: sh -x scripts/generate-apt-repo.sh + + - name: Prepare assets + working-directory: tools/packaging/osrepos + run: | + cp -aRv dragonfly.repo pgp-key.public dragonfly.sources _site/ + rm -rf _site/deb/conf + + - name: Generate Directory Listings + working-directory: tools/packaging/osrepos + run: python scripts/generate-index.py diff --git a/tools/packaging/osrepos/README.md b/tools/packaging/osrepos/README.md new file mode 100644 index 000000000000..185e02a13160 --- /dev/null +++ b/tools/packaging/osrepos/README.md @@ -0,0 +1,74 @@ +# Package repositories for rpm and debian packages + +This directory contains scripts and definitions for setting up YUM and apt repositories for Linux users to install +dragonfly packages. + +The repositories are served as static websites. The generate-site workflow is used to set up and deploy the sites using +scripts and definitions included here. + +The workflow does the following tasks: + +* Download the latest 5 releases from dragonfly releases page, specifically deb and rpm assets + * for deb files, only the latest package is downloaded and present (see note below) +* Set up a directory structure separating deb and rpm files into version specific paths +* Sign the packages (see note on GPG) +* Deploy the assets prepared, along with the public GPG key and repo definitions for apt and rpm tooling + +## Using the YUM repository + +Add the repository using: + +```shell +sudo dnf config-manager addrepo --from-repofile=https://{placeholder}/dragonfly.repo +``` + +Then install dragonfly as usual, or a specific version: + +```shell +sudo dnf -y install dragonfly-0:v1.33.1-1.fc30.x86_64 +``` + +## Using the APT repository + +First download the public GPG key to an appropriate location: + +```shell +sudo curl -Lo /usr/share/keyrings/dragonfly-keyring.public https://{placeholder}/pgp-key.public +``` + +Then add the sources file: + +```shell +sudo curl -Lo /etc/apt/sources.list.d/dragonfly.sources https://{placeholder}/dragonfly.sources +``` + +Finally install dragonfly using apt + +```shell +sudo apt update && sudo apt install dragonfly +``` + +#### Versions in APT repository + +Unlike the yum repo, the apt repo only has the latest version. The reason for this is the tool, `reprepro` supplied by +debian to build repositories only supports multiple +versions in version 5.4 onwards, and the github runner using ubuntu-latest does not have this version. + +Another option would be to use the components feature of apt repositories in the sources file we ask users to install, +but then the versions would need +to be hardcoded in the sources file and the user would have +to update the file with each new release whcih makes for a bad user experience. As of now users wanting older packages +should download them directly. + +### Signing packages + +The packages are signed using the GPG key imported from the secret GPG_PRIVATE_KEY in this repository. + +The corresponding public key is served with site assets, so the apt/yum/dnf based tooling can consume the public key to +verify package integrity. + +### TODO + +- [X] debian packages signing (not required? release file is signed) +- [X] debian repo metadata setup +- [ ] tests asserting that packages are installable? diff --git a/tools/packaging/osrepos/dragonfly.repo b/tools/packaging/osrepos/dragonfly.repo new file mode 100644 index 000000000000..ea5fd4e83b64 --- /dev/null +++ b/tools/packaging/osrepos/dragonfly.repo @@ -0,0 +1,6 @@ +[dragonfly] +name=Dragonfly Packages +baseurl=https://{placeholder}/rpm/ +enabled=1 +gpgcheck=1 +gpgkey=https://{placeholder}/pgp-key.public diff --git a/tools/packaging/osrepos/dragonfly.sources b/tools/packaging/osrepos/dragonfly.sources new file mode 100644 index 000000000000..9d9dc40f9d75 --- /dev/null +++ b/tools/packaging/osrepos/dragonfly.sources @@ -0,0 +1,5 @@ +Types: deb +URIs: http://{placeholder}/deb +Suites: noble +Components: main +Signed-By: /usr/share/keyrings/dragonfly-keyring.public diff --git a/tools/packaging/osrepos/pgp-key.public b/tools/packaging/osrepos/pgp-key.public new file mode 100644 index 000000000000..7b0ec1b21c95 --- /dev/null +++ b/tools/packaging/osrepos/pgp-key.public @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGjkpygBEADuvzXdOXChr/e4Uh2UBne60NPjmuhpjmArfMfqySeRezJ1Nuvd +AvKNuYRyCw+zsh0Zc/sSANpIdAeKPqrfZJgfEIJI0f8WVjfqsCKi+yWB7Bx0GjQ9 +y/xoFLKkT7p0P/F4yRlb8kQq2KVP9UvcZBETJY96TpQIJM4N3XoG+8DsELW5HYF2 +6sbhgmaNUsxm9oH5UqHcBc7TTgUp10GmZFR4dTeB1IffD/eLMVDMQ8ygzmVxkJPQ +zEKfpFFzseTVyreQlZ5U4GDR8FiB0mY4gZxbCywNqZRycyMM7v4EHuUO0fOgRHdl +5dseF+H1aEG/00JRo6zjiIbgMga0x9wYmVWvTU4wLnGoomukEMCkEQxlil1QjUlK +XI0EltU03DuGki5uhYc9dSS1h74ku2xWePaMsvmxrTphRo1WQBDutzVXSIZ6NBc3 +BN+VBHcumVvif9aRrsfsj2CXhnOB61AW+VWk3fk0evW9cceXZDA0NgGdyeTfS7EI +pioaWtmE3Uv3AfHTlNbMytxG7d7k7oAT2xV6z2IygyQZ5LI1tvSJJ+I5kZHKeruj +k2bFp6H9FGi+g4kA+z9QWgkt+0UXYbjKZAs5Es1uGrRk6o1rAyVTKBKz62F0YQbK +j8Q49Z6iSobaKeQG8naCVkALSM49i4Zpw3x1jUpd7k8/KhpJObq3rewqIQARAQAB +tCREcmFnb25mbHkgPHBhY2thZ2luZ0BkcmFnb25mbHlkYi5pbz6JAlIEEwEKADwW +IQRgvYPC7oTdikxvMGcSMEAYvD0qugUCaOSnKAMbLwQFCwkIBwICIgIGFQoJCAsC +BBYCAwECHgcCF4AACgkQEjBAGLw9KrpGbw//VH2zUjaoSh7SnKGdDOA7A95o2EET +ZvChxImyb6xNKfUoMajPnKcJFg514aPFKLuJl4qJmikxdqBF/bYkznCQSJcLQhsT +pvkqanUh/XwBqbJye1QjBq1o0qXLgeY/Ciz2nqupwLQdzvGHO6+2Yk04T89pnZEo +CDSoZKkacu8TpalStqzqDlumryXZzdZ35hAu9OT0fVc2wtcMiY3pznLG1iawNk8I +bzme0ezGA/fk7xEptEbGlb1OtUV5+iG/SFEVvic8GTNf1yLQNCVK3QzD1ciL3MzR +OTH8a04ov2bMxjl8bIefKE/dFBeCSKbvkfTSMAEgqUAuRp7gvoO7uHO05A5AHU2i +y4agskGkgQR9u1yqUXyYIM9kkpuUqqAkwRqg1pw55LG686Xe35QYH4zbpgvr45/Q +JRPFjCbLzR1ZcNyrecHgrq2M9WNlk6dtdWBSJuc7L0M8KJqfrPxQmMpMm/KR43Ey +um0FCgb2J+ceO2W4GrE/DHHoNTt2iio2gMcmRXM7XTmVupsigbYk7AqGncLIQ60B +94jtv16ggXIeA5sPqmyssARXtweTM+EzLLs4K79be4K5j/yyg3CxxvZcq5CZNwoi +fbQgGVNb4SS+nv2r1mVe9XNSonmVVrAqSIFpptH5ahqgaRDUnmy0Lzk7qiHv02OW +PjbSiwQGHDHwq98= +=SOT5 +-----END PGP PUBLIC KEY BLOCK----- diff --git a/tools/packaging/osrepos/reprepro-config/distributions b/tools/packaging/osrepos/reprepro-config/distributions new file mode 100644 index 000000000000..ce2da4fe7570 --- /dev/null +++ b/tools/packaging/osrepos/reprepro-config/distributions @@ -0,0 +1,8 @@ +Codename: noble +Suite: stable +Architectures: amd64 arm64 +Components: main +Origin: Dragonfly +Label: Dragonfly +Description: Dragonfly APT repository +SignWith: 60BD83C2EE84DD8A4C6F306712304018BC3D2ABA diff --git a/tools/packaging/osrepos/reprepro-config/options b/tools/packaging/osrepos/reprepro-config/options new file mode 100644 index 000000000000..c00ee94a5a35 --- /dev/null +++ b/tools/packaging/osrepos/reprepro-config/options @@ -0,0 +1 @@ +verbose diff --git a/tools/packaging/osrepos/requirements.txt b/tools/packaging/osrepos/requirements.txt new file mode 100644 index 000000000000..807763edbca8 --- /dev/null +++ b/tools/packaging/osrepos/requirements.txt @@ -0,0 +1,5 @@ +certifi>=2025.10.5 +charset-normalizer>=3.4.3 +idna>=3.10 +requests>=2.32.5 +urllib3>=2.5.0 diff --git a/tools/packaging/osrepos/scripts/fetch-releases.py b/tools/packaging/osrepos/scripts/fetch-releases.py new file mode 100644 index 000000000000..b709a66bfcbf --- /dev/null +++ b/tools/packaging/osrepos/scripts/fetch-releases.py @@ -0,0 +1,92 @@ +import dataclasses +import enum +import os.path +import time + +import requests + +RELEASE_URL = "https://api.github.com/repos/dragonflydb/dragonfly/releases" + + +class AssetKind(enum.Enum): + RPM = 1 + DEB = 2 + + +@dataclasses.dataclass +class Package: + kind: AssetKind + download_url: str + version: str + filename: str + arch: str + + @staticmethod + def from_url(url: str) -> "Package": + tokens = url.split("/") + filename = tokens[-1] + kind = AssetKind.RPM if filename.endswith(".rpm") else AssetKind.DEB + if kind == AssetKind.DEB: + arch = filename.split(".")[0].split("_")[1] + else: + arch = filename.split(".")[1] + return Package( + kind=kind, download_url=url, version=tokens[-2], filename=filename, arch=arch + ) + + def storage_path(self, root: str) -> str: + match self.kind: + case AssetKind.RPM: + return os.path.join(root, "rpm", self.version) + case AssetKind.DEB: + return os.path.join("deb_tmp", self.arch, self.version) + + +def collect_download_urls() -> list[Package]: + packages = [] + # TODO retry logic + response = requests.get(RELEASE_URL) + releases = response.json() + for release in releases[:5]: + for asset in release["assets"]: + if asset["name"].endswith(".rpm") or asset["name"].endswith(".deb"): + packages.append(Package.from_url(asset["browser_download_url"])) + return packages + + +def download_packages(root: str, packages: list[Package]): + # The debian repository building tool, reprepo, only supports a single package per version by default. + # The ability to support multiple versions has been added but is not present in ubuntu-latest on + # github action runners yet. So we only download one package, the latest, for ubuntu. + # The rest of the scripts work on a set of packages, so that when the Limit parameter is supported, + # we can remove this flag and start hosting more than the latest versions. + # Another alternative would be to use the components feature of reprepo, but it would involve updating + # the repository definition itself for each release, which is a bad experience for end users. + deb_done = False + for package in packages: + if package.kind == AssetKind.DEB and deb_done: + continue + + print(f"Downloading {package.download_url}") + path = package.storage_path(root) + if not os.path.exists(path): + os.makedirs(path) + + target = os.path.join(path, package.filename) + # TODO retry logic + response = requests.get(package.download_url) + with open(target, "wb") as f: + f.write(response.content) + print(f"Downloaded {package.download_url}") + time.sleep(0.5) + if package.kind == AssetKind.DEB: + deb_done = True + + +def main(): + packages = collect_download_urls() + download_packages("_site", packages) + + +if __name__ == "__main__": + main() diff --git a/tools/packaging/osrepos/scripts/generate-apt-repo.sh b/tools/packaging/osrepos/scripts/generate-apt-repo.sh new file mode 100644 index 000000000000..dc64934656fb --- /dev/null +++ b/tools/packaging/osrepos/scripts/generate-apt-repo.sh @@ -0,0 +1,15 @@ +set -e + +METADATA_ROOT=_site/deb +mkdir -pv ${METADATA_ROOT}/conf + +cp -av reprepro-config/* ${METADATA_ROOT}/conf + +reprepro -b ${METADATA_ROOT} createsymlinks +reprepro -b ${METADATA_ROOT} export + +for file in $(find deb_tmp -type f -name "*.deb"); do + reprepro -b ${METADATA_ROOT} includedeb noble "${file}" +done + +rm -rf deb_tmp diff --git a/tools/packaging/osrepos/scripts/generate-index.py b/tools/packaging/osrepos/scripts/generate-index.py new file mode 100644 index 000000000000..360f59c93b88 --- /dev/null +++ b/tools/packaging/osrepos/scripts/generate-index.py @@ -0,0 +1,32 @@ +import os.path + +HEADER = """ + +
+""" + +FOOTER = """ + +""" + + +def build_index(dirpath): + print(f"building index.html for {dirpath}") + target = os.path.join(dirpath, "index.html") + with open(target, "w") as f: + f.write(HEADER.format(dir=dirpath)) + for item in sorted(os.listdir(dirpath)): + if item == "index.html": + continue + name = item + "/" if os.path.isdir(os.path.join(dirpath, item)) else item + f.write(f"""{name}