master.cfg

# -*- python -*-
# ex: set filetype=python:

import os
from collections import defaultdict

import yaml

from buildbot.plugins import util
from common_factories import (
    getQuickBuildFactory,
    getDebAutobakeFactory,
    getRpmAutobakeFactory,
)
from constants import (
    GITHUB_STATUS_BUILDERS,
)
from locks import getLocks
from master_common import base_master_config
from utils import (
    canStartBuild,
    createWorker,
    nextBuild,
)

cfg_dir = os.path.abspath(os.path.dirname(__file__))
# Autogen master, see buildbot.tac for why this is the case.
base_dir = os.path.abspath(f"{cfg_dir}/../../")
with open(os.path.join(cfg_dir, "master-config.yaml"), "r") as file:
    master_config = yaml.safe_load(file)


# Load the slave, database passwords and 3rd-party tokens from an external
# private file, so that the rest of the configuration can be public.
config = {"private": {}}
with open(os.path.join(cfg_dir, "master-private.cfg"), "r") as file:
    exec(file.read(), config, {})

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = base_master_config(config)


#######
# DB URL
#######
mtrDbPool = util.EqConnectionPool(
    "MySQLdb",
    config["private"]["db_host"],
    config["private"]["db_user"],
    config["private"]["db_password"],
    config["private"]["db_mtr_db"],
)

#########
# WORKERS
#########

# The 'workers' list defines the set of recognized workers. Each element is
# a Worker object, specifying a unique worker name and password.  The same
# worker name and password must be configured on the worker.
c["workers"] = []

# Docker workers

workers = defaultdict(list)

# For each worker in master_config ['aarch64-bbw1', 2, 3, 4]
for w_name in master_config["workers"]:
    jobs = 7
    worker_name = w_name[:-1]  # aarch64-bbw
    worker_id = w_name[-1]     # 1, 2, 3, 4

    for arch in master_config["builders"]:
        builders = master_config["builders"][arch]
        for os_name, os_definition in builders.items():
            image_tag = os_definition['image_tag']

            # Skip s390x non-SLES builders on SLES host (bbw2)
            if ("s390x" in arch
                    and (worker_id == "2")
                    and ("sles" not in os_name)):
                continue

            quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}'
            if arch.startswith("x86"):
                os_name += "-i386"
                quay_name += "-386"

            base_name, name, worker_instance = createWorker(
                worker_name,
                worker_id,
                os_name,
                quay_name,
                jobs=jobs,
                save_packages=True,
                shm_size="15G",
            )

            workers[base_name].append(name)
            c["workers"].append(worker_instance)


####### FACTORY CODE

f_quick_build = getQuickBuildFactory("nm", mtrDbPool)
f_rpm_autobake = getRpmAutobakeFactory()
f_deb_autobake = getDebAutobakeFactory()


####### BUILDERS LIST
c["builders"] = []

for arch in master_config["builders"]:
    worker_prefix = "x64" if arch in ["amd64", "x86"] else arch
    worker_suffix = "-i386" if arch == "x86" else ""

    builders_group = master_config["builders"][arch]
    for os_name in builders_group:
        worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}'
        builder = f'{arch}-{os_name}'
        build_type = builders_group[os_name]["type"]

        tags = builders_group[os_name]["tags"]
        # For easier searching in Builders tab for all sub-builders, like
        # autobake, install, upgrade from the same OS.
        tags.append(os_name)
        # Add builder only if it's not a protected branches one
        if builder not in GITHUB_STATUS_BUILDERS:
            c["builders"].append(
                util.BuilderConfig(
                    name=builder,
                    workernames=workers[worker_name],
                    tags=tags,
                    collapseRequests=True,
                    nextBuild=nextBuild,
                    canStartBuild=canStartBuild,
                    locks=getLocks,
                    factory=f_quick_build,
                )
            )

        factory_instance = (
            f_deb_autobake if build_type != "rpm" else f_rpm_autobake
        )
        properties = {
            "verbose_build": "VERBOSE=1" if arch == "ppc64le" else "",
            "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else ""
        }

        autobake_tags = tags + [build_type, "autobake"]

        c["builders"].append(
            util.BuilderConfig(
                name=f"{builder}-{build_type}-autobake",
                workernames=workers[worker_name],
                tags=autobake_tags,
                collapseRequests=True,
                nextBuild=nextBuild,
                canStartBuild=canStartBuild,
                locks=getLocks,
                properties=properties,
                factory=factory_instance,
            )
        )