From 9ba7fd623c55a3b8cd0605ed6eb540a458122169 Mon Sep 17 00:00:00 2001 From: DaanDeMeyer Date: Jan 17 2025 14:40:44 +0000 Subject: Merge branch 'releng' into 'c10s-sig-hyperscale' Migrate releng.py from systemd-releng See merge request CentOS/Hyperscale/rpms/systemd!3 --- diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..08da0d2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,17 @@ +# Test locally on Fedora by doing the following: +# - Install gitlab-runner from https://docs.gitlab.com/runner/install/linux-manually.html#using-binary-file +# - sudo dnf install podman +# - systemctl enable --now --user podman.socket +# - sudo ln -s /run/user/$(id -u)/podman/podman.sock /var/run/docker.sock +# - gitlab-runner exec docker systemd-cd --env CENTOS_CERT=/root/.centos.cert --docker-volumes "$HOME/.centos.cert:/root/.centos.cert" +systemd-cd: + stage: build + image: quay.io/fedora/fedora:latest + script: + - env # Show variables that were passed to the job + - dnf install -y tar git-core python3 centos-packager mock mock-centos-sig-configs + - ./releng.py cd --cert $CENTOS_CERT --repo $REPO --release $RELEASE $PUBLISH + parallel: + matrix: + - REPO: [main, facebook] + RELEASE: [9, 10] diff --git a/README.md b/README.md new file mode 100644 index 0000000..15f16d3 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# systemd + +This repo contains the rpm spec for the Hyperscale systemd package and various tools and utilites used for +testing and releasing it. + +# CI + +On merge requests, CI pipelines run without specifying --publish to releng.py. On the main branch, CI +pipelines run with the --publish argument specified to releng.py so that results are only published on the +main branch but pipelines can still run in dry-run mode on merge requests. To make this work every script +that is used in a pipeline should run in dry-run mode by default and accept a --publish option to disable the +dry-run. Use the $PUBLISH variable in the pipeline configuration which will expand to an empty string on +merge requests and to --publish when the job is run as part of the daily scheduled pipeline in gitlab. diff --git a/releng.py b/releng.py new file mode 100755 index 0000000..b04a716 --- /dev/null +++ b/releng.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 + +import argparse +import logging +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, NoReturn, Optional, Sequence, Iterator +import enum +import contextlib +import textwrap + +SYSTEMD_REPO = "https://github.com/systemd/systemd" +AUTHOR = "CentOS Hyperscale SIG " + + +class LogFormatter(logging.Formatter): + def __init__(self, fmt: Optional[str] = None, *args: Any, **kwargs: Any) -> None: + fmt = fmt or "%(message)s" + + bold = "\033[0;1;39m" if sys.stderr.isatty() else "" + gray = "\x1b[38;20m" if sys.stderr.isatty() else "" + red = "\033[31;1m" if sys.stderr.isatty() else "" + yellow = "\033[33;1m" if sys.stderr.isatty() else "" + reset = "\033[0m" if sys.stderr.isatty() else "" + + self.formatters = { + logging.DEBUG: logging.Formatter(f"‣ {gray}{fmt}{reset}"), + logging.INFO: logging.Formatter(f"‣ {fmt}"), + logging.WARNING: logging.Formatter(f"‣ {yellow}{fmt}{reset}"), + logging.ERROR: logging.Formatter(f"‣ {red}{fmt}{reset}"), + logging.CRITICAL: logging.Formatter(f"‣ {red}{bold}{fmt}{reset}"), + } + + super().__init__(fmt, *args, **kwargs) + + def format(self, record: logging.LogRecord) -> str: + return self.formatters[record.levelno].format(record) + + +def run(cmd: Sequence[str], *args: Any, **kwargs: Any) -> subprocess.CompletedProcess: + try: + return subprocess.run(cmd, *args, **kwargs, check=True, text=True) + except FileNotFoundError: + die(f"{cmd[0]} not found in PATH.") + except subprocess.CalledProcessError as e: + logging.error( + f'"{" ".join(str(s) for s in cmd)}" returned non-zero exit code {e.returncode}.' + ) + raise e + + +def die(message: str) -> NoReturn: + logging.error(message) + sys.exit(1) + + +@contextlib.contextmanager +def restore(path: Path) -> Iterator[None]: + old = path.read_text() + try: + yield + finally: + path.write_text(old) + + +def do_cd(args: argparse.Namespace) -> None: + if not Path(".git").exists(): + die("The cd verb must be run from the rpm git repository") + + logging.info("Downloading sources") + run( + [ + "spectool", + "--define", + f"_sourcedir {Path.cwd()}", + "--define", + "branch main", + "--get-files", + "systemd.spec", + ], + ) + + # We can't determine the version dynamically in the spec so we retrieve it + # up front and pass it in via a macro. + version = run( + [ + "tar", + "--gunzip", + "--extract", + "--to-stdout", + "--file=main.tar.gz", + "systemd-main/meson.version", + ], + stdout=subprocess.PIPE, + ).stdout.strip() + + # The timestamp is to ensure the release is always monotonically increasing + rpmrelease = datetime.now().strftime(r"%Y%m%d%H%M%S") + + with restore(Path("systemd.spec")): + Path("systemd.spec").write_text( + textwrap.dedent( + f"""\ + %bcond upstream 1 + %define version_override {version} + %define release_override {rpmrelease} + %define branch main + """ + ) + + Path("systemd.spec").read_text() + ) + + if args.repo == "main": + root = f"centos-stream-hyperscale-{args.release}-x86_64" + else: + root = f"centos-stream-hyperscale-{args.repo}-{args.release}-x86_64" + + logging.info("Building src.rpm") + run( + [ + "mock", + "--root", + root, + "--sources=.", + "--spec=systemd.spec", + "--enable-network", + "--define", + "%_disable_source_fetch 0", + "--buildsrpm", + "--resultdir=.", + ], + ) + + srcrpm = next(Path.cwd().glob("*.src.rpm")) + logging.info(f"Wrote: {srcrpm}") + + run( + [ + "cbs", + *(["--cert", args.cert] if args.cert else []), + "build", + "--wait", + "--fail-fast", + "--skip-tag", + f"hyperscale{args.release}s-packages-{args.repo}-el{args.release}s", + str(srcrpm), + ], + ) + + if not args.publish: + logging.info("Publishing not requested, not tagging builds in testing") + return + + prefix = "hs+fb" if args.repo == "facebook" else "hs" + + run( + [ + "cbs", + *(["--cert", args.cert] if args.cert else []), + "tag-build", + f"hyperscale{args.release}s-packages-{args.repo}-testing", + f"systemd-{version}-{rpmrelease}.{prefix}.el{args.release}", + ] + ) + + +class Verb(enum.Enum): + cd = "cd" + + def __str__(self) -> str: + return self.value + + def run(self, args: argparse.Namespace) -> None: + {Verb.cd: do_cd}[self](args) + + +def main() -> None: + handler = logging.StreamHandler(stream=sys.stderr) + handler.setFormatter(LogFormatter()) + logging.getLogger().addHandler(handler) + logging.getLogger().setLevel("INFO") + + parser = argparse.ArgumentParser() + + parser.add_argument( + "--release", + help="CentOS Stream release to use (e.g 9)", + metavar="RELEASE", + default=9, + choices=[9, 10], + type=int, + ) + parser.add_argument( + "--repo", + help="Hyperscale repository to build against", + choices=["main", "facebook"], + default="main", + ) + parser.add_argument( + "--cert", + help="Path to the CentOS certificate to use", + metavar="PATH", + type=Path, + default=None, + ) + parser.add_argument( + "--publish", + action="store_true", + help="Publish results of operation (by default only a dry-run is done)", + ) + parser.add_argument( + "verb", + type=Verb, + choices=list(Verb), + help=argparse.SUPPRESS, + ) + + args = parser.parse_args() + + if args.cert: + args.cert = args.cert.absolute() + + try: + args.verb.run(args) + except SystemExit as e: + sys.exit(e.code) + except KeyboardInterrupt: + logging.error("Interrupted") + sys.exit(1) + except subprocess.CalledProcessError as e: + sys.exit(e.returncode) + + +if __name__ == "__main__": + main()