diff --git a/pmb/ci/__init__.py b/pmb/ci/__init__.py new file mode 100644 index 00000000..98b6022a --- /dev/null +++ b/pmb/ci/__init__.py @@ -0,0 +1,164 @@ +# Copyright 2022 Oliver Smith +# SPDX-License-Identifier: GPL-3.0-or-later +import collections +import glob +import logging +import os +import shlex +import pmb.chroot +import pmb.helpers.cli + + +def get_ci_scripts(topdir): + """ Find 'pmbootstrap ci'-compatible scripts inside a git repository, and + parse their metadata (description, options). The reference is at: + https://postmarketos.org/pmb-ci + :param topdir: top directory of the git repository, get it with: + pmb.helpers.git.get_topdir() + :returns: a dict of CI scripts found in the git repository, e.g. + {"flake8": {"description": "lint all python scripts", + "options": []}, + ...} """ + ret = {} + for script in glob.glob(f"{topdir}/.ci/*.sh"): + is_pmb_ci_script = False + description = "" + options = [] + + with open(script) as handle: + for line in handle: + if line.startswith("# https://postmarktos.org/pmb-ci"): + is_pmb_ci_script = True + elif line.startswith("# Description: "): + description = line.split(": ", 1)[1].rstrip() + elif line.startswith("# Options: "): + options = line.split(": ", 1)[1].rstrip().split(" ") + elif not line.startswith("#"): + # Stop parsing after the block of comments on top + break + + if not is_pmb_ci_script: + continue + + if not description: + logging.error(f"ERROR: {script}: missing '# Description: …' line") + exit(1) + + for option in options: + if option not in pmb.config.ci_valid_options: + raise RuntimeError(f"{script}: unsupported option '{option}'." + " Typo in script or pmbootstrap too old?") + + short_name = os.path.basename(script).split(".", -1)[0] + ret[short_name] = {"description": description, + "options": options} + return ret + + +def sort_scripts_by_speed(scripts): + """ Order the scripts, so fast scripts run before slow scripts. Whether a + script is fast or not is determined by the '# Options: slow' comment in + the file. + :param scripts: return of get_ci_scripts() + :returns: same format as get_ci_scripts(), but as ordered dict with + fast scripts before slow scripts """ + ret = collections.OrderedDict() + + # Fast scripts first + for script_name, script in scripts.items(): + if "slow" in script["options"]: + continue + ret[script_name] = script + + # Then slow scripts + for script_name, script in scripts.items(): + if "slow" not in script["options"]: + continue + ret[script_name] = script + return ret + + +def ask_which_scripts_to_run(scripts_available): + """ Display an interactive prompt about which of the scripts the user + wishes to run, or all of them. + :param scripts_available: same format as get_ci_scripts() + :returns: either full scripts_available (all selected), or a subset """ + count = len(scripts_available.items()) + choices = ["all"] + + logging.info(f"Available CI scripts ({count}):") + for script_name, script in scripts_available.items(): + extra = "" + if "slow" in script["options"]: + extra += " (slow)" + logging.info(f"* {script_name}: {script['description']}{extra}") + choices += [script_name] + + selection = pmb.helpers.cli.ask("Which script?", None, "all", + complete=choices) + if selection == "all": + return scripts_available + + ret = {} + ret[selection] = scripts_available[selection] + return ret + + +def copy_git_repo_to_chroot(args, topdir): + """ Create a tarball of the git repo (including unstaged changes and new + files) and extract it in chroot_native. + :param topdir: top directory of the git repository, get it with: + pmb.helpers.git.get_topdir() """ + pmb.chroot.init(args) + tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz" + + cmd = "(git ls-files && git ls-files --exclude-standard --other)" + cmd += f" | tar -cf {shlex.quote(tarball_path)} -T -" + pmb.helpers.run.user(args, ["sh", "-c", cmd], topdir) + + ci_dir = "/home/pmos/ci" + pmb.chroot.user(args, ["rm", "-rf", ci_dir]) + pmb.chroot.user(args, ["mkdir", ci_dir]) + pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"], + working_dir=ci_dir) + + +def run_scripts(args, topdir, scripts): + """ Run one of the given scripts after another, either natively or in a + chroot. Display a progress message and stop on error (without printing + a python stack trace). + :param topdir: top directory of the git repository, get it with: + pmb.helpers.git.get_topdir() + :param scripts: return of get_ci_scripts() """ + steps = len(scripts) + step = 0 + repo_copied = False + + for script_name, script in scripts.items(): + step += 1 + + where = "pmbootstrap chroot" + if "native" in script["options"]: + where = "native" + + script_path = f".ci/{script_name}.sh" + logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}" + f" [{where}] ***") + + if "native" in script["options"]: + rc = pmb.helpers.run.user(args, [script_path], topdir, + output="tui") + continue + else: + # Run inside pmbootstrap chroot + if not repo_copied: + copy_git_repo_to_chroot(args, topdir) + repo_copied = True + + env = {"TESTUSER": "pmos"} + rc = pmb.chroot.root(args, [script_path], check=False, env=env, + working_dir="/home/pmos/ci", + output="tui") + if rc: + logging.error(f"ERROR: CI script failed: {script_name}") + exit(1) diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index bf4de67a..8fcd50d2 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -1014,3 +1014,9 @@ upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*", # SIDELOAD # sideload_sudo_prompt = "[sudo] password for %u@%h: " + +# +# CI +# +# Valid options für 'pmbootstrap ci', see https://postmarketos.org/pmb-ci +ci_valid_options = ["native", "slow"] diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index c2222ec9..abb8961d 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -12,6 +12,7 @@ import pmb.build.autodetect import pmb.chroot import pmb.chroot.initfs import pmb.chroot.other +import pmb.ci import pmb.config import pmb.export import pmb.flasher @@ -623,3 +624,39 @@ def lint(args): def status(args): if not pmb.helpers.status.print_status(args, args.details): sys.exit(1) + + +def ci(args): + topdir = pmb.helpers.git.get_topdir(args, os.getcwd()) + if not os.path.exists(topdir): + logging.error("ERROR: change your current directory to a git" + " repository (e.g. pmbootstrap, pmaports) before running" + " 'pmbootstrap ci'.") + exit(1) + + scripts_available = pmb.ci.get_ci_scripts(topdir) + scripts_available = pmb.ci.sort_scripts_by_speed(scripts_available) + if not scripts_available: + logging.error("ERROR: no supported CI scripts found in current git" + " repository, see https://postmarketos.org/pmb-ci") + exit(1) + + scripts_selected = {} + if args.scripts: + for script in args.scripts: + if script not in scripts_available: + logging.error(f"ERROR: script '{script}' not found in git" + " repository, found these:" + f" {', '.join(scripts_available.keys())}") + exit(1) + scripts_selected[script] = scripts_available[script] + elif args.all: + scripts_selected = scripts_available + + if not pmb.helpers.git.clean_worktree(args, topdir): + logging.warning("WARNING: this git repository has uncommitted changes") + + if not scripts_selected: + scripts_selected = pmb.ci.ask_which_scripts_to_run(scripts_available) + + pmb.ci.run_scripts(args, topdir, scripts_selected) diff --git a/pmb/helpers/git.py b/pmb/helpers/git.py index 633e7f8a..d33ecf1b 100644 --- a/pmb/helpers/git.py +++ b/pmb/helpers/git.py @@ -246,3 +246,10 @@ def is_outdated(path): date_outdated = time.time() - pmb.config.git_repo_outdated return date_head <= date_outdated + + +def get_topdir(args, path): + """ :returns: a string with the top dir of the git repository, or an + empty string if it's not a git repository. """ + return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"], + path, output_return=True, check=False).rstrip() diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index b53609fb..b2e32a39 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -546,6 +546,19 @@ def arguments_netboot(subparser): return ret +def arguments_ci(subparser): + ret = subparser.add_parser("ci", help="run continuous integration scripts" + " locally of git repo in current" + " directory") + ret.add_argument("-a", "--all", action="store_true", + help="name of the CI script to run, depending on the git" + " repository") + ret.add_argument("scripts", nargs="*", metavar="script", + help="name of the CI script to run, depending on the git" + " repository") + return ret + + def package_completer(prefix, action, parser=None, parsed_args=None): args = parsed_args pmb.config.merge_with_args(args) @@ -693,6 +706,7 @@ def arguments(): arguments_newapkbuild(sub) arguments_lint(sub) arguments_status(sub) + arguments_ci(sub) # Action: log log = sub.add_parser("log", help="follow the pmbootstrap logfile")