pmbootstrap ci: new command
Add a new command that makes running CI scripts easy. The user goes to the git repository of choice, which has CI scripts written in a certain format, and then runs 'pmbootstrap ci' to get an interactive selection of which of the available scripts to run (or "all"). Specifying one or multiple scripts on the command-line is also possible, e.g. $ pmbootstrap ci flake8 $ pmbootstrap ci shellcheck flake8 pytest $ pmbootstrap ci --all pmbootstrap then either runs the selected scripts in a chroot (and installs dependencies as defined at the beginning of the CI scripts), or natively (with checks inside the scripts for having dependencies installed). Running natively is needed for .ci/pytest.sh in this pmbootstrap.git repository, as pmbootstrap can't run inside pmbootstrap. Running natively or in chroot is defined in an "# Options: " comment inside the script file. Documentation for this command and how script files look like: https://postmarketos.org/pmb-ci
This commit is contained in:
Oliver Smith
parent
6ac39d17e7
commit
0bcd58765c
|
|
@ -0,0 +1,164 @@
|
|||
# Copyright 2022 Oliver Smith
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
import collections
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
import pmb.chroot
|
||||
import pmb.helpers.cli
|
||||
|
||||
|
||||
def get_ci_scripts(topdir):
|
||||
""" Find 'pmbootstrap ci'-compatible scripts inside a git repository, and
|
||||
parse their metadata (description, options). The reference is at:
|
||||
https://postmarketos.org/pmb-ci
|
||||
:param topdir: top directory of the git repository, get it with:
|
||||
pmb.helpers.git.get_topdir()
|
||||
:returns: a dict of CI scripts found in the git repository, e.g.
|
||||
{"flake8": {"description": "lint all python scripts",
|
||||
"options": []},
|
||||
...} """
|
||||
ret = {}
|
||||
for script in glob.glob(f"{topdir}/.ci/*.sh"):
|
||||
is_pmb_ci_script = False
|
||||
description = ""
|
||||
options = []
|
||||
|
||||
with open(script) as handle:
|
||||
for line in handle:
|
||||
if line.startswith("# https://postmarktos.org/pmb-ci"):
|
||||
is_pmb_ci_script = True
|
||||
elif line.startswith("# Description: "):
|
||||
description = line.split(": ", 1)[1].rstrip()
|
||||
elif line.startswith("# Options: "):
|
||||
options = line.split(": ", 1)[1].rstrip().split(" ")
|
||||
elif not line.startswith("#"):
|
||||
# Stop parsing after the block of comments on top
|
||||
break
|
||||
|
||||
if not is_pmb_ci_script:
|
||||
continue
|
||||
|
||||
if not description:
|
||||
logging.error(f"ERROR: {script}: missing '# Description: …' line")
|
||||
exit(1)
|
||||
|
||||
for option in options:
|
||||
if option not in pmb.config.ci_valid_options:
|
||||
raise RuntimeError(f"{script}: unsupported option '{option}'."
|
||||
" Typo in script or pmbootstrap too old?")
|
||||
|
||||
short_name = os.path.basename(script).split(".", -1)[0]
|
||||
ret[short_name] = {"description": description,
|
||||
"options": options}
|
||||
return ret
|
||||
|
||||
|
||||
def sort_scripts_by_speed(scripts):
|
||||
""" Order the scripts, so fast scripts run before slow scripts. Whether a
|
||||
script is fast or not is determined by the '# Options: slow' comment in
|
||||
the file.
|
||||
:param scripts: return of get_ci_scripts()
|
||||
:returns: same format as get_ci_scripts(), but as ordered dict with
|
||||
fast scripts before slow scripts """
|
||||
ret = collections.OrderedDict()
|
||||
|
||||
# Fast scripts first
|
||||
for script_name, script in scripts.items():
|
||||
if "slow" in script["options"]:
|
||||
continue
|
||||
ret[script_name] = script
|
||||
|
||||
# Then slow scripts
|
||||
for script_name, script in scripts.items():
|
||||
if "slow" not in script["options"]:
|
||||
continue
|
||||
ret[script_name] = script
|
||||
return ret
|
||||
|
||||
|
||||
def ask_which_scripts_to_run(scripts_available):
|
||||
""" Display an interactive prompt about which of the scripts the user
|
||||
wishes to run, or all of them.
|
||||
:param scripts_available: same format as get_ci_scripts()
|
||||
:returns: either full scripts_available (all selected), or a subset """
|
||||
count = len(scripts_available.items())
|
||||
choices = ["all"]
|
||||
|
||||
logging.info(f"Available CI scripts ({count}):")
|
||||
for script_name, script in scripts_available.items():
|
||||
extra = ""
|
||||
if "slow" in script["options"]:
|
||||
extra += " (slow)"
|
||||
logging.info(f"* {script_name}: {script['description']}{extra}")
|
||||
choices += [script_name]
|
||||
|
||||
selection = pmb.helpers.cli.ask("Which script?", None, "all",
|
||||
complete=choices)
|
||||
if selection == "all":
|
||||
return scripts_available
|
||||
|
||||
ret = {}
|
||||
ret[selection] = scripts_available[selection]
|
||||
return ret
|
||||
|
||||
|
||||
def copy_git_repo_to_chroot(args, topdir):
|
||||
""" Create a tarball of the git repo (including unstaged changes and new
|
||||
files) and extract it in chroot_native.
|
||||
:param topdir: top directory of the git repository, get it with:
|
||||
pmb.helpers.git.get_topdir() """
|
||||
pmb.chroot.init(args)
|
||||
tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz"
|
||||
|
||||
cmd = "(git ls-files && git ls-files --exclude-standard --other)"
|
||||
cmd += f" | tar -cf {shlex.quote(tarball_path)} -T -"
|
||||
pmb.helpers.run.user(args, ["sh", "-c", cmd], topdir)
|
||||
|
||||
ci_dir = "/home/pmos/ci"
|
||||
pmb.chroot.user(args, ["rm", "-rf", ci_dir])
|
||||
pmb.chroot.user(args, ["mkdir", ci_dir])
|
||||
pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"],
|
||||
working_dir=ci_dir)
|
||||
|
||||
|
||||
def run_scripts(args, topdir, scripts):
|
||||
""" Run one of the given scripts after another, either natively or in a
|
||||
chroot. Display a progress message and stop on error (without printing
|
||||
a python stack trace).
|
||||
:param topdir: top directory of the git repository, get it with:
|
||||
pmb.helpers.git.get_topdir()
|
||||
:param scripts: return of get_ci_scripts() """
|
||||
steps = len(scripts)
|
||||
step = 0
|
||||
repo_copied = False
|
||||
|
||||
for script_name, script in scripts.items():
|
||||
step += 1
|
||||
|
||||
where = "pmbootstrap chroot"
|
||||
if "native" in script["options"]:
|
||||
where = "native"
|
||||
|
||||
script_path = f".ci/{script_name}.sh"
|
||||
logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}"
|
||||
f" [{where}] ***")
|
||||
|
||||
if "native" in script["options"]:
|
||||
rc = pmb.helpers.run.user(args, [script_path], topdir,
|
||||
output="tui")
|
||||
continue
|
||||
else:
|
||||
# Run inside pmbootstrap chroot
|
||||
if not repo_copied:
|
||||
copy_git_repo_to_chroot(args, topdir)
|
||||
repo_copied = True
|
||||
|
||||
env = {"TESTUSER": "pmos"}
|
||||
rc = pmb.chroot.root(args, [script_path], check=False, env=env,
|
||||
working_dir="/home/pmos/ci",
|
||||
output="tui")
|
||||
if rc:
|
||||
logging.error(f"ERROR: CI script failed: {script_name}")
|
||||
exit(1)
|
||||
|
|
@ -1014,3 +1014,9 @@ upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*",
|
|||
# SIDELOAD
|
||||
#
|
||||
sideload_sudo_prompt = "[sudo] password for %u@%h: "
|
||||
|
||||
#
|
||||
# CI
|
||||
#
|
||||
# Valid options für 'pmbootstrap ci', see https://postmarketos.org/pmb-ci
|
||||
ci_valid_options = ["native", "slow"]
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import pmb.build.autodetect
|
|||
import pmb.chroot
|
||||
import pmb.chroot.initfs
|
||||
import pmb.chroot.other
|
||||
import pmb.ci
|
||||
import pmb.config
|
||||
import pmb.export
|
||||
import pmb.flasher
|
||||
|
|
@ -623,3 +624,39 @@ def lint(args):
|
|||
def status(args):
|
||||
if not pmb.helpers.status.print_status(args, args.details):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def ci(args):
|
||||
topdir = pmb.helpers.git.get_topdir(args, os.getcwd())
|
||||
if not os.path.exists(topdir):
|
||||
logging.error("ERROR: change your current directory to a git"
|
||||
" repository (e.g. pmbootstrap, pmaports) before running"
|
||||
" 'pmbootstrap ci'.")
|
||||
exit(1)
|
||||
|
||||
scripts_available = pmb.ci.get_ci_scripts(topdir)
|
||||
scripts_available = pmb.ci.sort_scripts_by_speed(scripts_available)
|
||||
if not scripts_available:
|
||||
logging.error("ERROR: no supported CI scripts found in current git"
|
||||
" repository, see https://postmarketos.org/pmb-ci")
|
||||
exit(1)
|
||||
|
||||
scripts_selected = {}
|
||||
if args.scripts:
|
||||
for script in args.scripts:
|
||||
if script not in scripts_available:
|
||||
logging.error(f"ERROR: script '{script}' not found in git"
|
||||
" repository, found these:"
|
||||
f" {', '.join(scripts_available.keys())}")
|
||||
exit(1)
|
||||
scripts_selected[script] = scripts_available[script]
|
||||
elif args.all:
|
||||
scripts_selected = scripts_available
|
||||
|
||||
if not pmb.helpers.git.clean_worktree(args, topdir):
|
||||
logging.warning("WARNING: this git repository has uncommitted changes")
|
||||
|
||||
if not scripts_selected:
|
||||
scripts_selected = pmb.ci.ask_which_scripts_to_run(scripts_available)
|
||||
|
||||
pmb.ci.run_scripts(args, topdir, scripts_selected)
|
||||
|
|
|
|||
|
|
@ -246,3 +246,10 @@ def is_outdated(path):
|
|||
|
||||
date_outdated = time.time() - pmb.config.git_repo_outdated
|
||||
return date_head <= date_outdated
|
||||
|
||||
|
||||
def get_topdir(args, path):
|
||||
""" :returns: a string with the top dir of the git repository, or an
|
||||
empty string if it's not a git repository. """
|
||||
return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"],
|
||||
path, output_return=True, check=False).rstrip()
|
||||
|
|
|
|||
|
|
@ -546,6 +546,19 @@ def arguments_netboot(subparser):
|
|||
return ret
|
||||
|
||||
|
||||
def arguments_ci(subparser):
|
||||
ret = subparser.add_parser("ci", help="run continuous integration scripts"
|
||||
" locally of git repo in current"
|
||||
" directory")
|
||||
ret.add_argument("-a", "--all", action="store_true",
|
||||
help="name of the CI script to run, depending on the git"
|
||||
" repository")
|
||||
ret.add_argument("scripts", nargs="*", metavar="script",
|
||||
help="name of the CI script to run, depending on the git"
|
||||
" repository")
|
||||
return ret
|
||||
|
||||
|
||||
def package_completer(prefix, action, parser=None, parsed_args=None):
|
||||
args = parsed_args
|
||||
pmb.config.merge_with_args(args)
|
||||
|
|
@ -693,6 +706,7 @@ def arguments():
|
|||
arguments_newapkbuild(sub)
|
||||
arguments_lint(sub)
|
||||
arguments_status(sub)
|
||||
arguments_ci(sub)
|
||||
|
||||
# Action: log
|
||||
log = sub.add_parser("log", help="follow the pmbootstrap logfile")
|
||||
|
|
|
|||
Loading…
Reference in New Issue