From bbf0a70e5b80d4b55dffe8d76714d2c1c21dedab Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Fri, 27 Nov 2020 20:45:22 +0100 Subject: [PATCH] Add progress bar when running apk commands (MR 1996) This adds a progress bar when running apk commands both inside and outside of the chroot. Closes: #1700 --- pmb/chroot/apk.py | 13 ++++- pmb/chroot/apk_static.py | 3 +- pmb/chroot/init.py | 2 +- pmb/config/__init__.py | 8 +++ pmb/helpers/apk.py | 108 +++++++++++++++++++++++++++++++++++ pmb/helpers/cli.py | 36 ++++++++++++ pmb/helpers/run_core.py | 1 + pmb/install/_install.py | 5 ++ test/static_code_analysis.sh | 1 + 9 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 pmb/helpers/apk.py diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index 6ae0776f..bbb5200d 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -6,6 +6,7 @@ import shlex import pmb.chroot import pmb.config +import pmb.helpers.apk import pmb.helpers.pmaports import pmb.parse.apkindex import pmb.parse.arch @@ -231,10 +232,18 @@ def install(args, packages, suffix="native", build=True): commands = [["add", "-u", "--virtual", ".pmbootstrap"] + packages_todo, ["add"] + packages, ["del", ".pmbootstrap"]] - for command in commands: + for (i, command) in enumerate(commands): if args.offline: command = ["--no-network"] + command - pmb.chroot.root(args, ["apk", "--no-progress"] + command, suffix=suffix, disable_timeout=True) + if i == 0: + pmb.helpers.apk.apk_with_progress(args, ["apk"] + command, + chroot=True, suffix=suffix) + else: + # Virtual package related commands don't actually install or remove + # packages, but only mark the right ones as explicitly installed. + # They finish up almost instantly, so don't display a progress bar. + pmb.chroot.root(args, ["apk", "--no-progress"] + command, + suffix=suffix) def installed(args, suffix="native"): diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py index ec97dff2..e1b31b7f 100644 --- a/pmb/chroot/apk_static.py +++ b/pmb/chroot/apk_static.py @@ -7,6 +7,7 @@ import tarfile import tempfile import stat +import pmb.helpers.apk import pmb.helpers.run import pmb.config import pmb.config.load @@ -169,4 +170,4 @@ def init(args): def run(args, parameters): if args.offline: parameters = ["--no-network"] + parameters - pmb.helpers.run.root(args, [args.work + "/apk.static"] + parameters) + pmb.helpers.apk.apk_with_progress(args, [args.work + "/apk.static"] + parameters, chroot=False) diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index b6954f8e..b808c3ad 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -79,7 +79,7 @@ def init(args, suffix="native"): # Install alpine-base pmb.helpers.repo.update(args, arch) - pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot, + pmb.chroot.apk_static.run(args, ["--root", chroot, "--cache-dir", apk_cache, "--initdb", "--arch", arch, "add", "alpine-base"]) diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index b5dd188b..6e6f16ac 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import multiprocessing import os +import sys # # Exported functions @@ -99,6 +100,13 @@ defaults = { } +# Whether we're connected to a TTY (which allows things like e.g. printing +# progress bars) +is_interactive = sys.stdout.isatty() and \ + sys.stderr.isatty() and \ + sys.stdin.isatty() + + # pmbootstrap will kill programs which do not output anything for several # minutes and have one of the following output types. See # pmb.helpers.run_core.core() for more information. diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py new file mode 100644 index 00000000..37d18764 --- /dev/null +++ b/pmb/helpers/apk.py @@ -0,0 +1,108 @@ +# Copyright 2020 Johannes Marbach +# SPDX-License-Identifier: GPL-3.0-or-later +import os + +import pmb.chroot.root +import pmb.helpers.cli +import pmb.helpers.run + + +def _run(args, command, chroot=False, suffix="native", output="log"): + """ + Run a command. + + :param command: command in list form + :param chroot: whether to run the command inside the chroot or on the host + :param suffix: chroot suffix. Only applies if the "chroot" parameter is + set to True. + + See pmb.helpers.run_core.core() for a detailed description of all other + arguments and the return value. + """ + if chroot: + return pmb.chroot.root(args, command, output=output, suffix=suffix, + disable_timeout=True) + return pmb.helpers.run.root(args, command, output=output) + + +def _prepare_fifo(args, chroot=False, suffix="native"): + """ + Prepare the progress fifo for reading / writing. + + :param chroot: whether to run the command inside the chroot or on the host + :param suffix: chroot suffix. Only applies if the "chroot" parameter is + set to True. + :returns: A tuple consisting of the path to the fifo as needed by apk to + write into it (relative to the chroot, if applicable) and the + path of the fifo as needed by cat to read from it (always + relative to the host) + """ + if chroot: + fifo = "/tmp/apk_progress_fifo" + fifo_outside = f"{args.work}/chroot_{suffix}{fifo}" + else: + _run(args, ["mkdir", "-p", f"{args.work}/tmp"]) + fifo = fifo_outside = f"{args.work}/tmp/apk_progress_fifo" + if os.path.exists(fifo_outside): + _run(args, ["rm", "-f", fifo_outside]) + _run(args, ["mkfifo", fifo_outside]) + return (fifo, fifo_outside) + + +def _create_command_with_progress(command, fifo): + """ + Build a full apk command from a subcommand, set up to redirect progress + into a fifo. + + :param command: apk subcommand in list form + :param fifo: path of the fifo + :returns: full command in list form + """ + flags = ["--no-progress", "--progress-fd", "3"] + command_full = [command[0]] + flags + command[1:] + command_flat = pmb.helpers.run.flat_cmd(command_full) + command_flat = f"exec 3>{fifo}; {command_flat}" + return ["sh", "-c", command_flat] + + +def _compute_progress(line): + """ + Compute the progress as a number between 0 and 1. + + :param line: line as read from the progress fifo + :returns: progress as a number between 0 and 1 + """ + if not line: + return 1 + cur_tot = line.rstrip().split('/') + if len(cur_tot) != 2: + return 0 + cur = float(cur_tot[0]) + tot = float(cur_tot[1]) + return cur / tot if tot > 0 else 0 + + +def apk_with_progress(args, command, chroot=False, suffix="native"): + """ + Run an apk subcommand while printing a progress bar to STDOUT. + + :param command: apk subcommand in list form + :param chroot: whether to run commands inside the chroot or on the host + :param suffix: chroot suffix. Only applies if the "chroot" parameter is + set to True. + :raises RuntimeError: when the apk command fails + """ + fifo, fifo_outside = _prepare_fifo(args, chroot, suffix) + command_with_progress = _create_command_with_progress(command, fifo) + log_msg = " ".join(command) + with _run(args, ['cat', fifo], chroot=chroot, suffix=suffix, + output="pipe") as p_cat: + with _run(args, command_with_progress, chroot=chroot, suffix=suffix, + output="background") as p_apk: + while p_apk.poll() is None: + line = p_cat.stdout.readline().decode('utf-8') + progress = _compute_progress(line) + pmb.helpers.cli.progress_print(progress) + pmb.helpers.cli.progress_flush() + pmb.helpers.run_core.check_return_code(args, p_apk.returncode, + log_msg) diff --git a/pmb/helpers/cli.py b/pmb/helpers/cli.py index 68c88968..6edd74eb 100644 --- a/pmb/helpers/cli.py +++ b/pmb/helpers/cli.py @@ -2,8 +2,12 @@ # SPDX-License-Identifier: GPL-3.0-or-later import datetime import logging +import os import re import readline +import sys + +import pmb.config class ReadlineTabCompleter: @@ -99,3 +103,35 @@ def confirm(args, question="Continue?", default=False, no_assumptions=False): return True answer = ask(args, question, ["y", "n"], default_str, True, "(y|n)") return answer == "y" + + +def progress_print(progress): + """ + Print a snapshot of a progress bar to STDOUT. Call progress_flush to end + printing progress and clear the line. No output is printed in + non-interactive mode. + + :param progress: completion percentage as a number between 0 and 1 + """ + width = 79 + try: + width = os.get_terminal_size().columns - 6 + except OSError: + pass + chars = int(width * progress) + filled = "\u2588" * chars + empty = " " * (width - chars) + percent = int(progress * 100) + if pmb.config.is_interactive: + sys.stdout.write(f"\u001b7{percent:>3}% {filled}{empty}") + sys.stdout.flush() + sys.stdout.write("\u001b8\u001b[0K") + + +def progress_flush(): + """ + Finish printing a progress bar. This will erase the line. Does nothing in + non-interactive mode. + """ + if pmb.config.is_interactive: + sys.stdout.flush() diff --git a/pmb/helpers/run_core.py b/pmb/helpers/run_core.py index 97291aef..809413c2 100644 --- a/pmb/helpers/run_core.py +++ b/pmb/helpers/run_core.py @@ -213,6 +213,7 @@ def check_return_code(args, code, log_message): entering the chroot and more escaping :raises RuntimeError: when the code indicates that the command failed """ + if code: logging.debug("^" * 70) logging.info("NOTE: The failed command's output is above the ^^^ line" diff --git a/pmb/install/_install.py b/pmb/install/_install.py index fa7a7dc0..74c24dd7 100644 --- a/pmb/install/_install.py +++ b/pmb/install/_install.py @@ -150,6 +150,11 @@ def copy_files_from_chroot(args, suffix): if os.path.exists(qemu_binary): pmb.helpers.run.root(args, ["rm", qemu_binary]) + # Remove apk progress fifo + fifo = f"{args.work}/chroot_{suffix}/tmp/apk_progress_fifo" + if os.path.exists(fifo): + pmb.helpers.run.root(args, ["rm", fifo]) + # Get all folders inside the device rootfs (except for home) folders = [] for path in glob.glob(mountpoint_outside + "/*"): diff --git a/test/static_code_analysis.sh b/test/static_code_analysis.sh index 10ea6c74..0bd7d48c 100755 --- a/test/static_code_analysis.sh +++ b/test/static_code_analysis.sh @@ -74,6 +74,7 @@ py_files=" pmb/export/symlinks.py pmb/flasher/__init__.py pmb/helpers/__init__.py + pmb/helpers/apk.py pmb/helpers/aportupgrade.py pmb/helpers/args.py pmb/helpers/file.py