Add progress bar when running apk commands (MR 1996)

This adds a progress bar when running apk commands both inside and
outside of the chroot.

Closes: #1700

pmb/chroot/apk.py

@@ -6,6 +6,7 @@ import shlex
 
 import pmb.chroot
 import pmb.config
+import pmb.helpers.apk
 import pmb.helpers.pmaports
 import pmb.parse.apkindex
 import pmb.parse.arch
@@ -231,10 +232,18 @@ def install(args, packages, suffix="native", build=True):
         commands = [["add", "-u", "--virtual", ".pmbootstrap"] + packages_todo,
                     ["add"] + packages,
                     ["del", ".pmbootstrap"]]
-    for command in commands:
+    for (i, command) in enumerate(commands):
         if args.offline:
             command = ["--no-network"] + command
-        pmb.chroot.root(args, ["apk", "--no-progress"] + command, suffix=suffix, disable_timeout=True)
+        if i == 0:
+            pmb.helpers.apk.apk_with_progress(args, ["apk"] + command,
+                                              chroot=True, suffix=suffix)
+        else:
+            # Virtual package related commands don't actually install or remove
+            # packages, but only mark the right ones as explicitly installed.
+            # They finish up almost instantly, so don't display a progress bar.
+            pmb.chroot.root(args, ["apk", "--no-progress"] + command,
+                            suffix=suffix)
 
 
 def installed(args, suffix="native"):

pmb/chroot/apk_static.py

@@ -7,6 +7,7 @@ import tarfile
 import tempfile
 import stat
 
+import pmb.helpers.apk
 import pmb.helpers.run
 import pmb.config
 import pmb.config.load
@@ -169,4 +170,4 @@ def init(args):
 def run(args, parameters):
     if args.offline:
         parameters = ["--no-network"] + parameters
-    pmb.helpers.run.root(args, [args.work + "/apk.static"] + parameters)
+    pmb.helpers.apk.apk_with_progress(args, [args.work + "/apk.static"] + parameters, chroot=False)

pmb/chroot/init.py

@@ -79,7 +79,7 @@ def init(args, suffix="native"):
 
     # Install alpine-base
     pmb.helpers.repo.update(args, arch)
-    pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot,
+    pmb.chroot.apk_static.run(args, ["--root", chroot,
                                      "--cache-dir", apk_cache, "--initdb", "--arch", arch,
                                      "add", "alpine-base"])
 

pmb/config/__init__.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import multiprocessing
 import os
+import sys
 
 #
 # Exported functions
@@ -99,6 +100,13 @@ defaults = {
 }
 
 
+# Whether we're connected to a TTY (which allows things like e.g. printing
+# progress bars)
+is_interactive = sys.stdout.isatty() and \
+    sys.stderr.isatty() and \
+    sys.stdin.isatty()
+
+
 # pmbootstrap will kill programs which do not output anything for several
 # minutes and have one of the following output types. See
 # pmb.helpers.run_core.core() for more information.

pmb/helpers/apk.py

@@ -0,0 +1,108 @@
+# Copyright 2020 Johannes Marbach
+# SPDX-License-Identifier: GPL-3.0-or-later
+import os
+
+import pmb.chroot.root
+import pmb.helpers.cli
+import pmb.helpers.run
+
+
+def _run(args, command, chroot=False, suffix="native", output="log"):
+    """
+    Run a command.
+
+    :param command: command in list form
+    :param chroot: whether to run the command inside the chroot or on the host
+    :param suffix: chroot suffix. Only applies if the "chroot" parameter is
+                   set to True.
+
+    See pmb.helpers.run_core.core() for a detailed description of all other
+    arguments and the return value.
+    """
+    if chroot:
+        return pmb.chroot.root(args, command, output=output, suffix=suffix,
+                               disable_timeout=True)
+    return pmb.helpers.run.root(args, command, output=output)
+
+
+def _prepare_fifo(args, chroot=False, suffix="native"):
+    """
+    Prepare the progress fifo for reading / writing.
+
+    :param chroot: whether to run the command inside the chroot or on the host
+    :param suffix: chroot suffix. Only applies if the "chroot" parameter is
+                   set to True.
+    :returns: A tuple consisting of the path to the fifo as needed by apk to
+              write into it (relative to the chroot, if applicable) and the
+              path of the fifo as needed by cat to read from it (always
+              relative to the host)
+    """
+    if chroot:
+        fifo = "/tmp/apk_progress_fifo"
+        fifo_outside = f"{args.work}/chroot_{suffix}{fifo}"
+    else:
+        _run(args, ["mkdir", "-p", f"{args.work}/tmp"])
+        fifo = fifo_outside = f"{args.work}/tmp/apk_progress_fifo"
+    if os.path.exists(fifo_outside):
+        _run(args, ["rm", "-f", fifo_outside])
+    _run(args, ["mkfifo", fifo_outside])
+    return (fifo, fifo_outside)
+
+
+def _create_command_with_progress(command, fifo):
+    """
+    Build a full apk command from a subcommand, set up to redirect progress
+    into a fifo.
+
+    :param command: apk subcommand in list form
+    :param fifo: path of the fifo
+    :returns: full command in list form
+    """
+    flags = ["--no-progress", "--progress-fd", "3"]
+    command_full = [command[0]] + flags + command[1:]
+    command_flat = pmb.helpers.run.flat_cmd(command_full)
+    command_flat = f"exec 3>{fifo}; {command_flat}"
+    return ["sh", "-c", command_flat]
+
+
+def _compute_progress(line):
+    """
+    Compute the progress as a number between 0 and 1.
+
+    :param line: line as read from the progress fifo
+    :returns: progress as a number between 0 and 1
+    """
+    if not line:
+        return 1
+    cur_tot = line.rstrip().split('/')
+    if len(cur_tot) != 2:
+        return 0
+    cur = float(cur_tot[0])
+    tot = float(cur_tot[1])
+    return cur / tot if tot > 0 else 0
+
+
+def apk_with_progress(args, command, chroot=False, suffix="native"):
+    """
+    Run an apk subcommand while printing a progress bar to STDOUT.
+
+    :param command: apk subcommand in list form
+    :param chroot: whether to run commands inside the chroot or on the host
+    :param suffix: chroot suffix. Only applies if the "chroot" parameter is
+                   set to True.
+    :raises RuntimeError: when the apk command fails
+    """
+    fifo, fifo_outside = _prepare_fifo(args, chroot, suffix)
+    command_with_progress = _create_command_with_progress(command, fifo)
+    log_msg = " ".join(command)
+    with _run(args, ['cat', fifo], chroot=chroot, suffix=suffix,
+              output="pipe") as p_cat:
+        with _run(args, command_with_progress, chroot=chroot, suffix=suffix,
+                  output="background") as p_apk:
+            while p_apk.poll() is None:
+                line = p_cat.stdout.readline().decode('utf-8')
+                progress = _compute_progress(line)
+                pmb.helpers.cli.progress_print(progress)
+            pmb.helpers.cli.progress_flush()
+            pmb.helpers.run_core.check_return_code(args, p_apk.returncode,
+                                                   log_msg)

pmb/helpers/cli.py

@@ -2,8 +2,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import logging
+import os
 import re
 import readline
+import sys
+
+import pmb.config
 
 
 class ReadlineTabCompleter:
@@ -99,3 +103,35 @@ def confirm(args, question="Continue?", default=False, no_assumptions=False):
         return True
     answer = ask(args, question, ["y", "n"], default_str, True, "(y|n)")
     return answer == "y"
+
+
+def progress_print(progress):
+    """
+    Print a snapshot of a progress bar to STDOUT. Call progress_flush to end
+    printing progress and clear the line. No output is printed in
+    non-interactive mode.
+
+    :param progress: completion percentage as a number between 0 and 1
+    """
+    width = 79
+    try:
+        width = os.get_terminal_size().columns - 6
+    except OSError:
+        pass
+    chars = int(width * progress)
+    filled = "\u2588" * chars
+    empty = " " * (width - chars)
+    percent = int(progress * 100)
+    if pmb.config.is_interactive:
+        sys.stdout.write(f"\u001b7{percent:>3}% {filled}{empty}")
+        sys.stdout.flush()
+        sys.stdout.write("\u001b8\u001b[0K")
+
+
+def progress_flush():
+    """
+    Finish printing a progress bar. This will erase the line. Does nothing in
+    non-interactive mode.
+    """
+    if pmb.config.is_interactive:
+        sys.stdout.flush()

pmb/helpers/run_core.py

@@ -213,6 +213,7 @@ def check_return_code(args, code, log_message):
                         entering the chroot and more escaping
     :raises RuntimeError: when the code indicates that the command failed
     """
+
     if code:
         logging.debug("^" * 70)
         logging.info("NOTE: The failed command's output is above the ^^^ line"

pmb/install/_install.py

@@ -150,6 +150,11 @@ def copy_files_from_chroot(args, suffix):
     if os.path.exists(qemu_binary):
         pmb.helpers.run.root(args, ["rm", qemu_binary])
 
+    # Remove apk progress fifo
+    fifo = f"{args.work}/chroot_{suffix}/tmp/apk_progress_fifo"
+    if os.path.exists(fifo):
+        pmb.helpers.run.root(args, ["rm", fifo])
+
     # Get all folders inside the device rootfs (except for home)
     folders = []
     for path in glob.glob(mountpoint_outside + "/*"):

test/static_code_analysis.sh

@@ -74,6 +74,7 @@ py_files="
 	pmb/export/symlinks.py
 	pmb/flasher/__init__.py
 	pmb/helpers/__init__.py
+	pmb/helpers/apk.py
 	pmb/helpers/aportupgrade.py
 	pmb/helpers/args.py
 	pmb/helpers/file.py