Kill the child processes spawned by a run command

When the timeout occurs it is important to ensure clean up of child
processes. Killing only the direct process created by a command can
leave child processes running.

For example a pmbootstrap.py install will run apk add. This run command
creates multiple processes as follows:
(cmd line arguments snipped for readability)

  $ ps -e -o pid,ppid,pgid,cmd
  PID  PPID  PGID CMD
  31738 23247 31738 python3 ./pmbootstrap.py -t 15 install --no-fde
  31746 31738 31738 sudo env -i /bin/sh -c ... ;apk --no-progress add
  31747 31746 31738 /bin/sh -c ... ;apk --no-progress add
  31748 31747 31738 apk --no-progress add

The root process of the run command is PID 31746. We want to kill
the child processes too. Otherwise only running kill -9 31746 will leave
the processes 31747 and 31748 running.

README.md

@@ -10,7 +10,7 @@ Package build scripts live in the [`pmaports`](https://gitlab.com/postmarketOS/p
 * Linux distribution on the host system (`x86`, `x86_64`, or `aarch64`)
   * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
   * Kernels based on the grsec patchset [do **not** work](https://github.com/postmarketOS/pmbootstrap/issues/107) *(Alpine: use linux-vanilla instead of linux-hardened, Arch: linux-hardened [is not based on grsec](https://www.reddit.com/r/archlinux/comments/68b2jn/linuxhardened_in_community_repo_a_grsecurity/))*
-  * On Alpine Linux only: `apk add coreutils`
+  * On Alpine Linux only: `apk add coreutils procps`
   * [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
 * Python 3.4+
 * OpenSSL

pmb/helpers/run_core.py

@@ -93,6 +93,46 @@ def pipe_read(args, process, output_to_stdout=False, output_return=False,
         return
 
 
+def kill_process_tree(args, pid, ppids, kill_as_root):
+    """
+    Recursively kill a pid and its child processes
+
+    :param pid: process id that will be killed
+    :param ppids: list of process id and parent process id tuples (pid, ppid)
+    :param kill_as_root: use sudo to kill the process
+    """
+    if kill_as_root:
+        pmb.helpers.run.root(args, ["kill", "-9", str(pid)],
+                             check=False)
+    else:
+        pmb.helpers.run.user(args, ["kill", "-9", str(pid)],
+                             check=False)
+
+    for (child_pid, child_ppid) in ppids:
+        if child_ppid == str(pid):
+            kill_process_tree(args, child_pid, ppids, kill_as_root)
+
+
+def kill_command(args, pid, kill_as_root):
+    """
+    Kill a command process and recursively kill its child processes
+
+    :param pid: process id that will be killed
+    :param kill_as_root: use sudo to kill the process
+    """
+    cmd = ["ps", "-e", "-o", "pid=,ppid=", "--noheaders"]
+    ret = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
+    ppids = []
+    proc_entries = ret.stdout.decode("utf-8").rstrip().split('\n')
+    for row in proc_entries:
+        items = row.split()
+        if len(items) != 2:
+            raise RuntimeError("Unexpected ps output: " + row)
+        ppids.append(items)
+
+    kill_process_tree(args, pid, ppids, kill_as_root)
+
+
 def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False,
                     output_return=False, output_timeout=True,
                     kill_as_root=False):
@@ -141,11 +181,7 @@ def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False,
                              str(args.timeout) + " seconds. Killing it.")
                 logging.info("NOTE: The timeout can be increased with"
                              " 'pmbootstrap -t'.")
-                if kill_as_root:
-                    pmb.helpers.run.root(args, ["kill", "-9",
-                                         str(process.pid)])
-                else:
-                    process.kill()
+                kill_command(args, process.pid, kill_as_root)
                 continue
 
         # Read all currently available output

test/test_run_core.py

@@ -23,6 +23,7 @@ This file tests functions from pmb.helpers.run_core
 
 import os
 import sys
+import subprocess
 import pytest
 
 # Import from parent directory
@@ -108,6 +109,29 @@ def test_foreground_pipe(args):
     ret = func(args, cmd, output_return=True, output_timeout=True)
     assert ret == (0, "first\nsecond\nthird\nfourth\n")
 
+    # Check if all child processes are killed after timeout.
+    # The first command uses ps to get its process group id (pgid) and echo it
+    # to stdout. All of the test commmands will be running under that pgid.
+    cmd = ["sudo", "sh", "-c",
+           "pgid=$(ps -p ${1:-$$} -o pgid=);echo $pgid | tr -d '\n';" +
+           "sleep 10 | sleep 20 | sleep 30"]
+    args.timeout = 0.3
+    ret = func(args, cmd, output_return=True, output_timeout=True,
+               kill_as_root=True)
+    pgid = str(ret[1])
+
+    cmd = ["ps", "-e", "-o", "pgid=,comm=", "--noheaders"]
+    ret = subprocess.run(cmd, check=True, stdout=subprocess.PIPE)
+    procs = str(ret.stdout.decode("utf-8")).rstrip().split('\n')
+    child_procs = []
+    for process in procs:
+        items = process.split(maxsplit=1)
+        if len(items) != 2:
+            continue
+        if pgid == items[0] and "sleep" in items[1]:
+            child_procs.append(items)
+    assert len(child_procs) == 0
+
 
 def test_foreground_tui():
     func = pmb.helpers.run_core.foreground_tui