Close #69: add 'pmbootstrap initfs' and improve initfs workflow

* allows to build/extract/list initramfs, add/del hook
* rebuild the initfs whenever running install or trying to flash/boot it
* flasher flash/boot: automatically set up a minimal rootfs with kernel and initfs,
  if it does not exist yet

pmb/chroot/initfs.py

@@ -0,0 +1,108 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import logging
+import pmb.chroot.initfs_hooks
+import pmb.chroot.other
+import pmb.chroot.apk
+import pmb.helpers.cli
+
+
+def build(args, flavor, suffix):
+    logging.info("(" + suffix + ") mkinitfs " + flavor)
+    release_file = (args.work + "/chroot_" + suffix + "/usr/share/kernel/" +
+                    flavor + "/kernel.release")
+    with open(release_file, "r") as handle:
+        release = handle.read().rstrip()
+    pmb.chroot.root(args, ["mkinitfs", "-o", "/boot/initramfs-" + flavor, release],
+                    suffix)
+
+
+def extract(args, flavor, suffix, log_message=False):
+    """
+    Extract the initramfs to /tmp/initfs-extracted and return the outside
+    extraction path.
+    """
+    # Extraction folder
+    inside = "/tmp/initfs-extracted"
+    outside = args.work + "/chroot_" + suffix + inside
+    if os.path.exists(outside):
+        if pmb.helpers.cli.ask(args, "Extraction folder " + outside +
+                               " already exists. Do you want to overwrite it?") != "y":
+            raise RuntimeError("Aborted!")
+        pmb.chroot.root(args, ["rm", "-r", inside], suffix)
+
+    # Extraction script (because passing a file to stdin is not allowed
+    # in pmbootstrap's chroot/shell functions for security reasons)
+    with open(args.work + "/chroot_" + suffix + "/tmp/_extract.sh", "w") as handle:
+        handle.write(
+            "#!/bin/sh\n"
+            "cd " + inside + " && cpio -i < _initfs\n")
+
+    # Extract
+    commands = [["mkdir", "-p", inside],
+                ["cp", "/boot/initramfs-" + flavor, inside + "/_initfs.gz"],
+                ["gzip", "-d", inside + "/_initfs.gz"],
+                ["cat", "/tmp/_extract.sh"],  # for the log
+                ["sh", "/tmp/_extract.sh"],
+                ["rm", "/tmp/_extract.sh", inside + "/_initfs"]
+                ]
+    for command in commands:
+        pmb.chroot.root(args, command, suffix)
+
+    # Return outside path for logging
+    return outside
+
+
+def ls(args, flavor, suffix):
+    tmp = "/tmp/initfs-extracted"
+    extract(args, flavor, suffix)
+    pmb.chroot.user(args, ["ls", "-lahR", "."], suffix, tmp, log=False)
+    pmb.chroot.root(args, ["rm", "-r", tmp], suffix)
+
+
+def frontend(args):
+    # Find the appropriate kernel flavor
+    suffix = "rootfs_" + args.device
+    flavor = pmb.chroot.other.kernel_flavor_autodetect(args, suffix)
+    if hasattr(args, "flavor") and args.flavor:
+        flavor = args.flavor
+
+    # Handle initfs actions
+    action = args.action_initfs
+    if action == "build":
+        build(args, flavor, suffix)
+    elif action == "extract":
+        dir = extract(args, flavor, suffix)
+        logging.info("Successfully extracted to: " + dir)
+    elif action == "ls":
+        ls(args, flavor, suffix)
+
+    # Handle hook actions
+    elif action == "hook_ls":
+        pmb.chroot.initfs_hooks.ls(args, suffix)
+    else:
+        if action == "hook_add":
+            pmb.chroot.initfs_hooks.add(args, args.hook, suffix)
+        elif action == "hook_del":
+            pmb.chroot.initfs_hooks.delete(args, args.hook, suffix)
+
+        # Rebuild the initfs for all kernels after adding/removing a hook
+        for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix):
+            build(args, flavor, suffix)

pmb/chroot/initfs_hooks.py

@@ -0,0 +1,67 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import glob
+import logging
+
+import pmb.config
+import pmb.chroot.apk
+
+
+def list_chroot(args, suffix):
+    ret = []
+    prefix = pmb.config.initfs_hook_prefix
+    for pkgname in pmb.chroot.apk.installed(args, suffix):
+        if pkgname.startswith(prefix):
+            ret.append(pkgname[len(prefix):])
+    return ret
+
+
+def list_aports(args):
+    ret = []
+    prefix = pmb.config.initfs_hook_prefix
+    for path in glob.glob(args.aports + "/" + prefix + "*"):
+        ret.append(os.path.basename(path)[len(prefix):])
+    return ret
+
+
+def ls(args, suffix):
+    hooks_chroot = list_chroot(args, suffix)
+    hooks_aports = list_aports(args)
+
+    for hook in hooks_aports:
+        line = "* " + hook
+        if hook in hooks_chroot:
+            line += " (installed)"
+        logging.info(line)
+
+
+def add(args, hook, suffix):
+    if hook not in list_aports(args):
+        raise RuntimeError("Invalid hook name! Run 'pmbootstrap initfs hook_ls'"
+                           " to get a list of all hooks.")
+    prefix = pmb.config.initfs_hook_prefix
+    pmb.chroot.apk.install(args, [prefix + hook], suffix)
+
+
+def delete(args, hook, suffix):
+    if hook not in list_chroot(args, suffix):
+        raise RuntimeError("There is no such hook installed!")
+    prefix = pmb.config.initfs_hook_prefix
+    pmb.chroot.root(args, ["apk", "del", prefix + hook], suffix)

pmb/chroot/other.py

@@ -18,9 +18,10 @@ along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import os
 import glob
+import pmb.chroot.apk
 
 
-def installed_kernel_flavors(args, suffix):
+def kernel_flavors_installed(args, suffix):
     prefix = "vmlinuz-"
     prefix_len = len(prefix)
     pattern = args.work + "/chroot_" + suffix + "/boot/" + prefix + "*"
@@ -28,3 +29,12 @@ def installed_kernel_flavors(args, suffix):
     for file in glob.glob(pattern):
         ret.append(os.path.basename(file)[prefix_len:])
     return ret
+
+
+def kernel_flavor_autodetect(args, suffix):
+    """
+    Make sure, that there is at least one kernel installed, return the first
+    kernel that can be found.
+    """
+    pmb.chroot.apk.install(args, ["device-" + args.device], suffix)
+    return kernel_flavors_installed(args, suffix)[0]

pmb/config/__init__.py

@@ -142,6 +142,12 @@ apkbuild_attributes = {
     "_llvmver": {"array": False},
 }
 
+#
+# INITFS
+#
+initfs_hook_prefix = "postmarketos-mkinitfs-hook-"
+
+
 #
 # INSTALL
 #

pmb/flasher/frontend.py

@@ -21,24 +21,33 @@ import os
 
 import pmb.flasher
 import pmb.install
+import pmb.chroot.apk
+import pmb.chroot.initfs
 import pmb.chroot.other
 
 
 def kernel(args):
-    # Parse the kernel flavor
+    # Make sure, that at least one kernel is installed
     suffix = "rootfs_" + args.device
+    pmb.chroot.apk.install(args, ["device-" + args.device], suffix)
+
+    # Parse the kernel flavor
     flavor = args.flavor
-    flavors = pmb.chroot.other.installed_kernel_flavors(args, suffix)
+    flavors = pmb.chroot.other.kernel_flavors_installed(args, suffix)
     if flavor:
         if flavor not in flavors:
             raise RuntimeError("No kernel installed with flavor " + flavor + "!" +
                                " Run 'pmbootstrap flasher list_flavors' to get a list.")
     elif not len(flavors):
         raise RuntimeError(
-            "No kernel flavors installed in chroot " + suffix + "!")
+            "No kernel flavors installed in chroot " + suffix + "! Please let"
+            " your device package depend on a package starting with 'linux-'.")
     else:
         flavor = flavors[0]
 
+    # Rebuild the initramfs, just to make sure (see #69)
+    pmb.chroot.initfs.build(args, flavor, "rootfs_" + args.device)
+
     # Generate the paths and run the flasher
     pmb.flasher.init(args)
     mnt = "/mnt/rootfs_" + args.device
@@ -48,14 +57,14 @@ def kernel(args):
         logging.info("(native) boot " + flavor + " kernel")
         pmb.flasher.run(args, "boot", kernel, ramdisk)
     else:
-        logging.info("(native) flash kernel '" + flavor + "'")
+        logging.info("(native) flash kernel " + flavor)
         pmb.flasher.run(args, "flash_kernel", kernel, ramdisk)
 
 
 def list_flavors(args):
     suffix = "rootfs_" + args.device
     logging.info("(" + suffix + ") installed kernel flavors:")
-    for flavor in pmb.chroot.other.installed_kernel_flavors(args, suffix):
+    for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix):
         logging.info("* " + flavor)
 
 

pmb/flasher/run.py

@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import pmb.flasher
+import pmb.chroot.initfs
 
 
 def run(args, action, kernel=None, ramdisk=None, image=None):
@@ -42,7 +43,7 @@ def run(args, action, kernel=None, ramdisk=None, image=None):
         "$PAGE_SIZE": args.deviceinfo["flash_pagesize"],
     }
 
-    # Each action has multiple commands
+    # Run the commands of each action
     for command in cfg["actions"][action]:
         # Variable replacement
         for key, value in vars.items():

pmb/install/install.py

@@ -22,6 +22,8 @@ import glob
 
 import pmb.chroot
 import pmb.chroot.apk
+import pmb.chroot.other
+import pmb.chroot.initfs
 import pmb.config
 import pmb.helpers.run
 import pmb.install.blockdevice
@@ -97,8 +99,14 @@ def install(args, show_flash_msg=True):
     for pkgname in install_packages:
         pmb.build.package(args, pkgname, args.deviceinfo["arch"])
 
-    # Install all packages to device rootfs chroot
+    # Install all packages to device rootfs chroot (and rebuild the initramfs,
+    # because that doesn't always happen automatically yet, e.g. when the user
+    # installed a hook without pmbootstrap - see #69 for more info)
     pmb.chroot.apk.install(args, install_packages, suffix)
+    for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix):
+        pmb.chroot.initfs.build(args, flavor, suffix)
+
+    # Finally set the user password
     set_user_password(args)
 
     # Partition and fill image/sdcard

pmb/parse/arguments.py

@@ -41,6 +41,38 @@ def arguments_flasher(subparser):
     return ret
 
 
+def arguments_initfs(subparser):
+    ret = subparser.add_parser(
+        "initfs", help="do something with the initramfs")
+    sub = ret.add_subparsers(dest="action_initfs")
+
+    # hook ls
+    sub.add_parser(
+        "hook_ls",
+        help="list available and installed hook packages")
+
+    # hook add/del
+    hook_add = sub.add_parser("hook_add", help="add a hook package")
+    hook_del = sub.add_parser("hook_del", help="uninstall a hook package")
+    for action in [hook_add, hook_del]:
+        action.add_argument("hook", help="name of the hook aport, without the"
+                            " '" + pmb.config.initfs_hook_prefix + "' prefix, for example: 'usb-shell'")
+
+    # ls, build, extract
+    ls = sub.add_parser("ls", help="list initfs contents")
+    build = sub.add_parser("build", help="(re)build the initramfs")
+    extract = sub.add_parser("extract", help="extract the initramfs to a temporary folder"
+                             " inside the (native) chroot")
+    for action in [ls, build, extract]:
+        action.add_argument(
+            "--flavor",
+            default=None,
+            help="name of the kernel flavor (run 'pmbootstrap flasher list_flavors'"
+            " to get a list of all installed flavors")
+
+    return ret
+
+
 def arguments():
     parser = argparse.ArgumentParser(prog="pmbootstrap")
 
@@ -77,6 +109,7 @@ def arguments():
     sub.add_parser("index", help="re-index all repositories with custom built"
                    " packages (do this after manually removing package files)")
     arguments_flasher(sub)
+    arguments_initfs(sub)
 
     # Action: log
     log = sub.add_parser("log", help="follow the pmbootstrap logfile")

pmbootstrap.py

@@ -29,6 +29,7 @@ import pmb.aportgen
 import pmb.build
 import pmb.config
 import pmb.chroot
+import pmb.chroot.initfs
 import pmb.chroot.other
 import pmb.flasher
 import pmb.helpers.logging
@@ -64,6 +65,8 @@ def main():
             pmb.chroot.root(args, args.command, args.suffix, log=False)
         elif args.action == "index":
             pmb.build.index_repo(args)
+        elif args.action == "initfs":
+            pmb.chroot.initfs.frontend(args)
         elif args.action == "install":
             pmb.install.install(args)
         elif args.action == "flasher":