pmbootstrap install: add --ondev --cp

Allow to copy one or more files to the install chroot. It will be
possible to use this to put a non-pmOS image into the generated
installer OS.

pmb/helpers/frontend.py

@@ -223,6 +223,12 @@ def install(args):
         if args.rsync:
             raise ValueError("--on-device-installer cannot be combined with"
                              " --rsync")
+    else:
+        if not args.ondev_rootfs:
+            raise ValueError("--no-rootfs can only be combined with --ondev."
+                             " Do you mean --no-image?")
+        if args.ondev_cp:
+            raise ValueError("--cp can only be combined with --ondev")
 
     # On-device installer overrides
     if args.on_device_installer:

pmb/install/_install.py

@@ -342,13 +342,16 @@ def setup_hostname(args):
     pmb.chroot.root(args, ["sed", "-i", "-e", regex, "/etc/hosts"], suffix)
 
 
-def embed_firmware(args):
+def embed_firmware(args, suffix):
     """
     This method will embed firmware, located at /usr/share, that are specified
     by the "sd_embed_firmware" deviceinfo parameter into the SD card image
     (e.g. u-boot). Binaries that would overwrite the first partition are not
     accepted, and if multiple binaries are specified then they will be checked
     for collisions with each other.
+
+    :param suffix: of the chroot, which holds the firmware files (either the
+                   f"rootfs_{args.device}", or f"installer_{args.device}")
     """
     if not args.deviceinfo["sd_embed_firmware"]:
         return
@@ -362,7 +365,7 @@ def embed_firmware(args):
                                "deviceinfo_sd_embed_firmware_step_size "
                                "is not valid: {}".format(step))
 
-    device_rootfs = mount_device_rootfs(args, f"rootfs_{args.device}")
+    device_rootfs = mount_device_rootfs(args, suffix)
     binaries = args.deviceinfo["sd_embed_firmware"].split(",")
 
     # Perform three checks prior to writing binaries to disk: 1) that binaries
@@ -377,12 +380,12 @@ def embed_firmware(args):
         except ValueError:
             raise RuntimeError("Value for firmware binary offset is "
                                "not valid: {}".format(offset))
-        binary_path = os.path.join(args.work, "chroot_rootfs_" +
-                                   args.device, "usr/share", binary)
+        binary_path = os.path.join(args.work, f"chroot_{suffix}", "usr/share",
+                                   binary)
         if not os.path.exists(binary_path):
             raise RuntimeError("The following firmware binary does not "
-                               "exist in the device rootfs: "
-                               "{}".format("/usr/share/" + binary))
+                               f"exist in the {suffix} chroot: "
+                               f"/usr/share/{binary}")
         # Insure that embedding the firmware will not overrun the
         # first partition
         boot_part_start = args.deviceinfo["boot_part_start"] or "2048"
@@ -495,7 +498,7 @@ def install_system_image(args, size_reserve, suffix, root_label="pmOS_root",
     create_home_from_skel(args)
     configure_apk(args)
     copy_ssh_keys(args)
-    embed_firmware(args)
+    embed_firmware(args, suffix)
     pmb.chroot.shutdown(args, True)
 
     # Convert rootfs to sparse using img2simg
@@ -591,10 +594,11 @@ def install_recovery_zip(args):
 
 def install_on_device_installer(args, step, steps):
     # Generate the rootfs image
-    suffix_rootfs = f"rootfs_{args.device}"
-    install_system_image(args, 0, suffix_rootfs, step=step, steps=steps,
-                         split=True)
-    step += 2
+    if args.ondev_rootfs:
+        suffix_rootfs = f"rootfs_{args.device}"
+        install_system_image(args, 0, suffix_rootfs, step=step, steps=steps,
+                             split=True)
+        step += 2
 
     # Prepare the installer chroot
     logging.info(f"*** ({step}/{steps}) CREATE ON-DEVICE INSTALLER ROOTFS ***")
@@ -607,12 +611,14 @@ def install_on_device_installer(args, step, steps):
     pmb.chroot.apk.install(args, packages, suffix_installer)
 
     # Move rootfs image into installer chroot
-    img = f"{args.device}-root.img"
-    img_path_src = f"{args.work}/chroot_native/home/pmos/rootfs/{img}"
-    img_path_dest = f"{args.work}/chroot_{suffix_installer}/var/lib/rootfs.img"
-    logging.info(f"({suffix_installer}) add {img} as /var/lib/rootfs.img")
-    pmb.install.losetup.umount(args, img_path_src)
-    pmb.helpers.run.root(args, ["mv", img_path_src, img_path_dest])
+    if args.ondev_rootfs:
+        img = f"{args.device}-root.img"
+        img_path_src = f"{args.work}/chroot_native/home/pmos/rootfs/{img}"
+        img_path_dest = f"{args.work}/chroot_{suffix_installer}/var/lib/" \
+            "rootfs.img"
+        logging.info(f"({suffix_installer}) add {img} as /var/lib/rootfs.img")
+        pmb.install.losetup.umount(args, img_path_src)
+        pmb.helpers.run.root(args, ["mv", img_path_src, img_path_dest])
 
     # Run ondev-prepare, so it may generate nice configs from the channel
     # properties (e.g. to display the version number), or transform the image
@@ -631,11 +637,20 @@ def install_on_device_installer(args, step, steps):
            "ONDEV_UI": args.ui}
     pmb.chroot.root(args, ["ondev-prepare"], suffix_installer, env=env)
 
+    # Copy files specified with 'pmbootstrap install --ondev --cp'
+    if args.ondev_cp:
+        for host_src, chroot_dest in args.ondev_cp:
+            host_dest = f"{args.work}/chroot_{suffix_installer}/{chroot_dest}"
+            logging.info(f"({suffix_installer}) add {host_src} as"
+                         f" {chroot_dest}")
+            pmb.helpers.run.root(args, ["cp", host_src, host_dest])
+
     # Remove $DEVICE-boot.img (we will generate a new one if --split was
     # specified, otherwise the separate boot image is not needed)
-    img_boot = f"{args.device}-boot.img"
-    logging.info(f"(native) rm {img_boot}")
-    pmb.chroot.root(args, ["rm", f"/home/pmos/rootfs/{img_boot}"])
+    if args.ondev_rootfs:
+        img_boot = f"{args.device}-boot.img"
+        logging.info(f"(native) rm {img_boot}")
+        pmb.chroot.root(args, ["rm", f"/home/pmos/rootfs/{img_boot}"])
 
     # Generate installer image
     size_reserve = round(os.path.getsize(img_path_dest) / 1024 / 1024) + 200
@@ -643,30 +658,11 @@ def install_on_device_installer(args, step, steps):
                          step, steps, args.split, args.sdcard)
 
 
-def install(args):
-    # Sanity checks
-    if not args.android_recovery_zip and args.sdcard:
-        sanity_check_sdcard(args)
-        sanity_check_sdcard_size(args)
-    if args.on_device_installer:
-        sanity_check_ondev_version(args)
-
-    # Number of steps for the different installation methods.
-    if args.no_image:
-        steps = 2
-    elif args.android_recovery_zip:
-        steps = 4
-    elif args.on_device_installer:
-        steps = 8
-    else:
-        steps = 5
-
-    # Install required programs in native chroot
-    logging.info("*** (1/{}) PREPARE NATIVE CHROOT ***".format(steps))
-    pmb.chroot.apk.install(args, pmb.config.install_native_packages,
-                           build=False)
-
-    logging.info(f'*** (2/{steps} CREATE DEVICE ROOTFS("{args.device}") ***')
+def create_device_rootfs(args, step, steps):
+    # List all packages to be installed (including the ones specified by --add)
+    # and upgrade the installed packages/apkindexes
+    logging.info(f'*** ({step}/{steps}) CREATE DEVICE ROOTFS ("{args.device}")'
+                 ' ***')
 
     # Create user before installing packages, so post-install scripts of
     # pmaports can figure out the username (legacy reasons: pmaports#820)
@@ -718,6 +714,36 @@ def install(args):
     # Set the hostname as the device name
     setup_hostname(args)
 
+
+def install(args):
+    # Sanity checks
+    if not args.android_recovery_zip and args.sdcard:
+        sanity_check_sdcard(args)
+        sanity_check_sdcard_size(args)
+    if args.on_device_installer:
+        sanity_check_ondev_version(args)
+
+    # Number of steps for the different installation methods.
+    if args.no_image:
+        steps = 2
+    elif args.android_recovery_zip:
+        steps = 4
+    elif args.on_device_installer:
+        steps = 8 if args.ondev_rootfs else 5
+    else:
+        steps = 5
+
+    # Install required programs in native chroot
+    step = 1
+    logging.info(f"*** ({step}/{steps}) PREPARE NATIVE CHROOT ***")
+    pmb.chroot.apk.install(args, pmb.config.install_native_packages,
+                           build=False)
+    step += 1
+
+    if args.ondev_rootfs:
+        create_device_rootfs(args, step, steps)
+        step += 1
+
     if args.no_image:
         return
     elif args.android_recovery_zip:
@@ -725,8 +751,8 @@ def install(args):
 
     if args.on_device_installer:
         # Runs install_system_image twice
-        install_on_device_installer(args, 3, steps)
+        install_on_device_installer(args, step, steps)
     else:
-        install_system_image(args, 0, f"rootfs_{args.device}",
-                             split=args.split, sdcard=args.sdcard)
+        install_system_image(args, 0, f"rootfs_{args.device}", step=step,
+                             steps=steps, split=args.split, sdcard=args.sdcard)
     print_flash_info(args, steps, steps)

pmb/parse/arguments.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import argparse
 import copy
+import os
 
 try:
     import argcomplete
@@ -22,6 +23,132 @@ import pmb.helpers.pmaports
     See pmb/helpers/args.py for more information about the args variable. """
 
 
+def type_ondev_cp(val):
+    """ Parse and validate arguments to 'pmbootstrap install --ondev --cp'.
+
+        :param val: 'HOST_SRC:CHROOT_DEST' string
+        :returns: (HOST_SRC, CHROOT_DEST) """
+    ret = val.split(":")
+
+    if len(ret) != 2:
+        raise argparse.ArgumentTypeError("does not have HOST_SRC:CHROOT_DEST"
+                                         f" format: {val}")
+    host_src = ret[0]
+    if not os.path.exists(host_src):
+        raise argparse.ArgumentTypeError(f"HOST_SRC not found: {host_src}")
+    if not os.path.isfile(host_src):
+        raise argparse.ArgumentTypeError(f"HOST_SRC is not a file: {host_src}")
+
+    chroot_dest = ret[1]
+    if not chroot_dest.startswith("/"):
+        raise argparse.ArgumentTypeError("CHROOT_DEST must start with '/':"
+                                         f" {chroot_dest}")
+    return ret
+
+
+def arguments_install(subparser):
+    ret = subparser.add_parser("install", help="set up device specific"
+                               " chroot and install to SD card or image file")
+
+    # Image type
+    group_desc = ret.add_argument_group(
+        "optional image type",
+        "Format of the resulting image. Default is generating a combined image"
+        " of the postmarketOS boot and root partitions (--no-split). (If the"
+        " device's deviceinfo_flash_method requires separate boot and root"
+        " partitions, then --split is the default.) Related:"
+        " https://postmarketos.org/partitions")
+    group = group_desc.add_mutually_exclusive_group()
+    group.add_argument("--no-split", help="create combined boot and root image"
+                       " file", dest="split", action="store_false",
+                       default=None)
+    group.add_argument("--split", help="create separate boot and root image"
+                       " files", action="store_true")
+    group.add_argument("--sdcard", help="do not create an image file, instead"
+                       " write to the given SD card device (e.g."
+                       " '/dev/mmcblk0')", metavar="BLOCKDEV")
+    group.add_argument("--android-recovery-zip",
+                       help="generate TWRP flashable zip (recommended read:"
+                            " https://postmarketos.org/recoveryzip)",
+                       action="store_true", dest="android_recovery_zip")
+    group.add_argument("--no-image", help="do not generate an image",
+                       action="store_true", dest="no_image")
+
+    # Image type "--sdcard" related
+    group = ret.add_argument_group("optional image type 'sdcard' arguments")
+    group.add_argument("--rsync", help="update the SD card using rsync",
+                       action="store_true")
+
+    # Image type "--android-recovery-zip" related
+    group = ret.add_argument_group("optional image type 'android-recovery-zip'"
+                                   " arguments")
+    group.add_argument("--flavor", help="kernel flavor to include in recovery"
+                       " flashable zip", default=None)
+    group.add_argument("--recovery-install-partition", default="system",
+                       help="partition to flash from recovery (e.g."
+                            " 'external_sd')",
+                       dest="recovery_install_partition")
+    group.add_argument("--recovery-no-kernel",
+                       help="do not overwrite the existing kernel",
+                       action="store_false", dest="recovery_flash_kernel")
+
+    # Full disk encryption (disabled by default, --no-fde has no effect)
+    group = ret.add_argument_group("optional full disk encryption arguments")
+    group.add_argument("--fde", help="use full disk encryption",
+                       action="store_true", dest="full_disk_encryption")
+    group.add_argument("--no-fde", help=argparse.SUPPRESS,
+                       action="store_true", dest="no_fde")
+    group.add_argument("--cipher", help="cryptsetup cipher used to encrypt the"
+                       " the rootfs (e.g. 'aes-xts-plain64')")
+    group.add_argument("--iter-time", help="cryptsetup iteration time (in"
+                       " milliseconds) to use when encrypting the system"
+                       " partition")
+
+    # Packages
+    group = ret.add_argument_group(
+        "optional packages arguments",
+        "Select or deselect packages to be included in the installation.")
+    group.add_argument("--add", help="comma separated list of packages to be"
+                       " added to the rootfs (e.g. 'vim,gcc')",
+                       metavar="PACKAGES")
+    group.add_argument("--no-base",
+                       help="do not install postmarketos-base (advanced)",
+                       action="store_false", dest="install_base")
+    group.add_argument("--no-recommends", dest="install_recommends",
+                       help="do not install packages listed in _pmb_recommends"
+                            " of the UI pmaports",
+                       action="store_false")
+
+    # Sparse image
+    group_desc = ret.add_argument_group(
+        "optional sparse image arguments",
+        "Override deviceinfo_flash_sparse for testing purpose.")
+    group = group_desc.add_mutually_exclusive_group()
+    group.add_argument("--sparse", help="generate sparse image file",
+                       default=None, action="store_true")
+    group.add_argument("--no-sparse", help="do not generate sparse image file",
+                       dest="sparse", action="store_false")
+
+    # On-device installer
+    group = ret.add_argument_group(
+        "optional on-device installer arguments",
+        "Wrap the resulting image in a postmarketOS based installation OS, so"
+        " it can be encrypted and customized on first boot."
+        " Related: https://postmarketos.org/on-device-installer")
+    group.add_argument("--on-device-installer", "--ondev", action="store_true",
+                       help="enable on-device installer")
+    group.add_argument("--no-local-pkgs", dest="install_local_pkgs",
+                       help="do not install locally compiled packages and"
+                            " package signing keys", action="store_false")
+    group.add_argument("--no-rootfs", dest="ondev_rootfs",
+                       help="do not generate the postmarketOS rootfs to be"
+                            " installed", action="store_false")
+    group.add_argument("--cp", dest="ondev_cp", nargs="+",
+                       metavar="HOST_SRC:CHROOT_DEST", type=type_ondev_cp,
+                       help="copy one or more files from the host system path"
+                            " HOST_SRC to the target path CHROOT_DEST")
+
+
 def arguments_export(subparser):
     ret = subparser.add_parser("export", help="create convenience symlinks"
                                " to generated image files (system, kernel,"
@@ -516,67 +643,7 @@ def arguments():
                                  " 'native'")
 
     # Action: install
-    install = sub.add_parser("install", help="set up device specific" +
-                             " chroot and install to sdcard or image file")
-    group = install.add_mutually_exclusive_group()
-    group.add_argument("--sdcard", help="path to the sdcard device,"
-                       " eg. /dev/mmcblk0")
-    group.add_argument("--split", help="install the boot and root partition"
-                       " in separated image files (default: only if flash method"
-                       " requires it)", action="store_true", default=None)
-    group.add_argument("--no-split", help="create combined boot + root image"
-                       " even if flash method requires it",
-                       dest="split", action="store_false")
-    group.add_argument("--android-recovery-zip",
-                       help="generate TWRP flashable zip",
-                       action="store_true", dest="android_recovery_zip")
-    group.add_argument("--no-image", help="do not generate the image",
-                       action="store_true", dest="no_image")
-    install.add_argument("--rsync", help="update the sdcard using rsync,"
-                         " does not work with --fde", action="store_true")
-    install.add_argument("--cipher", help="cryptsetup cipher used to"
-                         " encrypt the rootfs, eg. aes-xts-plain64")
-    install.add_argument("--iter-time", help="cryptsetup iteration time (in"
-                         " milliseconds) to use when encrypting the system"
-                         " partition")
-    install.add_argument("--add", help="comma separated list of packages to be"
-                         " added to the rootfs (e.g. 'vim,gcc')")
-    install.add_argument("--no-fde", help=argparse.SUPPRESS,
-                         action="store_true", dest="no_fde")
-    install.add_argument("--fde", help="use full disk encryption",
-                         action="store_true", dest="full_disk_encryption")
-    install.add_argument("--flavor",
-                         help="Specify kernel flavor to include in recovery"
-                              " flashable zip", default=None)
-    install.add_argument("--recovery-install-partition", default="system",
-                         help="partition to flash from recovery,"
-                              " eg. external_sd",
-                         dest="recovery_install_partition")
-    install.add_argument("--recovery-no-kernel",
-                         help="do not overwrite the existing kernel",
-                         action="store_false", dest="recovery_flash_kernel")
-    install.add_argument("--no-base",
-                         help="do not install postmarketos-base (advanced)",
-                         action="store_false", dest="install_base")
-    install.add_argument("--on-device-installer", "--ondev",
-                         action="store_true",
-                         help="wrap the resulting image in a graphical"
-                              " on-device installer, so the installation can"
-                              " be customized after flashing")
-    install.add_argument("--no-local-pkgs", dest="install_local_pkgs",
-                         help="do not install locally compiled packages and"
-                              " package signing keys", action="store_false")
-    install.add_argument("--no-recommends", dest="install_recommends",
-                         help="do not install packages listed in"
-                              " _pmb_recommends of the UI pmaports",
-                         action="store_false")
-    group = install.add_mutually_exclusive_group()
-    group.add_argument("--sparse", help="generate sparse image file"
-                       " (even if unsupported by device)", default=None,
-                       action="store_true")
-    group.add_argument("--no-sparse", help="do not generate sparse image file"
-                       " (even if supported by device)", dest="sparse",
-                       action="store_false")
+    arguments_install(sub)
 
     # Action: checksum
     checksum = sub.add_parser("checksum", help="update aport checksums")