From 3c8a93fa7bb7a54454768d5c4f7eb2fc51c51f80 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Tue, 23 Jan 2018 00:54:48 +0000 Subject: [PATCH] Improved pmbootstrap init (#1095) This PR makes the workflow faster and pmbootstrap will produce less traffic. Details: * Check if it's possible to create and read from a device node directly when initializing a chroot (closes #472) * Copy the Qemu binary into the forign-arch chroots before initializing them, so the post-install script directly work during the chroot setup and we don't need to call apk fix afterwards * Use pmb.helpers.repo.update(), which only updates the APKINDEX files if they are older than 4 hours, instead of using apk's repo update function which always downloads the APKINDEX files * Chroot initialization * Getting the initial APKINDEX to download apk-tools-static * Updating the APKINDEX at the start of pmbootstrap install * Fixed a bug in from_chroot_suffix: the buildroot_x86_64 has architecture x86_64, not x86. --- pmb/chroot/apk.py | 5 ++- pmb/chroot/apk_static.py | 21 +++++++---- pmb/chroot/init.py | 78 ++++++++++++++++++++++++++-------------- pmb/parse/arch.py | 2 +- 4 files changed, 69 insertions(+), 37 deletions(-) diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index ec50c006..0f6972a3 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -226,15 +226,14 @@ def install(args, packages, suffix="native", build=True): suffix) -def upgrade(args, suffix="native", update_index=True): +def upgrade(args, suffix="native"): """ Upgrade all packages installed in a chroot """ # Prepare apk and update index check_min_version(args, suffix) pmb.chroot.init(args, suffix) - if update_index: - pmb.chroot.root(args, ["apk", "update"], suffix) + pmb.helpers.repo.update(args) # Rebuild and upgrade out-of-date packages packages = installed(args, suffix).keys() diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py index 7950c385..5a143dd3 100644 --- a/pmb/chroot/apk_static.py +++ b/pmb/chroot/apk_static.py @@ -158,21 +158,28 @@ def init(args): """ Download, verify, extract $WORK/apk.static. """ - apkindex = download(args, "APKINDEX.tar.gz") + # Get the APKINDEX + pmb.helpers.repo.update(args) + url = args.mirror_alpine + args.alpine_version + "/main" + apkindex = (args.work + "/cache_apk_" + args.arch_native + "/APKINDEX." + + pmb.helpers.repo.hash(url) + ".tar.gz") + + # Extract and verify the apk-tools-static version index_data = pmb.parse.apkindex.read(args, "apk-tools-static", apkindex) version = index_data["version"] version_min = pmb.config.apk_tools_static_min_version apk_name = "apk-tools-static-" + version + ".apk" if pmb.parse.version.compare(version, version_min) == -1: - raise RuntimeError("You have an outdated version of apk-tools-static" - " (your version: " + version + - ", expected at least:" - " " + version_min + "). Delete your http cache and zap" - " all chroots, then try again: 'pmbootstrap zap -hc'") + raise RuntimeError("Your APKINDEX has an outdated version of" + " apk-tools-static (your version: " + version + + ", expected at least:" + version_min + "). Please" + + " run 'pmbootstrap update'.") + + # Download, extract, verify apk-tools-static apk_static = download(args, apk_name) extract(args, version, apk_static) -def run(args, parameters, check): +def run(args, parameters, check=True): pmb.helpers.run.root( args, [args.work + "/apk.static"] + parameters, check=check) diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index 0c34f079..73ac75a8 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -45,6 +45,49 @@ def copy_resolv_conf(args, suffix="native"): pmb.helpers.run.root(args, ["touch", chroot]) +def create_device_nodes(args, suffix): + error = ("Failed to create nodes in the '" + suffix + "' chroot. Please" + " run 'pmbootstrap init' again and put your work folder on a" + " normal Linux filesystem. (No ntfs, fat, encfs or encfs" + " encrypted home folder, shared folder etc.)") + + # Folder sturcture + chroot = args.work + "/chroot_" + suffix + pmb.helpers.run.root(args, ["mkdir", "-p", chroot + "/dev"]) + + try: + # Create all device nodes as specified in the config + for dev in pmb.config.chroot_device_nodes: + path = chroot + "/dev/" + str(dev[4]) + if not os.path.exists(path): + pmb.helpers.run.root(args, ["mknod", + "-m", str(dev[0]), # permissions + path, # name + str(dev[1]), # type + str(dev[2]), # major + str(dev[3]), # minor + ]) + + # Verify major and minor numbers of created nodes + for dev in pmb.config.chroot_device_nodes: + path = chroot + "/dev/" + str(dev[4]) + stat_result = os.stat(path) + rdev = stat_result.st_rdev + assert os.major(rdev) == dev[2], "Wrong major in " + path + assert os.minor(rdev) == dev[3], "Wrong minor in " + path + + # Verify /dev/zero reading and writing + path = chroot + "/dev/zero" + with open(path, "r+b", 0) as handle: + assert handle.write(bytes([0xff])), "Write failed for " + path + assert handle.read(1) == bytes([0x00]), "Read failed for " + path + + # On failure: Show filesystem-related error + except Exception as e: + logging.info(str(e) + "!") + raise RuntimeError(error) + + def init(args, suffix="native"): # When already initialized: just prepare the chroot chroot = args.work + "/chroot_" + suffix @@ -70,48 +113,31 @@ def init(args, suffix="native"): logging.info("(" + suffix + ") install alpine-base") - # Initialize cache + # Initialize device nodes and cache + create_device_nodes(args, suffix) apk_cache = args.work + "/cache_apk_" + arch pmb.helpers.run.root(args, ["ln", "-s", "-f", "/var/cache/apk", chroot + "/etc/apk/cache"]) # Initialize /etc/apk/keys/, resolv.conf, repositories - logging.debug(pmb.config.apk_keys_path) for key in glob.glob(pmb.config.apk_keys_path + "/*.pub"): pmb.helpers.run.root(args, ["cp", key, args.work + "/config_apk_keys/"]) copy_resolv_conf(args, suffix) pmb.chroot.apk.update_repository_list(args, suffix) - # Install alpine-base (no clean exit for non-native chroot!) - pmb.chroot.apk_static.run(args, ["-U", "--root", chroot, - "--cache-dir", apk_cache, "--initdb", "--arch", arch, - "add", "alpine-base"], check=(not emulate)) - - # Create device nodes - for dev in pmb.config.chroot_device_nodes: - path = chroot + "/dev/" + str(dev[4]) - if not os.path.exists(path): - pmb.helpers.run.root(args, ["mknod", - "-m", str(dev[0]), # permissions - path, # name - str(dev[1]), # type - str(dev[2]), # major - str(dev[3]), # minor - ]) - if not os.path.exists(path): - raise RuntimeError("Failed to create device node in chroot for " + - dev[4] + "! (This might be caused by setting the work folder" + - " to an eCryptfs folder.)") - - # Non-native chroot: install qemu-user-binary, run apk fix + # Non-native chroot: install qemu-user-binary if emulate: arch_debian = pmb.parse.arch.alpine_to_debian(arch) + pmb.helpers.run.root(args, ["mkdir", "-p", chroot + "/usr/bin"]) pmb.helpers.run.root(args, ["cp", args.work + "/chroot_native/usr/bin/qemu-" + arch_debian + "-static", chroot + "/usr/bin/qemu-" + arch_debian + "-static"]) - pmb.chroot.root(args, ["apk", "fix"], suffix, - auto_init=False) + + # Install alpine-base (no clean exit for non-native chroot!) + pmb.chroot.apk_static.run(args, ["--root", chroot, + "--cache-dir", apk_cache, "--initdb", "--arch", arch, + "add", "alpine-base"]) # Building chroots: create "pmos" user, add symlinks to /home/pmos if not suffix.startswith("rootfs_"): diff --git a/pmb/parse/arch.py b/pmb/parse/arch.py index 0bd1db81..9c9d0ede 100644 --- a/pmb/parse/arch.py +++ b/pmb/parse/arch.py @@ -42,7 +42,7 @@ def from_chroot_suffix(args, suffix): if suffix == "rootfs_" + args.device: return args.deviceinfo["arch"] if suffix.startswith("buildroot_"): - return suffix.split("_", 2)[1] + return suffix.split("_", 1)[1] raise ValueError("Invalid chroot suffix: " + suffix + " (wrong device chosen in 'init' step?)")