From 1c13ca4fd9e32952a8e559987f82cd0925cddee9 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Fri, 18 Aug 2017 19:19:48 +0000 Subject: [PATCH] Fix #363: Omit /home/user when calculating system image size (#389) * The system image size is now calculated as: root size - home size. * New function in `pmb/helpers/other.py`: `folder_size()`, with a testcase. * Instead of copying everything to the system image folder, and deleting the home folder afterwards, do not copy the home folder in the first place. * Added `pmbootstrap -s` to skip generating the initramfs for faster debugging. * Set the default value in the "are you sure, that your partition has at least..." to "y", so we can run `yes '' | pmbootstrap install` to make it run through the whole installation process. * Increase full size to 120%, boot partition gets 15 MB free space now --- pmb/chroot/initfs.py | 5 +++ pmb/chroot/shutdown.py | 5 +++ pmb/helpers/other.py | 15 +++++++ pmb/install/blockdevice.py | 20 ++++++--- pmb/install/install.py | 88 +++++++++++++++++++++----------------- pmb/install/partition.py | 13 ++++-- pmb/parse/arguments.py | 3 ++ test/test_folder_size.py | 51 ++++++++++++++++++++++ 8 files changed, 152 insertions(+), 48 deletions(-) create mode 100644 test/test_folder_size.py diff --git a/pmb/chroot/initfs.py b/pmb/chroot/initfs.py index 15609dce..4f689e4a 100644 --- a/pmb/chroot/initfs.py +++ b/pmb/chroot/initfs.py @@ -25,6 +25,11 @@ import pmb.helpers.cli def build(args, flavor, suffix): + # Bail out when '-s' is set + if args.skip_initfs: + logging.info("NOTE: Skipped initramfs generation (-s)!") + return + # Update mkinitfs and hooks pmb.chroot.apk.install(args, ["postmarketos-mkinitfs"], suffix) pmb.chroot.initfs_hooks.update(args, suffix) diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index dedde64d..f3db2a5f 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -68,6 +68,11 @@ def shutdown(args, only_install_related=False): path = path_outside[len(chroot):] pmb.install.losetup.umount(args, path) + # Umount device rootfs chroot + chroot_rootfs = args.work + "/chroot_rootfs_" + args.device + if os.path.exists(chroot_rootfs): + pmb.helpers.mount.umount_all(args, chroot_rootfs) + if not only_install_related: # Clean up the rest pmb.helpers.mount.umount_all(args, args.work) diff --git a/pmb/helpers/other.py b/pmb/helpers/other.py index 1921f539..a922c184 100644 --- a/pmb/helpers/other.py +++ b/pmb/helpers/other.py @@ -17,6 +17,21 @@ You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ import os +import pmb.helpers.run + + +def folder_size(args, path): + """ + Run `du` to calculate the size of a folder (this is less code and + faster than doing the same task in pure Python). + + :returns: folder size in bytes + """ + output = pmb.helpers.run.root(args, ["du", "--summarize", + "--block-size=1", + path], return_stdout=True) + ret = int(output.split("\t")[0]) + return ret def check_grsec(args): diff --git a/pmb/install/blockdevice.py b/pmb/install/blockdevice.py index 2ba9be92..dc60ebe7 100644 --- a/pmb/install/blockdevice.py +++ b/pmb/install/blockdevice.py @@ -47,6 +47,11 @@ def mount_sdcard(args): def create_and_mount_image(args, size): + """ + Create a new image file, and mount it as /dev/install. + + :param size: of the whole image in bytes + """ # Short variables for paths chroot = args.work + "/chroot_native" img_path = "/home/user/rootfs/" + args.device + ".img" @@ -61,15 +66,18 @@ def create_and_mount_image(args, size): raise RuntimeError("Failed to remove old image file: " + img_path_outside) - # Create empty image file - logging.info("(native) create " + args.device + ".img (" + size + ")") + # Convert to MB and ask for confirmation + mb = str(round(size / 1024 / 1024)) + "M" + logging.info("(native) create " + args.device + ".img (" + mb + ")") logging.info("WARNING: Make sure, that your target device's partition" - " table has allocated at least " + size + " as system partition!") - if not pmb.helpers.cli.confirm(args): + " table has allocated at least " + mb + " as system" + " partition!") + if not pmb.helpers.cli.confirm(args, default=True): raise RuntimeError("Aborted.") + # Create empty image file pmb.chroot.user(args, ["mkdir", "-p", "/home/user/rootfs"]) - pmb.chroot.root(args, ["truncate", "-s", size, img_path]) + pmb.chroot.root(args, ["truncate", "-s", mb, img_path]) # Mount to /dev/install logging.info("(native) mount /dev/install (" + args.device + ".img)") @@ -82,6 +90,8 @@ def create_and_mount_image(args, size): def create(args, size): """ Create /dev/install (the "install blockdevice"). + + :param size: of the whole image in bytes """ pmb.helpers.mount.umount_all( args, args.work + "/chroot_native/dev/install") diff --git a/pmb/install/install.py b/pmb/install/install.py index 31fe0131..6fb74725 100644 --- a/pmb/install/install.py +++ b/pmb/install/install.py @@ -26,65 +26,78 @@ import pmb.chroot.other import pmb.chroot.initfs import pmb.config import pmb.helpers.run +import pmb.helpers.other import pmb.install.blockdevice import pmb.install def mount_device_rootfs(args): # Mount the device rootfs - logging.info("(native) copy rootfs_" + args.device + " to" + - " /mnt/install/") mountpoint = "/mnt/rootfs_" + args.device pmb.helpers.mount.bind(args, args.work + "/chroot_rootfs_" + args.device, args.work + "/chroot_native" + mountpoint) return mountpoint -def get_chroot_size(args): +def get_subpartitions_size(args): + """ + Calculate the size of the whole image and boot subpartition. + + :returns: (full, boot) the size of the full image and boot + partition as integer in bytes + """ + # Calculate required sizes first + chroot = args.work + "/chroot_rootfs_" + args.device + root = pmb.helpers.other.folder_size(args, chroot) + boot = pmb.helpers.other.folder_size(args, chroot + "/boot") + home = pmb.helpers.other.folder_size(args, chroot + "/home") + + # The home folder gets omitted when copying the rootfs to + # /dev/installp2 + full = root - home + + # Add some free space, see also: + # https://github.com/postmarketOS/pmbootstrap/pull/336 + full *= 1.20 + boot += 15 * 1024 * 1024 + return (full, boot) + + +def copy_files_from_chroot(args): + """ + Copy all files from the rootfs chroot to /mnt/install, except + for the home folder (because /home will contain some empty + mountpoint folders). + """ # Mount the device rootfs + logging.info("(native) copy rootfs_" + args.device + " to" + + " /mnt/install/") mountpoint = mount_device_rootfs(args) + mountpoint_outside = args.work + "/chroot_native" + mountpoint - # Run the du command - result = pmb.chroot.root(args, ["sh", "-c", "du -cm . | grep total$ | cut -f1"], - working_dir=mountpoint, return_stdout=True) - return result - - -def get_chroot_boot_size(args): - # Mount the device rootfs - mountpoint = mount_device_rootfs(args) - - # Run the du command - result = pmb.chroot.root(args, ["sh", "-c", "du -cm ./boot | grep total$ | cut -f1"], - working_dir=mountpoint, return_stdout=True) - return result - - -def copy_files(args): - # Mount the device rootfs - mountpoint = mount_device_rootfs(args) - - # Get all folders inside the device rootfs + # Get all folders inside the device rootfs (except for home) folders = [] - for path in glob.glob(args.work + "/chroot_native" + mountpoint + "/*"): + for path in glob.glob(mountpoint_outside + "/*"): + if path.endswith("/home"): + continue folders += [os.path.basename(path)] # Run the copy command pmb.chroot.root(args, ["cp", "-a"] + folders + ["/mnt/install/"], working_dir=mountpoint) -# copy over keys and delete unneded mount folders - -def fix_mount_folders(args): - # copy over keys - rootfs = args.work + "/chroot_native/mnt/install/" +def copy_files_other(args): + """ + Copy over keys, create /home/user. + """ + # Copy over keys + rootfs = args.work + "/chroot_native/mnt/install" for key in glob.glob(args.work + "/config_apk_keys/*.pub"): pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"]) - # delete everything (-> empty mount folders) in /home/user - pmb.helpers.run.root(args, ["rm", "-r", rootfs + "/home/user"]) - pmb.helpers.run.root(args, ["mkdir", rootfs + "/home/user"]) + # Create /home/user + pmb.helpers.run.root(args, ["mkdir", "-p", rootfs + "/home/user"]) pmb.helpers.run.root(args, ["chown", pmb.config.chroot_uid_user, rootfs + "/home/user"]) @@ -153,27 +166,24 @@ def install(args): for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix): pmb.chroot.initfs.build(args, flavor, suffix) - size_image = str(int(float(get_chroot_size(args)) * 1.15)) + "M" - size_boot = str(int(get_chroot_boot_size(args)) + 5) + "M" - # Set the user password set_user_password(args) # Partition and fill image/sdcard logging.info("*** (3/5) PREPARE INSTALL BLOCKDEVICE ***") pmb.chroot.shutdown(args, True) + (size_image, size_boot) = get_subpartitions_size(args) pmb.install.blockdevice.create(args, size_image) pmb.install.partition(args, size_boot) pmb.install.format(args) # Just copy all the files logging.info("*** (4/5) FILL INSTALL BLOCKDEVICE ***") - copy_files(args) - fix_mount_folders(args) + copy_files_from_chroot(args) + copy_files_other(args) # If user has a ssh pubkey, offer to copy it to device copy_ssh_key(args) - pmb.chroot.shutdown(args, True) # Convert system image to sparse using img2simg diff --git a/pmb/install/partition.py b/pmb/install/partition.py index b28777e6..d3670b5e 100644 --- a/pmb/install/partition.py +++ b/pmb/install/partition.py @@ -51,14 +51,19 @@ def partitions_mount(args): def partition(args, size_boot): """ Partition /dev/install and create /dev/install{p1,p2} - """ - logging.info("(native) partition /dev/install (boot: " + size_boot + + size_boot: size of the boot partition in bytes. + """ + # Convert to MB and print info + mb_boot = str(round(size_boot / 1024 / 1024)) + "M" + logging.info("(native) partition /dev/install (boot: " + mb_boot + ", root: the rest)") + + # Actual partitioning with 'parted' commands = [ ["mktable", "msdos"], - ["mkpart", "primary", "ext2", "2048s", size_boot], - ["mkpart", "primary", size_boot, "100%"], + ["mkpart", "primary", "ext2", "2048s", mb_boot], + ["mkpart", "primary", mb_boot, "100%"], ["set", "1", "boot", "on"] ] for command in commands: diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 0e5cedf4..71fac305 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -103,6 +103,9 @@ def arguments(): parser.add_argument("-j", "--jobs", help="parallel jobs when compiling") parser.add_argument("-p", "--aports", help="postmarketos aports paths") + parser.add_argument("-s", "--skip-initfs", dest="skip_initfs", + help="do not re-generate the initramfs", + action="store_true") parser.add_argument("-w", "--work", help="folder where all data" " gets stored (chroots, caches, built packages)") diff --git a/test/test_folder_size.py b/test/test_folder_size.py new file mode 100644 index 00000000..5a54143a --- /dev/null +++ b/test/test_folder_size.py @@ -0,0 +1,51 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import sys +import pytest + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.helpers.logging +import pmb.helpers.other +import pmb.helpers.run + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + args.details_to_stdout = True + pmb.helpers.logging.init(args) + return args + + +def test_get_folder_size(args, tmpdir): + # Write five 2 KB files to tmpdir + tmpdir = str(tmpdir) + files = 5 + for i in range(files): + pmb.helpers.run.user(args, ["dd", "if=/dev/zero", "of=" + + tmpdir + "/" + str(i), "bs=1K", + "count=2", "conv=notrunc"]) + + # Check if the size is correct + assert pmb.helpers.other.folder_size(args, tmpdir) == 20480