From 147863d98a6ba3810fe532cca6b20f8ee3dd6dec Mon Sep 17 00:00:00 2001 From: drebrez Date: Thu, 3 Aug 2017 18:01:00 +0200 Subject: [PATCH] Automatically compute the minimum size for the partition and resize it during the boot process (#127) * Automatically compute the minimum size for the partitions * Automatically resize the pmOS_root partition during the boot process * Resize root partition only if there is unallocated space at the end of the device. * Added more echos to make debugging easier while looking at the pmOS_init.log. * Updated static_code_analysis.sh script to run shellcheck with `-x` option. --- aports/main/postmarketos-mkinitfs/APKBUILD | 8 +- aports/main/postmarketos-mkinitfs/init.sh.in | 7 +- .../postmarketos-mkinitfs/init_functions.sh | 102 ++++++++++++------ aports/main/postmarketos-mkinitfs/mkinitfs.sh | 4 + pmb/config/__init__.py | 2 - pmb/install/blockdevice.py | 7 +- pmb/install/install.py | 35 +++++- pmb/install/partition.py | 3 +- test/static_code_analysis.sh | 16 ++- 9 files changed, 133 insertions(+), 51 deletions(-) diff --git a/aports/main/postmarketos-mkinitfs/APKBUILD b/aports/main/postmarketos-mkinitfs/APKBUILD index 29aebc8c..98693ab0 100644 --- a/aports/main/postmarketos-mkinitfs/APKBUILD +++ b/aports/main/postmarketos-mkinitfs/APKBUILD @@ -1,5 +1,5 @@ pkgname=postmarketos-mkinitfs -pkgver=0.2.0 +pkgver=0.3.0 pkgrel=1 pkgdesc="Tool to generate initramfs images for postmarketOS" url="https://github.com/postmarketOS" @@ -25,6 +25,6 @@ package() { "$pkgdir/sbin/mkinitfs" mkdir -p "$pkgdir/etc/postmarketos-mkinitfs/hooks/" } -sha512sums="318f4d925df001c1f7378c1577332fba4964bbd822c1b050c41b908c2eb37365c418e1efb6f329cf51c515b34ef3e97d1b9e8cc85509f1cefea1015b3c9db663 init.sh.in -ccd05065a8f66f181351987740dff62b5a38f21aa764a3dbdb4601c494e756a5885ff17f69a6916eaae28498cd3c79d49e5962190ee0affd389f8f584dbaa3c2 init_functions.sh -e0255a5e7debe41efe97a3a156ef866ec5e3e9f6d57d20c0acd889470501179a9eace709afa3b98d46d513c47113701ac5b17fbf116642e683972db91a4b4824 mkinitfs.sh" +sha512sums="95e49eb5e7891d319543165290fb7279733e696db8c768d864d87ef36a20b04e1c4e8a1702a2927e7492ea2e9f3d4fdd83d9ea55811969689b9c2483fb3b44c4 init.sh.in +66207a12dba2be8868d080e69e54dce43c8a3edbbd08a6585f8ad49034e32bbe9184c4c77b5fe49049d7475f930781f48919a12eb4d82c9e0008f6fa6da0d035 init_functions.sh +9bf70dda59c4c64bef829dcdaf0973168601e91b5478b5d827dbb967c9682b0eedd2a84211fb679e95d5b5e1a12d01ec5baef9b3b12995e376322fd81303c716 mkinitfs.sh" diff --git a/aports/main/postmarketos-mkinitfs/init.sh.in b/aports/main/postmarketos-mkinitfs/init.sh.in index 349dd057..2c78b63e 100644 --- a/aports/main/postmarketos-mkinitfs/init.sh.in +++ b/aports/main/postmarketos-mkinitfs/init.sh.in @@ -18,6 +18,8 @@ for hook in /etc/postmarketos-mkinitfs/hooks/*.sh; do sh "$hook" done +show_splash /splash-loading.ppm.gz + # Always run dhcp daemon/usb networking for now (later this should only # be enabled, when having the usb-shell hook installed for debugging, # or get activated after the initramfs is done with an OpenRC service). @@ -26,10 +28,13 @@ start_udhcpd mount_boot_partition extract_initramfs_extra @INITRAMFS_EXTRA@ +wait_root_partition +resize_root_partition unlock_root_partition +resize_root_filesystem +mount_root_partition # Switch root -show_splash /splash-loading.ppm.gz killall telnetd mdev 2>/dev/null umount /boot umount /proc diff --git a/aports/main/postmarketos-mkinitfs/init_functions.sh b/aports/main/postmarketos-mkinitfs/init_functions.sh index 9cb47f4a..657a4340 100644 --- a/aports/main/postmarketos-mkinitfs/init_functions.sh +++ b/aports/main/postmarketos-mkinitfs/init_functions.sh @@ -43,7 +43,7 @@ mount_subpartitions() { for i in /dev/mmcblk*; do case "$(kpartx -l "$i" 2>/dev/null | wc -l)" in 2) - echo "mount subpartitions of $i" + echo "Mount subpartitions of $i" kpartx -afs "$i" break ;; @@ -104,6 +104,7 @@ mount_boot_partition() { show_splash /splash-noboot.ppm.gz loop_forever fi + echo "Mount boot partition ($partition)" mount -r -t ext2 "$partition" /boot } @@ -115,9 +116,74 @@ extract_initramfs_extra() { show_splash /splash-noinitramfsextra.ppm.gz loop_forever fi + echo "Extract $initramfs_extra" gzip -d -c "$initramfs_extra" | cpio -i } +wait_root_partition() { + while [ -z "$(find_root_partition)" ]; do + show_splash /splash-nosystem.ppm.gz + echo "Could not find the root partition." + echo "Maybe you need to insert the sdcard, if your device has" + echo "any? Trying again in one second..." + sleep 1 + done +} + +resize_root_partition() { + partition=$(find_root_partition) + # Only resize the partition if it's inside the device-mapper, which means + # that the partition is stored as a subpartition inside another one. + # In this case we want to resize it to use all the unused space of the + # external partition. + if [ -z "${partition##"/dev/mapper/"*}" ]; then + # Get physical device + partition_dev=$(dmsetup deps -o devname "$partition" | \ + awk -F "[()]" '{print "/dev/"$2}') + # Check if there is unallocated space at the end of the device + if parted -s "$partition_dev" print free | tail -n2 | \ + head -n1 | grep -qi "free space"; then + echo "Resize root partition ($partition)" + # unmount subpartition, resize and remount it + kpartx -d "$partition" + parted -s "$partition_dev" resizepart 2 100% + kpartx -afs "$partition_dev" + fi + fi +} + +unlock_root_partition() { + partition="$(find_root_partition)" + if cryptsetup isLuks "$partition"; then + until cryptsetup status root | grep -qwi active; do + start_usb_unlock + cryptsetup luksOpen "$partition" root || continue + done + # Show again the loading splashscreen + show_splash /splash-loading.ppm.gz + fi +} + +resize_root_filesystem() { + partition="$(find_root_partition)" + touch /etc/mtab # see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=673323 + echo "Check/repair root filesystem ($partition)" + e2fsck -f -y "$partition" + echo "Resize root filesystem ($partition)" + resize2fs -f "$partition" +} + +mount_root_partition() { + partition="$(find_root_partition)" + echo "Mount root partition ($partition)" + mount -w -t ext4 "$partition" /sysroot + if ! [ -e /sysroot/usr ]; then + echo "ERROR: unable to mount root partition!" + show_splash /splash-mounterror.ppm.gz + loop_forever + fi +} + setup_usb_network_android() { # Only run, when we have the android usb driver SYS=/sys/class/android_usb/android0 @@ -156,7 +222,7 @@ setup_usb_network() { _marker="/tmp/_setup_usb_network" [ -e "$_marker" ] && return touch "$_marker" - + echo "Setup usb network" # Run all usb network setup functions (add more below!) setup_usb_network_android setup_usb_network_configfs @@ -182,7 +248,7 @@ start_udhcpd() { echo "option subnet 255.255.255.0" } >/etc/udhcpd.conf - # Start the dhcpcd daemon (forks into background) + echo "Start the dhcpcd daemon (forks into background)" udhcpd } @@ -199,43 +265,19 @@ start_usb_unlock() { # Telnet splash show_splash /splash-telnet.ppm.gz - # Start the telnet daemon + echo "Start the telnet daemon (unlock encrypted partition)" { echo '#!/bin/sh' echo '. /init_functions.sh' echo 'unlock_root_partition' echo 'echo_connect_ssh_message' - echo 'killall cryptsetup telnetd' + echo 'killall cryptsetup' + echo "pkill -f telnetd.*:${TELNET_PORT}" } >/telnet_connect.sh chmod +x /telnet_connect.sh telnetd -b "${IP}:${TELNET_PORT}" -l /telnet_connect.sh } -unlock_root_partition() { - # Wait for the root partition (and unlock it if it is encrypted) - while ! [ -e /sysroot/usr ]; do - partition="$(find_root_partition)" - if [ -z "$partition" ]; then - show_splash /splash-nosystem.ppm.gz - echo "Could not find the root partition." - echo "Maybe you need to insert the sdcard, if your device has" - echo "any? Trying again in one second..." - sleep 1 - elif cryptsetup isLuks "$partition"; then - start_usb_unlock - cryptsetup luksOpen "$partition" root || continue - partition="/dev/mapper/root" - break - else - # Unencrypted - break - fi - done - - # Mount the root partition - [ -e /sysroot/usr ] || mount -w -t ext4 "$partition" /sysroot -} - # $1: path to ppm.gz file show_splash() { gzip -c -d "$1" >/tmp/splash.ppm diff --git a/aports/main/postmarketos-mkinitfs/mkinitfs.sh b/aports/main/postmarketos-mkinitfs/mkinitfs.sh index eda7e21c..8d2e4efd 100644 --- a/aports/main/postmarketos-mkinitfs/mkinitfs.sh +++ b/aports/main/postmarketos-mkinitfs/mkinitfs.sh @@ -223,6 +223,10 @@ generate_splash_screens() pmos-make-splash --text="system partition not found\nhttps://postmarketos.org/troubleshooting" --center \ --config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-nosystem.ppm" gzip "${tmpdir}/splash-nosystem.ppm" + + pmos-make-splash --text="unable to mount root partition\nhttps://postmarketos.org/troubleshooting" --center \ + --config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-mounterror.ppm" + gzip "${tmpdir}/splash-mounterror.ppm" } # Append the correct device tree to the linux image file diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 3cbeda72..81e8ee1e 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -214,8 +214,6 @@ install_device_packages = [ # other "ttf-droid" ] -install_size_image = "835M" -install_size_boot = "100M" # diff --git a/pmb/install/blockdevice.py b/pmb/install/blockdevice.py index 779f834f..2ba9be92 100644 --- a/pmb/install/blockdevice.py +++ b/pmb/install/blockdevice.py @@ -46,7 +46,7 @@ def mount_sdcard(args): args.work + "/chroot_native/dev/install") -def create_and_mount_image(args): +def create_and_mount_image(args, size): # Short variables for paths chroot = args.work + "/chroot_native" img_path = "/home/user/rootfs/" + args.device + ".img" @@ -62,7 +62,6 @@ def create_and_mount_image(args): img_path_outside) # Create empty image file - size = pmb.config.install_size_image logging.info("(native) create " + args.device + ".img (" + size + ")") logging.info("WARNING: Make sure, that your target device's partition" " table has allocated at least " + size + " as system partition!") @@ -80,7 +79,7 @@ def create_and_mount_image(args): "/chroot_native/dev/install") -def create(args): +def create(args, size): """ Create /dev/install (the "install blockdevice"). """ @@ -89,4 +88,4 @@ def create(args): if args.sdcard: mount_sdcard(args) else: - create_and_mount_image(args) + create_and_mount_image(args, size) diff --git a/pmb/install/install.py b/pmb/install/install.py index cf2c73c8..9e566de2 100644 --- a/pmb/install/install.py +++ b/pmb/install/install.py @@ -30,13 +30,39 @@ import pmb.install.blockdevice import pmb.install -def copy_files(args): +def mount_device_rootfs(args): # Mount the device rootfs logging.info("(native) copy rootfs_" + args.device + " to" + " /mnt/install/") mountpoint = "/mnt/rootfs_" + args.device pmb.helpers.mount.bind(args, args.work + "/chroot_rootfs_" + args.device, args.work + "/chroot_native" + mountpoint) + return mountpoint + + +def get_chroot_size(args): + # Mount the device rootfs + mountpoint = mount_device_rootfs(args) + + # Run the du command + result = pmb.chroot.root(args, ["sh", "-c", "du -cm . | grep total$ | cut -f1"], + working_dir=mountpoint, return_stdout=True) + return result + + +def get_chroot_boot_size(args): + # Mount the device rootfs + mountpoint = mount_device_rootfs(args) + + # Run the du command + result = pmb.chroot.root(args, ["sh", "-c", "du -cm ./boot | grep total$ | cut -f1"], + working_dir=mountpoint, return_stdout=True) + return result + + +def copy_files(args): + # Mount the device rootfs + mountpoint = mount_device_rootfs(args) # Get all folders inside the device rootfs folders = [] @@ -111,14 +137,17 @@ def install(args): for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix): pmb.chroot.initfs.build(args, flavor, suffix) + size_image = str(int(get_chroot_size(args)) + 50) + "M" + size_boot = str(int(get_chroot_boot_size(args)) + 5) + "M" + # Finally set the user password set_user_password(args) # Partition and fill image/sdcard logging.info("*** (3/5) PREPARE INSTALL BLOCKDEVICE ***") pmb.chroot.shutdown(args, True) - pmb.install.blockdevice.create(args) - pmb.install.partition(args) + pmb.install.blockdevice.create(args, size_image) + pmb.install.partition(args, size_boot) pmb.install.format(args) # Just copy all the files diff --git a/pmb/install/partition.py b/pmb/install/partition.py index 35bff981..b28777e6 100644 --- a/pmb/install/partition.py +++ b/pmb/install/partition.py @@ -48,12 +48,11 @@ def partitions_mount(args): pmb.helpers.mount.bind_blockdevice(args, source, target) -def partition(args): +def partition(args, size_boot): """ Partition /dev/install and create /dev/install{p1,p2} """ - size_boot = pmb.config.install_size_boot logging.info("(native) partition /dev/install (boot: " + size_boot + ", root: the rest)") commands = [ diff --git a/test/static_code_analysis.sh b/test/static_code_analysis.sh index cc003efe..d4ea8a91 100755 --- a/test/static_code_analysis.sh +++ b/test/static_code_analysis.sh @@ -18,19 +18,26 @@ set -e DIR="$(cd "$(dirname "$0")" && pwd -P)" -cd "$DIR"/.. # Shell: shellcheck -sh_files="test/static_code_analysis.sh $(find . -name '*.trigger')" -echo "Test with shellcheck: $sh_files" +cd "$DIR"/.. +sh_files=" + ./test/static_code_analysis.sh + ./aports/main/postmarketos-mkinitfs/init.sh.in + ./aports/main/postmarketos-mkinitfs/init_functions.sh + $(find . -name '*.trigger') +" for file in ${sh_files}; do - shellcheck "${file}" + echo "Test with shellcheck: $file" + cd "$DIR/../$(dirname "$file")" + shellcheck -x "$(basename "$file")" done # Python: flake8 # E501: max line length # F401: imported, but not used, does not make sense in __init__ files # E402: module import not on top of file, not possible for testcases +cd "$DIR"/.. echo "Test with flake8: *.py" echo "NOTE: Run 'autopep8 -ria $PWD' to fix code style issues" py_files="$(find . -name '*.py')" @@ -42,4 +49,3 @@ flake8 --filename=__init__.py --ignore "F401,$_ignores" $py_files # Done echo "Success!" -