Split up initramfs to make it smaller (Close #211) (#257)

Huge thank you to @drebrez for his amazing work on this PR!

* Add generation of initramfs-extra with additional binaries
Extract both initramfs with `pmbootstrap initfs extract`
Add new splashscreens for missing partitions/files errors
Changes in init script:
 - use busybox findfs applet to find boot partition
 - mount boot partion
 - extract initramfs-extra
 - show error splashscreens accordingly and loop forever
 - start usb unlock directly from unlock_root_partition (hook removed)

* Print out a text message for serial debugging in case of errors
Add initramfs-extra files to `pmbootstrap initfs ls` output

* Fix trailing whitespace in comment...

* ls: Indicate which initramfs we're looking at / add wiki link

I've rewritten the initramfs-development article to reflect the
changes made in this PR. It will be a good read for someone who
extracted the initramfs and wants to know why we have two files.

aports/postmarketos-mkinitfs/10-usb-unlock.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-. ./init_functions.sh
-
-TELNET_PORT=23
-
-start_usb_unlock() {
-	# Only run if we have an encrypted partition
-	cryptsetup isLuks "$(find_root_partition)" || return
-
-	# Set up networking
-	setup_usb_network
-	start_udhcpd
-
-	# Telnet splash
-	show_splash /splash1.ppm.gz
-
-	# Start the telnet daemon
-	{
-		echo '#!/bin/sh'
-		echo '. /init_functions.sh'
-		echo 'unlock_root_partition'
-		echo 'echo_connect_ssh_message'
-		echo 'killall cryptsetup telnetd'
-	} >/telnet_connect.sh
-	chmod +x /telnet_connect.sh
-	telnetd -b "${IP}:${TELNET_PORT}" -l /telnet_connect.sh
-}
-
-start_usb_unlock

aports/postmarketos-mkinitfs/APKBUILD

@@ -1,12 +1,12 @@
 pkgname=postmarketos-mkinitfs
-pkgver=0.1.8
+pkgver=0.2.0
 pkgrel=1
 pkgdesc="Tool to generate initramfs images for postmarketOS"
 url="https://github.com/postmarketOS"
 # multipath-tools: kpartx
-depends="busybox-extras lddtree cryptsetup kmod multipath-tools postmarketos-splash"
+depends="busybox-extras lddtree cryptsetup kmod multipath-tools postmarketos-splash device-mapper parted e2fsprogs e2fsprogs-extra"
 triggers="$pkgname.trigger=/etc/postmarketos-mkinitfs/hooks:/usr/share/kernel/*"
-source="init.sh.in init_functions.sh mkinitfs.sh 10-usb-unlock.sh"
+source="init.sh.in init_functions.sh mkinitfs.sh"
 arch="noarch"
 license="GPL2"
 provides="mkinitfs=0.0.1"
@@ -24,10 +24,7 @@ package() {
 	install -Dm755 "$srcdir/mkinitfs.sh" \
 		"$pkgdir/sbin/mkinitfs"
 	mkdir -p "$pkgdir/etc/postmarketos-mkinitfs/hooks/"
-	install -Dm644 "$srcdir/10-usb-unlock.sh" \
-		"$pkgdir/etc/postmarketos-mkinitfs/hooks/"
 }
-sha512sums="0bb930b74acec8fd31bff4893a548e4be4a2fdebb79e10804381eb277fa3db95f3a963268ef79c36c8979371349488218a275c28385ace51dd0f5640c980880a  init.sh.in
+sha512sums="318f4d925df001c1f7378c1577332fba4964bbd822c1b050c41b908c2eb37365c418e1efb6f329cf51c515b34ef3e97d1b9e8cc85509f1cefea1015b3c9db663  init.sh.in
-6d130be59507ad6c755ce8dfe345335e54b3891ca484e7865246ce69fe9d4e6441247acf6d17cc2ff414aaa0aca13fc28c8956b90f1626b784429adc3b5bf866  init_functions.sh
+ccd05065a8f66f181351987740dff62b5a38f21aa764a3dbdb4601c494e756a5885ff17f69a6916eaae28498cd3c79d49e5962190ee0affd389f8f584dbaa3c2  init_functions.sh
-81d2782d0c4098dc503c52172a62ea90b6495d5a397f49c6a1544444e48f74e3645f17db6b710330ba70eb13defd81c47b9b5d1326146cbcd17bee59ff89b99d  mkinitfs.sh
+e0255a5e7debe41efe97a3a156ef866ec5e3e9f6d57d20c0acd889470501179a9eace709afa3b98d46d513c47113701ac5b17fbf116642e683972db91a4b4824  mkinitfs.sh"
-8129300894f0b91ba669c12df2e51fb8dd9a9366da26496523047782026714444f40037b88d3e56dc28e3049901c32bf2a4959390883dfbbff8cfa19bb82045f  10-usb-unlock.sh"

aports/postmarketos-mkinitfs/init.sh.in

@@ -24,12 +24,14 @@ done
 setup_usb_network
 start_udhcpd
 
-# Unlock root partition
+mount_boot_partition
+extract_initramfs_extra @INITRAMFS_EXTRA@
 unlock_root_partition
 
 # Switch root
-show_splash /splash2.ppm.gz
+show_splash /splash-loading.ppm.gz
 killall telnetd mdev 2>/dev/null
+umount /boot
 umount /proc
 umount /sys
 umount /dev/pts
@@ -41,6 +43,4 @@ exec switch_root /sysroot /sbin/init
 echo "ERROR: switch_root failed!"
 echo "Looping forever. Install and use the usb-shell hook to debug this."
 echo "For more information, see <https://postmarketos.org/usbhook>"
-while true; do
+loop_forever
-	sleep 1
-done

aports/postmarketos-mkinitfs/init_functions.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 # This file will be in /init_functions.sh inside the initramfs.
 IP=172.16.42.1
+TELNET_PORT=23
 
 # Redirect stdout and stderr to logfile
 setup_log() {
@@ -92,6 +93,31 @@ find_root_partition() {
 	echo "$DEVICE"
 }
 
+find_boot_partition() {
+	findfs LABEL="pmOS_boot"
+}
+
+mount_boot_partition() {
+	partition=$(find_boot_partition)
+	if [ -z "$partition" ]; then
+		echo "ERROR: boot partition not found!"
+		show_splash /splash-noboot.ppm.gz
+		loop_forever
+	fi
+	mount -r -t ext2 "$partition" /boot
+}
+
+# $1: initramfs-extra path
+extract_initramfs_extra() {
+	initramfs_extra="$1"
+	if [ ! -e "$initramfs_extra" ]; then
+		echo "ERROR: initramfs-extra not found!"
+		show_splash /splash-noinitramfsextra.ppm.gz
+		loop_forever
+	fi
+	gzip -d -c "$initramfs_extra" | cpio -i
+}
+
 setup_usb_network_android() {
 	# Only run, when we have the android usb driver
 	SYS=/sys/class/android_usb/android0
@@ -122,7 +148,7 @@ setup_usb_network_configfs() {
 	printf "%s" "rndis" > $CONFIGFS/g1/configs/c.1/strings/0x409/configuration
 
 	ln -s $CONFIGFS/g1/functions/rndis.usb0 $CONFIGFS/g1/configs/c.1
-	echo "$(ls /sys/class/udc)" > $CONFIGFS/g1/UDC
+	ls /sys/class/udc > $CONFIGFS/g1/UDC
 }
 
 setup_usb_network() {
@@ -160,16 +186,43 @@ start_udhcpd() {
 	udhcpd
 }
 
+start_usb_unlock() {
+	# Only run once
+	_marker="/tmp/_start_usb_unlock"
+	[ -e "$_marker" ] && return
+	touch "$_marker"
+
+	# Set up networking
+	setup_usb_network
+	start_udhcpd
+
+	# Telnet splash
+	show_splash /splash-telnet.ppm.gz
+
+	# Start the telnet daemon
+	{
+		echo '#!/bin/sh'
+		echo '. /init_functions.sh'
+		echo 'unlock_root_partition'
+		echo 'echo_connect_ssh_message'
+		echo 'killall cryptsetup telnetd'
+	} >/telnet_connect.sh
+	chmod +x /telnet_connect.sh
+	telnetd -b "${IP}:${TELNET_PORT}" -l /telnet_connect.sh
+}
+
 unlock_root_partition() {
 	# Wait for the root partition (and unlock it if it is encrypted)
 	while ! [ -e /sysroot/usr ]; do
 		partition="$(find_root_partition)"
 		if [ -z "$partition" ]; then
+			show_splash /splash-nosystem.ppm.gz
 			echo "Could not find the root partition."
 			echo "Maybe you need to insert the sdcard, if your device has"
 			echo "any? Trying again in one second..."
 			sleep 1
 		elif cryptsetup isLuks "$partition"; then
+			start_usb_unlock
 			cryptsetup luksOpen "$partition" root || continue
 			partition="/dev/mapper/root"
 			break
@@ -190,7 +243,13 @@ show_splash() {
 }
 
 echo_connect_ssh_message() {
-    echo "Your root partition has been decrypted successfully!"
+	echo "Your root partition has been decrypted successfully!"
-    echo "You can connect to your device using SSH in a few seconds:"
+	echo "You can connect to your device using SSH in a few seconds:"
-    echo "ssh user@$IP"
+	echo "ssh user@$IP"
+}
+
+loop_forever() {
+	while true; do
+		sleep 1
+	done
 }

aports/postmarketos-mkinitfs/mkinitfs.sh

@@ -16,11 +16,12 @@ parse_commandline()
 {
 	if [ "$1" != "-o" ]; then
 		echo "postmarketos-mkinitfs"
-		echo "usage: $(basename $0) -o OUTFILE KERNELVERSION"
+		echo "usage: $(basename "$0") -o OUTFILE KERNELVERSION"
 		exit 1
 	fi
 
 	outfile=$2
+	outfile_extra=$2-extra
 	kernel=$3
 	modules_path="/lib/modules/${kernel}"
 
@@ -33,7 +34,7 @@ parse_commandline()
 create_folders()
 {
 	for dir in /bin /sbin /usr/bin /usr/sbin /proc /sys /dev /tmp /lib \
-		/sysroot; do
+		/boot /sysroot; do
 		mkdir -p "$tmpdir$dir"
 	done
 }
@@ -67,9 +68,9 @@ get_modules_by_globs()
 	for glob in $globs; do
 		for file in /lib/modules/$kernel/$glob; do
 			if [ -d "$file" ]; then
-				find $file -type f
+				find "$file" -type f
 			elif [ -e "$file" ]; then
-				echo $file
+				echo "$file"
 			fi
 		done
 	done
@@ -100,38 +101,61 @@ get_modules()
 # Get the paths to all binaries and their dependencies
 get_binaries()
 {
-	BINARIES="/bin/busybox /bin/busybox-extras /sbin/cryptsetup /usr/sbin/telnetd /sbin/kpartx"
+	BINARIES="/bin/busybox /bin/busybox-extras /usr/sbin/telnetd /sbin/kpartx"
 	lddtree -l $BINARIES | sort -u
 }
 
-# Copy files to the same destination in the initramfs
+get_binaries_extra()
+{
+	BINARIES_EXTRA="
+		/sbin/cryptsetup
+		/sbin/dmsetup
+		/usr/sbin/parted
+		/sbin/e2fsck
+		/usr/sbin/resize2fs
+	"
+	tmp1=$(mktemp /tmp/mkinitfs.XXXXXX)
+	get_binaries > "$tmp1"
+	tmp2=$(mktemp /tmp/mkinitfs.XXXXXX)
+	lddtree -l $BINARIES_EXTRA | sort -u > "$tmp2"
+	ret=$(comm -13 "$tmp1" "$tmp2")
+	rm "$tmp1" "$tmp2"
+	echo "${ret}"
+}
+
+# Copy files to the destination specified
 # FIXME: this is a performance bottleneck
 # $1: files
+# $2: destination
 copy_files()
 {
 	for file in $1; do
-		install -Dm755 $file $tmpdir$file
+		install -Dm755 "$file" "$2$file"
 	done
 }
 
 create_device_nodes()
 {
-	mknod -m 666 $tmpdir/dev/null c 1 3
+	mknod -m 666 "$tmpdir/dev/null" c 1 3
-	mknod -m 644 $tmpdir/dev/random c 1 8
+	mknod -m 644 "$tmpdir/dev/random" c 1 8
-	mknod -m 644 $tmpdir/dev/urandom c 1 9
+	mknod -m 644 "$tmpdir/dev/urandom" c 1 9
 }
 
 replace_init_variables()
 {
-	sed -i "s:@MODULES@:${deviceinfo_modules_initfs} ext4:g" $tmpdir/init
+	sed -i "s:@MODULES@:${deviceinfo_modules_initfs} ext4:g" "$tmpdir/init"
+	sed -i "s:@INITRAMFS_EXTRA@:${outfile_extra}:g" "$tmpdir/init"
 }
 
+# Create a cpio image of the specified folder
+# $1: folder
+# $2: outfile
 create_cpio_image()
 {
-	cd "$tmpdir"
+	cd "$1"
 	find . -print0 \
 		| cpio --quiet -o -H newc \
-		| gzip -1 > "$outfile"
+		| gzip -1 > "$2"
 }
 
 # Legacy u-boot images
@@ -181,13 +205,24 @@ generate_splash_screens()
 	height=${deviceinfo_screen_height:-1280}
 
 	pmos-make-splash --text="On-screen keyboard is not implemented yet, plug in a USB cable and run on your PC:\ntelnet 172.16.42.1" \
-		--config /etc/postmarketos/splash.ini $width $height "${tmpdir}/splash1.ppm"
+		--config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-telnet.ppm"
+	gzip "${tmpdir}/splash-telnet.ppm"
 
 	pmos-make-splash --text="Loading..." --center \
-		--config /etc/postmarketos/splash.ini $width $height "${tmpdir}/splash2.ppm"
+		--config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-loading.ppm"
+	gzip "${tmpdir}/splash-loading.ppm"
 
-	gzip "${tmpdir}/splash1.ppm"
+	pmos-make-splash --text="boot partition not found\nhttps://postmarketos.org/troubleshooting" --center \
-	gzip "${tmpdir}/splash2.ppm"
+		--config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-noboot.ppm"
+	gzip "${tmpdir}/splash-noboot.ppm"
+
+	pmos-make-splash --text="initramfs-extra not found\nhttps://postmarketos.org/troubleshooting" --center \
+		--config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-noinitramfsextra.ppm"
+	gzip "${tmpdir}/splash-noinitramfsextra.ppm"
+
+	pmos-make-splash --text="system partition not found\nhttps://postmarketos.org/troubleshooting" --center \
+		--config /etc/postmarketos/splash.ini "$width" "$height" "${tmpdir}/splash-nosystem.ppm"
+	gzip "${tmpdir}/splash-nosystem.ppm"
 }
 
 # Append the correct device tree to the linux image file
@@ -202,15 +237,15 @@ append_device_tree()
 
 # initialize
 source_deviceinfo
-parse_commandline $1 $2 $3
+parse_commandline "$1" "$2" "$3"
 echo "==> initramfs: creating $outfile"
 tmpdir=$(mktemp -d /tmp/mkinitfs.XXXXXX)
 
 # set up initfs in temp folder
 create_folders
-copy_files "$(get_modules)"
+copy_files "$(get_modules)" "$tmpdir"
-copy_files "$(get_binaries)"
+copy_files "$(get_binaries)" "$tmpdir"
-copy_files "/etc/postmarketos-mkinitfs/hooks/*.sh"
+copy_files "/etc/postmarketos-mkinitfs/hooks/*.sh" "$tmpdir"
 create_device_nodes
 ln -s "/bin/busybox" "$tmpdir/bin/sh"
 install -Dm755 "/usr/share/postmarketos-mkinitfs/init.sh.in" \
@@ -221,10 +256,24 @@ install -Dm755 "/usr/share/postmarketos-mkinitfs/init_functions.sh" \
 # finish up
 generate_splash_screens
 replace_init_variables
-create_cpio_image
+create_cpio_image "$tmpdir" "$outfile"
 append_device_tree
 create_uboot_files
 create_bootimg
 
 rm -rf "$tmpdir"
+
+# initialize initramfs-extra
+echo "==> initramfs: creating $outfile_extra"
+tmpdir_extra=$(mktemp -d /tmp/mkinitfs.XXXXXX)
+
+# set up initfs-extra in temp folder
+mkdir -p "$tmpdir_extra"
+copy_files "$(get_binaries_extra)" "$tmpdir_extra"
+
+# finish up
+create_cpio_image "$tmpdir_extra" "$outfile_extra"
+
+rm -rf "$tmpdir_extra"
+
 exit 0

pmb/chroot/initfs.py

@@ -39,13 +39,16 @@ def build(args, flavor, suffix):
                     suffix)
 
 
-def extract(args, flavor, suffix, log_message=False):
+def extract(args, flavor, suffix, extra=False):
     """
-    Extract the initramfs to /tmp/initfs-extracted and return the outside
+    Extract the initramfs to /tmp/initfs-extracted or the initramfs-extra to
-    extraction path.
+    /tmp/initfs-extra-extracted and return the outside extraction path.
     """
     # Extraction folder
     inside = "/tmp/initfs-extracted"
+    if extra:
+        inside = "/tmp/initfs-extra-extracted"
+        flavor += "-extra"
     outside = args.work + "/chroot_" + suffix + inside
     if os.path.exists(outside):
         if not pmb.helpers.cli.confirm(args, "Extraction folder " + outside +
@@ -75,9 +78,11 @@ def extract(args, flavor, suffix, log_message=False):
     return outside
 
 
-def ls(args, flavor, suffix):
+def ls(args, flavor, suffix, extra=False):
     tmp = "/tmp/initfs-extracted"
-    extract(args, flavor, suffix)
+    if extra:
+        tmp = "/tmp/initfs-extra-extracted"
+    extract(args, flavor, suffix, extra)
     pmb.chroot.user(args, ["ls", "-lahR", "."], suffix, tmp, log=False)
     pmb.chroot.root(args, ["rm", "-r", tmp], suffix)
 
@@ -95,9 +100,14 @@ def frontend(args):
         build(args, flavor, suffix)
     elif action == "extract":
         dir = extract(args, flavor, suffix)
-        logging.info("Successfully extracted to: " + dir)
+        logging.info("Successfully extracted initramfs to: " + dir)
+        dir_extra = extract(args, flavor, suffix, True)
+        logging.info("Successfully extracted initramfs-extra to: " + dir_extra)
     elif action == "ls":
+        logging.info("*** initramfs ***")
         ls(args, flavor, suffix)
+        logging.info("*** initramfs-extra ***")
+        ls(args, flavor, suffix, True)
 
     # Handle hook actions
     elif action == "hook_ls":
@@ -111,3 +121,7 @@ def frontend(args):
         # Rebuild the initfs for all kernels after adding/removing a hook
         for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix):
             build(args, flavor, suffix)
+
+    if action in ["ls", "extract"]:
+        link = "https://github.com/postmarketOS/pmbootstrap/wiki/initramfs-development"
+        logging.info("See also: <" + link + ">")