synit
/
pmbootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix #363: Omit /home/user when calculating system image size (#389) 

* The system image size is now calculated as: root size - home size.
* New function in `pmb/helpers/other.py`: `folder_size()`, with a
  testcase.
* Instead of copying everything to the system image folder, and
  deleting the home folder afterwards, do not copy the home folder
  in the first place.
* Added `pmbootstrap -s` to skip generating the initramfs for faster
  debugging.
* Set the default value in the "are you sure, that your partition has
  at least..." to "y", so we can run `yes '' | pmbootstrap install`
  to make it run through the whole installation process.
* Increase full size to 120%, boot partition gets 15 MB free space now
This commit is contained in:
Oliver Smith 2017-08-18 19:19:48 +00:00 committed by GitHub
parent 2de2bd5bee
commit 1c13ca4fd9
8 changed files with 152 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pmb/chroot/initfs.py
View File

@ -25,6 +25,11 @@ import pmb.helpers.cli
def build(args, flavor, suffix):
# Bail out when '-s' is set
if args.skip_initfs:
logging.info("NOTE: Skipped initramfs generation (-s)!")
return
# Update mkinitfs and hooks
pmb.chroot.apk.install(args, ["postmarketos-mkinitfs"], suffix)
pmb.chroot.initfs_hooks.update(args, suffix)

5
pmb/chroot/shutdown.py
View File

@ -68,6 +68,11 @@ def shutdown(args, only_install_related=False):
path = path_outside[len(chroot):]
pmb.install.losetup.umount(args, path)
# Umount device rootfs chroot
chroot_rootfs = args.work + "/chroot_rootfs_" + args.device
if os.path.exists(chroot_rootfs):
pmb.helpers.mount.umount_all(args, chroot_rootfs)
if not only_install_related:
# Clean up the rest
pmb.helpers.mount.umount_all(args, args.work)

15
pmb/helpers/other.py
View File

@ -17,6 +17,21 @@ You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import pmb.helpers.run
def folder_size(args, path):
"""
Run `du` to calculate the size of a folder (this is less code and
faster than doing the same task in pure Python).
:returns: folder size in bytes
"""
output = pmb.helpers.run.root(args, ["du", "--summarize",
"--block-size=1",
path], return_stdout=True)
ret = int(output.split("\t")[0])
return ret
def check_grsec(args):

20
pmb/install/blockdevice.py
View File

@ -47,6 +47,11 @@ def mount_sdcard(args):
def create_and_mount_image(args, size):
"""
Create a new image file, and mount it as /dev/install.
:param size: of the whole image in bytes
"""
# Short variables for paths
chroot = args.work + "/chroot_native"
img_path = "/home/user/rootfs/" + args.device + ".img"
@ -61,15 +66,18 @@ def create_and_mount_image(args, size):
raise RuntimeError("Failed to remove old image file: " +
img_path_outside)
# Create empty image file
logging.info("(native) create " + args.device + ".img (" + size + ")")
# Convert to MB and ask for confirmation
mb = str(round(size / 1024 / 1024)) + "M"
logging.info("(native) create " + args.device + ".img (" + mb + ")")
logging.info("WARNING: Make sure, that your target device's partition"
" table has allocated at least " + size + " as system partition!")
if not pmb.helpers.cli.confirm(args):
" table has allocated at least " + mb + " as system"
" partition!")
if not pmb.helpers.cli.confirm(args, default=True):
raise RuntimeError("Aborted.")
# Create empty image file
pmb.chroot.user(args, ["mkdir", "-p", "/home/user/rootfs"])
pmb.chroot.root(args, ["truncate", "-s", size, img_path])
pmb.chroot.root(args, ["truncate", "-s", mb, img_path])
# Mount to /dev/install
logging.info("(native) mount /dev/install (" + args.device + ".img)")
@ -82,6 +90,8 @@ def create_and_mount_image(args, size):
def create(args, size):
"""
Create /dev/install (the "install blockdevice").
:param size: of the whole image in bytes
"""
pmb.helpers.mount.umount_all(
args, args.work + "/chroot_native/dev/install")

88
pmb/install/install.py
View File

@ -26,65 +26,78 @@ import pmb.chroot.other
import pmb.chroot.initfs
import pmb.config
import pmb.helpers.run
import pmb.helpers.other
import pmb.install.blockdevice
import pmb.install
def mount_device_rootfs(args):
# Mount the device rootfs
logging.info("(native) copy rootfs_" + args.device + " to" +
" /mnt/install/")
mountpoint = "/mnt/rootfs_" + args.device
pmb.helpers.mount.bind(args, args.work + "/chroot_rootfs_" + args.device,
args.work + "/chroot_native" + mountpoint)
return mountpoint
def get_chroot_size(args):
def get_subpartitions_size(args):
"""
Calculate the size of the whole image and boot subpartition.
:returns: (full, boot) the size of the full image and boot
partition as integer in bytes
"""
# Calculate required sizes first
chroot = args.work + "/chroot_rootfs_" + args.device
root = pmb.helpers.other.folder_size(args, chroot)
boot = pmb.helpers.other.folder_size(args, chroot + "/boot")
home = pmb.helpers.other.folder_size(args, chroot + "/home")
# The home folder gets omitted when copying the rootfs to
# /dev/installp2
full = root - home
# Add some free space, see also:
# https://github.com/postmarketOS/pmbootstrap/pull/336
full *= 1.20
boot += 15 * 1024 * 1024
return (full, boot)
def copy_files_from_chroot(args):
"""
Copy all files from the rootfs chroot to /mnt/install, except
for the home folder (because /home will contain some empty
mountpoint folders).
"""
# Mount the device rootfs
logging.info("(native) copy rootfs_" + args.device + " to" +
" /mnt/install/")
mountpoint = mount_device_rootfs(args)
mountpoint_outside = args.work + "/chroot_native" + mountpoint
# Run the du command
result = pmb.chroot.root(args, ["sh", "-c", "du -cm . | grep total$ | cut -f1"],
working_dir=mountpoint, return_stdout=True)
return result
def get_chroot_boot_size(args):
# Mount the device rootfs
mountpoint = mount_device_rootfs(args)
# Run the du command
result = pmb.chroot.root(args, ["sh", "-c", "du -cm ./boot | grep total$ | cut -f1"],
working_dir=mountpoint, return_stdout=True)
return result
def copy_files(args):
# Mount the device rootfs
mountpoint = mount_device_rootfs(args)
# Get all folders inside the device rootfs
# Get all folders inside the device rootfs (except for home)
folders = []
for path in glob.glob(args.work + "/chroot_native" + mountpoint + "/*"):
for path in glob.glob(mountpoint_outside + "/*"):
if path.endswith("/home"):
continue
folders += [os.path.basename(path)]
# Run the copy command
pmb.chroot.root(args, ["cp", "-a"] + folders + ["/mnt/install/"],
working_dir=mountpoint)
# copy over keys and delete unneded mount folders
def fix_mount_folders(args):
# copy over keys
rootfs = args.work + "/chroot_native/mnt/install/"
def copy_files_other(args):
"""
Copy over keys, create /home/user.
"""
# Copy over keys
rootfs = args.work + "/chroot_native/mnt/install"
for key in glob.glob(args.work + "/config_apk_keys/*.pub"):
pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"])
# delete everything (-> empty mount folders) in /home/user
pmb.helpers.run.root(args, ["rm", "-r", rootfs + "/home/user"])
pmb.helpers.run.root(args, ["mkdir", rootfs + "/home/user"])
# Create /home/user
pmb.helpers.run.root(args, ["mkdir", "-p", rootfs + "/home/user"])
pmb.helpers.run.root(args, ["chown", pmb.config.chroot_uid_user,
rootfs + "/home/user"])
@ -153,27 +166,24 @@ def install(args):
for flavor in pmb.chroot.other.kernel_flavors_installed(args, suffix):
pmb.chroot.initfs.build(args, flavor, suffix)
size_image = str(int(float(get_chroot_size(args)) * 1.15)) + "M"
size_boot = str(int(get_chroot_boot_size(args)) + 5) + "M"
# Set the user password
set_user_password(args)
# Partition and fill image/sdcard
logging.info("*** (3/5) PREPARE INSTALL BLOCKDEVICE ***")
pmb.chroot.shutdown(args, True)
(size_image, size_boot) = get_subpartitions_size(args)
pmb.install.blockdevice.create(args, size_image)
pmb.install.partition(args, size_boot)
pmb.install.format(args)
# Just copy all the files
logging.info("*** (4/5) FILL INSTALL BLOCKDEVICE ***")
copy_files(args)
fix_mount_folders(args)
copy_files_from_chroot(args)
copy_files_other(args)
# If user has a ssh pubkey, offer to copy it to device
copy_ssh_key(args)
pmb.chroot.shutdown(args, True)
# Convert system image to sparse using img2simg

13
pmb/install/partition.py
View File

@ -51,14 +51,19 @@ def partitions_mount(args):
def partition(args, size_boot):
"""
Partition /dev/install and create /dev/install{p1,p2}
"""
logging.info("(native) partition /dev/install (boot: " + size_boot +
size_boot: size of the boot partition in bytes.
"""
# Convert to MB and print info
mb_boot = str(round(size_boot / 1024 / 1024)) + "M"
logging.info("(native) partition /dev/install (boot: " + mb_boot +
", root: the rest)")
# Actual partitioning with 'parted'
commands = [
["mktable", "msdos"],
["mkpart", "primary", "ext2", "2048s", size_boot],
["mkpart", "primary", size_boot, "100%"],
["mkpart", "primary", "ext2", "2048s", mb_boot],
["mkpart", "primary", mb_boot, "100%"],
["set", "1", "boot", "on"]
]
for command in commands:

3
pmb/parse/arguments.py
View File

@ -103,6 +103,9 @@ def arguments():
parser.add_argument("-j", "--jobs", help="parallel jobs when compiling")
parser.add_argument("-p", "--aports",
help="postmarketos aports paths")
parser.add_argument("-s", "--skip-initfs", dest="skip_initfs",
help="do not re-generate the initramfs",
action="store_true")
parser.add_argument("-w", "--work", help="folder where all data"
" gets stored (chroots, caches, built packages)")

51
test/test_folder_size.py Normal file
View File

@ -0,0 +1,51 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import sys
import pytest
# Import from parent directory
sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.helpers.logging
import pmb.helpers.other
import pmb.helpers.run
@pytest.fixture
def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.details_to_stdout = True
pmb.helpers.logging.init(args)
return args
def test_get_folder_size(args, tmpdir):
# Write five 2 KB files to tmpdir
tmpdir = str(tmpdir)
files = 5
for i in range(files):
pmb.helpers.run.user(args, ["dd", "if=/dev/zero", "of=" +
tmpdir + "/" + str(i), "bs=1K",
"count=2", "conv=notrunc"])
# Check if the size is correct
assert pmb.helpers.other.folder_size(args, tmpdir) == 20480