parse.apkindex: fix typo

CI apparently enforces spellcheck for comments now, so this also fixes
CI :D

.b4-config

@@ -0,0 +1,9 @@
+# Allow this repository to be used with the 'b4' tool. See
+# https://postmarketos.org/patch-review for details.
+
+[b4]
+  midmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%s
+  linkmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%%3C%s%%3E
+  send-series-to = ~postmarketos/pmbootstrap-devel@lists.sr.ht
+  send-endpoint-web = NONE
+  backend = sourcehut

.build.yml

@@ -0,0 +1,26 @@
+image: alpine/edge
+packages:
+  - sudo
+sources:
+  - https://git.sr.ht/~postmarketos/pmbootstrap
+tasks:
+  - note: |
+      pmbootstrap/.ci/note.sh
+  - shellcheck: |
+      cd pmbootstrap
+      sudo .ci/shellcheck.sh
+  - ruff: |
+      cd pmbootstrap
+      sudo .ci/ruff.sh
+  - vermin: |
+      cd pmbootstrap
+      sudo .ci/vermin.sh
+  - codespell: |
+      cd pmbootstrap
+      sudo .ci/codespell.sh
+  - pytest: |
+      cd pmbootstrap
+      sudo .ci/pytest.sh
+artifacts:
+  - ".local/var/pmbootstrap/log.txt"
+  - ".local/var/pmbootstrap/log_testsuite.txt"

.ci/codespell.sh

@@ -0,0 +1,20 @@
+#!/bin/sh -ex
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Copyright 2023 Oliver Smith
+# Description: find typos
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add \
+		py3-codespell
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+set -x
+
+# -L: words to ignore
+codespell \
+	-L crate \
+	-L hda \
+	.

.ci/note.sh

@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+printf "\n"
+printf "PROTIP: use"
+printf " \e[1;32mpmbootstrap ci\e[0m"
+printf " to run these scripts locally.\n"

.ci/prepare.sh

@@ -1,36 +0,0 @@
-#!/bin/sh -e
-# Install pmbootstrap depends, set up pmos user with sudo
-if [ "$(id -u)" != 0 ]; then
-	echo "ERROR: this script is meant to be executed in the gitlab-ci"
-	echo "environment only."
-	exit 1
-fi
-
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
-
-ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
-
-apk add -q \
-	git \
-	openssl \
-	py3-pytest \
-	py3-pytest-cov \
-	sudo
-
-adduser -D pmos
-chown -R pmos:pmos .
-echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-su pmos -c "git config --global user.email postmarketos-ci@localhost"
-su pmos -c "git config --global user.name postmarketOS_CI"
-
-echo "Initializing pmbootstrap"
-if ! su pmos -c "yes '' | pmbootstrap \
-		--details-to-stdout \
-		init \
-		>/tmp/pmb_init 2>&1"; then
-	cat /tmp/pmb_init
-	exit 1
-fi
-echo ""

.ci/pytest.sh

@@ -1,6 +1,39 @@
 #!/bin/sh -e
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
+# Description: run pmbootstrap python testsuite
+# Options: native slow
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add \
+		git \
+		openssl \
+		py3-pytest \
+		py3-pytest-cov \
+		sudo
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+# Require pytest to be installed on the host system
+if [ -z "$(command -v pytest)" ]; then
+	echo "ERROR: pytest command not found, make sure it is in your PATH."
+	exit 1
+fi
+
+# Use pytest-cov if it is installed to display code coverage
+cov_arg=""
+if python -c "import pytest_cov" >/dev/null 2>&1; then
+	cov_arg="--cov=pmb"
+fi
+
+echo "Initializing pmbootstrap..."
+if ! yes '' | ./pmbootstrap.py \
+		--details-to-stdout \
+		init \
+		>/tmp/pmb_init 2>&1; then
+	cat /tmp/pmb_init
+	exit 1
+fi
 
 # Make sure that the work folder format is up to date, and that there are no
 # mounts from aborted test cases (#1595)
@@ -21,4 +54,19 @@ if ! [ -e "$deviceinfo" ]; then
 	exit 1
 fi
 
-pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"
+# Make sure pmaports is clean, some of the tests will fail otherwise
+if [ -n "$(git -C "$pmaports" status --porcelain)" ]; then
+	echo "ERROR: pmaports dir is not clean"
+	exit 1
+fi
+
+echo "Running pytest..."
+echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
+pytest \
+	--color=yes \
+	-vv \
+	-x \
+	$cov_arg \
+	test \
+		-m "not skip_ci" \
+		"$@"

.ci/ruff.sh

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+# Description: lint all python scripts
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add ruff
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+set -x
+
+# __init__.py with additional ignore:
+# F401: imported, but not used
+# shellcheck disable=SC2046
+ruff --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py')
+
+# Check all other files
+ruff  --exclude=__init__.py .

.ci/shellcheck.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -e
+# Description: lint all shell scripts
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add shellcheck
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+find . -name '*.sh' |
+while read -r file; do
+	echo "shellcheck: $file"
+	shellcheck "$file"
+done

.ci/vermin.sh

@@ -1,17 +1,23 @@
 #!/bin/sh -e
-_vermin() {
-	if ! vermin -q "$@" >/dev/null 2>&1; then
-		vermin -vv "$@"
-	fi
-}
+# Description: verify that we don't use too new python features
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add vermin
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
 
 # shellcheck disable=SC2046
-_vermin \
-	-t=3.6- \
+vermin \
+	-t=3.7- \
 	--backport argparse \
 	--backport configparser \
 	--backport enum \
 	--backport typing \
+	--lint \
+	--no-parse-comments \
+	--eval-annotations \
 	$(find . -name '*.py' \
 		-a -not -path "./.venv/*" \
 		-a -not -path "./venv/*")

.gitlab-ci.yml

@@ -1,66 +0,0 @@
----
-# Author: Clayton Craft <clayton@craftyguy.net>
-
-image: alpine:latest
-
-# defaults for "only"
-# We need to run the CI jobs in a "merge request specific context", if CI is
-# running in a merge request. Otherwise the environment variable that holds the
-# merge request ID is not available. This means, we must set the "only"
-# variable accordingly - and if we only do it for one job, all other jobs will
-# not get executed. So have the defaults here, and use them in all jobs that
-# should run on both the master branch, and in merge requests.
-# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
-.only-default: &only-default
-  only:
-    - master
-    - merge_requests
-    - tags
-
-static-code-analysis:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c "test/static_code_analysis.sh"
-
-vermin:
-  image: alpine:latest
-  <<: *only-default
-  before_script:
-  - "apk -q add py3-pip"
-  - "pip3 -q --disable-pip-version-check install vermin"
-  script:
-  - ".ci/vermin.sh"
-
-# MR settings
-# (Checks for "Allow commits from members who can merge to the target branch")
-mr-settings:
-  only:
-    - merge_requests
-  before_script:
-    - apk -q add python3
-  script:
-    - wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
-    - python3 ./check_mr_settings.py
-
-pytest:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c .ci/pytest.sh
-  after_script:
-     # Move logs so it can be saved as artifacts
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
-    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
-    - "dmesg > $CI_PROJECT_DIR/dmesg.txt"
-  artifacts:
-    when: always
-    paths:
-      - "log.txt"
-      - "log_testsuite.txt"
-      - "dmesg.txt"
-      - "pmbootstrap.cfg"
-    expire_in: 1 week

README.md

@@ -1,22 +1,62 @@
 # pmbootstrap
-[**Introduction**](https://postmarketos.org/blog/2017/05/26/intro/) | [**Security Warning**](https://ollieparanoid.github.io/post/security-warning/) | [**Devices**](https://wiki.postmarketos.org/wiki/Devices)
 
-Sophisticated chroot/build/flash tool to develop and install [postmarketOS](https://postmarketos.org).
+Sophisticated chroot/build/flash tool to develop and install
+[postmarketOS](https://postmarketos.org).
 
-Package build scripts live in the [`pmaports`](https://gitlab.com/postmarketOS/pmaports) repository now.
+## Development
+
+pmbootstrap is being developed on SourceHut
+([what](https://postmarketos.org/blog/2022/07/25/considering-sourcehut/)):
+
+https://git.sr.ht/~postmarketos/pmbootstrap
+
+Send patches via mail or web UI to
+[pmbootstrap-devel](https://lists.sr.ht/~postmarketos/pmbootstrap-devel)
+([subscribe](mailto:~postmarketos/pmbootstrap-devel+subscribe@lists.sr.ht)):
+```
+~postmarketos/pmbootstrap-devel@lists.sr.ht
+```
+
+You can set the default values for sending email in the git checkout
+```
+$ git config sendemail.to "~postmarketos/pmbootstrap-devel@lists.sr.ht"
+$ git config format.subjectPrefix "PATCH pmbootstrap"
+```
+
+Run CI scripts locally with:
+```
+$ pmbootstrap ci
+```
+
+Run a single test file:
+```
+$ pytest -vv ./test/test_keys.py
+```
+
+## Issues
+
+Issues are being tracked
+[here](https://gitlab.com/postmarketOS/pmbootstrap/-/issues).
 
 ## Requirements
-* 2 GB of RAM recommended for compiling
-* Linux distribution on the host system (`x86`, `x86_64`, or `aarch64`)
-  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
-  * Kernels based on the grsec patchset [do **not** work](https://github.com/postmarketOS/pmbootstrap/issues/107)
+* Linux distribution on the host system (`x86`, `x86_64`, `aarch64` or `armv7`)
+  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+    does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
   * [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
-* Python 3.6+
+* Python 3.7+
 * OpenSSL
 * git
+* ps
+* tar
 
 ## Usage Examples
-Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for in-depth coverage of topics such as [porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The help output (`pmbootstrap -h`) has detailed usage instructions for every command. Read on for some generic examples of what can be done with `pmbootstrap`.
+Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for
+in-depth coverage of topics such as
+[porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device)
+or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The
+help output (`pmbootstrap -h`) has detailed usage instructions for every
+command. Read on for some generic examples of what can be done with
+`pmbootstrap`.
 
 ### Installing pmbootstrap
 <https://wiki.postmarketos.org/wiki/Installing_pmbootstrap>
@@ -63,6 +103,27 @@ Generate a template for a new package:
 $ pmbootstrap newapkbuild "https://gitlab.com/postmarketOS/osk-sdl/-/archive/0.52/osk-sdl-0.52.tar.bz2"
 ```
 
+#### Default architecture
+
+Packages will be compiled for the architecture of the device running
+pmbootstrap by default. For example, if your `x86_64` PC runs pmbootstrap, it
+would build a package for `x86_64` with this command:
+```
+$ pmbootstrap build hello-world
+```
+
+If you would rather build for the target device selected in `pmbootstrap init`
+by default, then use the `build_default_device_arch` option:
+```
+$ pmbootstrap config build_default_device_arch True
+```
+
+If your target device is `pine64-pinephone` for example, pmbootstrap will now
+build this package for `aarch64`:
+```
+$ pmbootstrap build hello-world
+```
+
 ### Chroots
 Enter the `armhf` building chroot:
 ```
@@ -80,7 +141,9 @@ $ pmbootstrap zap
 ```
 
 ### Device Porting Assistance
-Analyze Android [`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also works with recovery OS images like TWRP):
+Analyze Android
+[`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also
+works with recovery OS images like TWRP):
 ```
 $ pmbootstrap bootimg_analyze ~/Downloads/twrp-3.2.1-0-fp2.img
 ```
@@ -144,12 +207,14 @@ List pmaports that don't have a binary package:
 $ pmbootstrap repo_missing --arch=armhf --overview
 ```
 
-Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps):
+Increase the `pkgrel` for each aport where the binary package has outdated
+dependencies (e.g. after soname bumps):
 ```
 $ pmbootstrap pkgrel_bump --auto
 ```
 
-Generate cross-compiler aports based on the latest version from Alpine's aports:
+Generate cross-compiler aports based on the latest version from Alpine's
+aports:
 ```
 $ pmbootstrap aportgen binutils-armhf gcc-armhf
 ```
@@ -190,44 +255,32 @@ $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hel
 $ pmbootstrap stats --arch=armhf
 ```
 
-`distccd` log:
-```
-$ pmbootstrap log_distccd
-```
+### Use alternative sudo
 
-## Development
-### Requirements for running tests
-* [Shellcheck](https://shellcheck.net/)
+pmbootstrap supports `doas` and `sudo`.
+If multiple sudo implementations are installed, pmbootstrap will use `doas`.
+You can set the `PMB_SUDO` environmental variable to define the sudo
+implementation you want to use.
 
-You also need to install the following python packages (pip can be useful if you distribution hasn't got them packaged):
-* `pytest`
-* `pytest-cov`
-* `flake8`
+### Select SSH keys to include and make authorized in new images
 
-On Alpine Linux it can be done with:
-```shell
-$ sudo apk add grep shellcheck py3-pytest py3-pytest-cov py3-flake8
-```
+If the config file option `ssh_keys` is set to `True` (it defaults to `False`),
+then all files matching the glob `~/.ssh/id_*.pub` will be placed in
+`~/.ssh/authorized_keys` in the user's home directory in newly-built images.
 
-### Running linters
-The easiest way is to run the same script CI runs:
-```shell
-$ ./test/static_code_analysis.sh
-```
+Sometimes, for example if you have a large number of SSH keys, you may wish to
+select a different set of public keys to include in an image. To do this, set
+the `ssh_key_glob` configuration parameter in the pmbootstrap config file to a
+string containing a glob that is to match the file or files you wish to
+include.
 
-### Running tests
-You can now run `pytest -vv` inside the pmbootstrap folder to run all available tests.
+For example, a `~/.config/pmbootstrap.cfg` may contain:
 
-CI runs slightly reduces set of tests (it skips tests that require running qemu) by this:
-```shell
-$ .ci/pytest.sh
-```
-This is the easiest way to do the same as CI.
-
-Alternatively you can run a single test file if you wish:
-```shell
-$ pytest -vv ./test/test_keys.py
-```
+    [pmbootstrap]
+    # ...
+    ssh_keys = True
+    ssh_key_glob = ~/.ssh/postmarketos-dev.pub
+    # ...
 
 ## License
 [GPLv3](LICENSE)

helpers/envkernel.sh

@@ -49,14 +49,14 @@ export_pmbootstrap_dir() {
 	# See also: <https://stackoverflow.com/a/29835459>
 	# shellcheck disable=SC3054
 	if [ -n "${BASH_SOURCE[0]}" ]; then
-		script_dir="$(dirname "${BASH_SOURCE[0]}")"
+		script_dir="$(dirname "$(realpath "$BASH_SOURCE")")"
 	else
 		script_dir="$(dirname "$1")"
 	fi
 
 	# Fail with debug information
 	# shellcheck disable=SC2155
-	export pmbootstrap_dir=$(realpath "$script_dir/..")
+	export pmbootstrap_dir="$(realpath "$script_dir/..")"
 	if ! [ -e "$pmbootstrap_dir/pmbootstrap.py" ]; then
 		echo "ERROR: Failed to get the script's location with your shell."
 		echo "Please adjust export_pmbootstrap_dir in envkernel.sh. Debug info:"
@@ -120,7 +120,9 @@ initialize_chroot() {
 	# shellcheck disable=SC3057
 	arch_substr="${host_arch:0:3}"
 	if [ "$arch" = "$host_arch" ] || \
-		{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; }; then
+		{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; } || \
+		{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; } || \
+		{ [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; }; then
 		need_cross_compiler=0
 	fi
 
@@ -167,7 +169,9 @@ initialize_chroot() {
 		musl-dev \
 		ncurses-dev \
 		perl \
+		py3-dt-schema \
 		sed \
+		yamllint \
 		yaml-dev \
 		xz || return 1
 
@@ -215,9 +219,14 @@ set_alias_make() {
 		cross_compiler="/usr/bin/$prefix-"
 	fi
 
+	if [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; then
+		cc=$hostcc
+	fi
+
 	# Build make command
 	cmd="echo '*** pmbootstrap envkernel.sh active for $PWD! ***';"
 	cmd="$cmd pmbootstrap -q chroot --user --"
+	cmd="$cmd CCACHE_DISABLE=1"
 	cmd="$cmd ARCH=$arch"
 	if [ "$need_cross_compiler" = 1 ]; then
 		cmd="$cmd CROSS_COMPILE=$cross_compiler"
@@ -259,7 +268,7 @@ set_alias_pmbroot_kernelroot() {
 
 cross_compiler_version() {
 	if [ "$need_cross_compiler" = 1 ]; then
-		pmbootstrap chroot --user -- "${cross_compiler}gcc"  --version \
+		"$pmbootstrap" chroot --user -- "${cross_compiler}gcc"  --version \
 			2> /dev/null | grep "^.*gcc " | \
 			awk -F'[()]' '{ print $1 "("$2")" }'
 	else
@@ -306,7 +315,7 @@ set_reactivate() {
 
 check_and_deactivate() {
 	if [ "$POSTMARKETOS_ENVKERNEL_ENABLED" = 1 ]; then
-		# we already are runnning in envkernel
+		# we already are running in envkernel
 		deactivate
 	fi
 }
@@ -315,9 +324,7 @@ check_and_deactivate() {
 print_usage() {
 	# shellcheck disable=SC3054
 	if [ -n "${BASH_SOURCE[0]}" ]; then
-		echo "usage: source $(basename "${BASH_SOURCE[0]}")"
-	else
-		echo "usage: source $(basename "$1")"
+		echo "usage: source $(basename "$(realpath "$BASH_SOURCE")")"
 	fi
 	echo "optional arguments:"
 	echo "    --fish        Print fish alias syntax (internally used)"
@@ -403,7 +410,7 @@ main() {
 
 # Print fish alias syntax (when called from envkernel.fish)
 fish_compat() {
-	[ "$1" = "--fish" ] || return
+	[ "$1" = "--fish" ] || return 0
 	for name in make kernelroot pmbootstrap pmbroot; do
 		echo "alias $(alias $name | sed 's/=/ /')"
 	done

pmb/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 # PYTHON_ARGCOMPLETE_OK
 import sys
@@ -14,6 +14,17 @@ from .helpers import logging as pmb_logging
 from .helpers import mount
 from .helpers import other
 
+# pmbootstrap version
+__version__ = "2.0.0"
+
+# Python version check
+version = sys.version_info
+if version < (3, 7):
+    print("You need at least Python 3.7 to run pmbootstrap")
+    print("(You are running it with Python " + str(version.major) +
+          "." + str(version.minor) + ")")
+    sys.exit()
+
 
 def main():
     # Wrap everything to display nice error messages
@@ -23,8 +34,11 @@ def main():
         args = parse.arguments()
         os.umask(0o22)
 
+        # Store script invocation command
+        os.environ["PMBOOTSTRAP_CMD"] = sys.argv[0]
+
         # Sanity checks
-        other.check_grsec(args)
+        other.check_grsec()
         if not args.as_root and os.geteuid() == 0:
             raise RuntimeError("Do not run pmbootstrap as root!")
 
@@ -56,6 +70,10 @@ def main():
                          " shutdown' as necessary)")
         logging.info("DONE!")
 
+    except KeyboardInterrupt:
+        print("\nCaught KeyboardInterrupt, exiting …")
+        sys.exit(130)  # SIGINT(2) + 128
+
     except Exception as e:
         # Dump log to stdout when args (and therefore logging) init failed
         if not args:
@@ -71,7 +89,14 @@ def main():
             log_hint += (" Alternatively you can use '--details-to-stdout' to"
                          " get more output, e.g. 'pmbootstrap"
                          " --details-to-stdout init'.")
+        print()
         print(log_hint)
+        print()
+        print("Before you report this error, ensure that pmbootstrap is "
+              "up to date.")
+        print("Find the latest version here:"
+              " https://git.sr.ht/~postmarketos/pmbootstrap/refs")
+        print(f"Your version: {__version__}")
         return 1
 
 

pmb/aportgen/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/aportgen/binutils.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.helpers.git
@@ -13,48 +13,21 @@ def generate(args, pkgname):
 
     # Rewrite APKBUILD
     fields = {
-        "pkgname": pkgname,
+        "arch": pmb.config.arch_native,
+        "makedepends_host": "zlib-dev jansson-dev zstd-dev",
         "pkgdesc": f"Tools necessary to build programs for {arch} targets",
-        "arch": args.arch_native,
-        "makedepends_build": "",
-        "makedepends_host": "",
-        "makedepends": "gettext libtool autoconf automake bison texinfo",
-        "subpackages": "",
+        "pkgname": pkgname,
     }
 
-    replace_functions = {
-        "build": """
-            _target="$(arch_to_hostspec """ + arch + """)"
-            "$builddir"/configure \\
-                --build="$CBUILD" \\
-                --target=$_target \\
-                --with-lib-path=/usr/lib \\
-                --prefix=/usr \\
-                --with-sysroot=/usr/$_target \\
-                --enable-ld=default \\
-                --enable-gold=yes \\
-                --enable-plugins \\
-                --enable-deterministic-archives \\
-                --disable-multilib \\
-                --disable-werror \\
-                --disable-nls
-            make
-        """,
-        "package": """
-            make install DESTDIR="$pkgdir"
-
-            # remove man, info folders
-            rm -rf "$pkgdir"/usr/share
-
-            # remove files that conflict with non-cross binutils
-            rm -rf "$pkgdir"/usr/lib/bfd-plugins
-        """,
-        "libs": None,
-        "gold": None,
+    replace_simple = {
+        "*--with-bugurl=*": "\t\t--with-bugurl=\"https://postmarketos.org/issues\" \\"
     }
 
-    replace_simple = {"\tsubpackages=*": "\tsubpackages=\"\""}
+    below_header = """
+        CTARGET_ARCH=""" + arch + """
+        CTARGET="$(arch_to_hostspec $CTARGET_ARCH)"
+    """
 
     pmb.aportgen.core.rewrite(args, pkgname, "main/binutils", fields,
-                              "binutils", replace_functions, replace_simple,
-                              remove_indent=8)
+                              "binutils", replace_simple=replace_simple,
+                              below_header=below_header)

pmb/aportgen/busybox_static.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -47,7 +47,7 @@ def generate(args, pkgname):
 
             url="http://busybox.net"
             license="GPL2"
-            arch="{args.arch_native}"
+            arch="{pmb.config.arch_native}"
             options="!check !strip"
             pkgdesc="Statically linked Busybox for $_arch"
             _target="$(arch_to_hostspec $_arch)"
@@ -67,7 +67,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/core.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fnmatch
 import logging
@@ -189,7 +189,7 @@ def get_upstream_aport(args, pkgname, arch=None):
     aport_path = paths[0]
 
     # Parse APKBUILD
-    apkbuild = pmb.parse.apkbuild(args, aport_path + "/APKBUILD",
+    apkbuild = pmb.parse.apkbuild(f"{aport_path}/APKBUILD",
                                   check_pkgname=False)
     apkbuild_version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
 
@@ -205,16 +205,18 @@ def get_upstream_aport(args, pkgname, arch=None):
     if compare == 0:
         return aport_path
 
-    # Different version message
-    logging.error("ERROR: Package '" + pkgname + "' has a different version in"
+    # APKBUILD > binary: this is fine
+    if compare == 1:
+        logging.info(f"NOTE: {pkgname} {arch} binary package has a lower"
+                     f" version {package['version']} than the APKBUILD"
+                     f" {apkbuild_version}")
+        return aport_path
+
+    # APKBUILD < binary: aports.git is outdated
+    logging.error("ERROR: Package '" + pkgname + "' has a lower version in"
                   " local checkout of Alpine's aports (" + apkbuild_version +
                   ") compared to Alpine's binary package (" +
                   package["version"] + ")!")
 
-    # APKBUILD < binary
-    if compare == -1:
-        raise RuntimeError("You can update your local checkout with: "
-                           "'pmbootstrap pull'")
-    # APKBUILD > binary
-    raise RuntimeError("You can force an update of your binary package"
-                       " APKINDEX files with: 'pmbootstrap update'")
+    raise RuntimeError("You can update your local checkout with: "
+                       "'pmbootstrap pull'")

pmb/aportgen/device.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -9,14 +9,14 @@ import pmb.parse.apkindex
 import pmb.parse.bootimg
 
 
-def ask_for_architecture(args):
+def ask_for_architecture():
     architectures = pmb.config.build_device_architectures
     # Don't show armhf, new ports shouldn't use this architecture
     if "armhf" in architectures:
         architectures.remove("armhf")
     while True:
-        ret = pmb.helpers.cli.ask(args, "Device architecture", architectures,
-                                  architectures[0], complete=architectures)
+        ret = pmb.helpers.cli.ask("Device architecture", architectures,
+                                  "aarch64", complete=architectures)
         if ret in architectures:
             return ret
         logging.fatal("ERROR: Invalid architecture specified. If you want to"
@@ -25,14 +25,14 @@ def ask_for_architecture(args):
                       " pmb/config/__init__.py.")
 
 
-def ask_for_manufacturer(args):
+def ask_for_manufacturer():
     logging.info("Who produced the device (e.g. LG)?")
-    return pmb.helpers.cli.ask(args, "Manufacturer", None, None, False)
+    return pmb.helpers.cli.ask("Manufacturer", None, None, False)
 
 
-def ask_for_name(args, manufacturer):
+def ask_for_name(manufacturer):
     logging.info("What is the official name (e.g. Google Nexus 5)?")
-    ret = pmb.helpers.cli.ask(args, "Name", None, None, False)
+    ret = pmb.helpers.cli.ask("Name", None, None, False)
 
     # Always add the manufacturer
     if not ret.startswith(manufacturer) and \
@@ -41,19 +41,19 @@ def ask_for_name(args, manufacturer):
     return ret
 
 
-def ask_for_year(args):
+def ask_for_year():
     # Regex from https://stackoverflow.com/a/12240826
     logging.info("In what year was the device released (e.g. 2012)?")
-    return pmb.helpers.cli.ask(args, "Year", None, None, False,
+    return pmb.helpers.cli.ask("Year", None, None, False,
                                validation_regex=r'^[1-9]\d{3,}$')
 
 
-def ask_for_chassis(args):
+def ask_for_chassis():
     types = pmb.config.deviceinfo_chassis_types
 
     logging.info("What type of device is it?")
     logging.info("Valid types are: " + ", ".join(types))
-    return pmb.helpers.cli.ask(args, "Chassis", None, None, True,
+    return pmb.helpers.cli.ask("Chassis", None, None, True,
                                validation_regex='|'.join(types),
                                complete=types)
 
@@ -68,12 +68,13 @@ def ask_for_external_storage(args):
                                    " other external storage medium?")
 
 
-def ask_for_flash_method(args):
+def ask_for_flash_method():
     while True:
         logging.info("Which flash method does the device support?")
-        method = pmb.helpers.cli.ask(args, "Flash method",
+        method = pmb.helpers.cli.ask("Flash method",
                                      pmb.config.flash_methods,
-                                     pmb.config.flash_methods[0])
+                                     "none",
+                                     complete=pmb.config.flash_methods)
 
         if method in pmb.config.flash_methods:
             if method == "heimdall":
@@ -84,7 +85,7 @@ def ask_for_flash_method(args):
                     logging.info("<https://wiki.postmarketos.org/wiki"
                                  "/Deviceinfo_flash_methods#Isorec_or_bootimg"
                                  ".3F>")
-                    heimdall_type = pmb.helpers.cli.ask(args, "Type",
+                    heimdall_type = pmb.helpers.cli.ask("Type",
                                                         heimdall_types,
                                                         heimdall_types[0])
                     if heimdall_type in heimdall_types:
@@ -106,7 +107,7 @@ def ask_for_bootimg(args):
                  " 'pmbootstrap bootimg_analyze').")
 
     while True:
-        response = pmb.helpers.cli.ask(args, "Path", None, "", False)
+        response = pmb.helpers.cli.ask("Path", None, "", False)
         path = os.path.expanduser(response)
         if not path:
             return None
@@ -116,7 +117,7 @@ def ask_for_bootimg(args):
             logging.fatal("ERROR: " + str(e) + ". Please try again.")
 
 
-def generate_deviceinfo_fastboot_content(args, bootimg=None):
+def generate_deviceinfo_fastboot_content(bootimg=None):
     if bootimg is None:
         bootimg = {"cmdline": "",
                    "qcdt": "false",
@@ -142,7 +143,14 @@ def generate_deviceinfo_fastboot_content(args, bootimg=None):
         content += f"""\
         deviceinfo_header_version="{bootimg["header_version"]}"
         """
-    else:
+
+        if bootimg["header_version"] == "2":
+            content += f"""\
+            deviceinfo_append_dtb="false"
+            deviceinfo_flash_offset_dtb="{bootimg["dtb_offset"]}"
+            """
+
+    if "base" in bootimg.keys():
         content += f"""\
         deviceinfo_flash_offset_base="{bootimg["base"]}"
         deviceinfo_flash_offset_kernel="{bootimg["kernel_offset"]}"
@@ -171,7 +179,6 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
         deviceinfo_codename="{codename}"
         deviceinfo_year="{year}"
         deviceinfo_dtb=""
-        deviceinfo_modules_initfs=""
         deviceinfo_arch="{arch}"
 
         # Device related
@@ -187,13 +194,13 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
 
     content_heimdall_bootimg = """\
         deviceinfo_flash_heimdall_partition_kernel=""
-        deviceinfo_flash_heimdall_partition_system=""
+        deviceinfo_flash_heimdall_partition_rootfs=""
         """
 
     content_heimdall_isorec = """\
         deviceinfo_flash_heimdall_partition_kernel=""
         deviceinfo_flash_heimdall_partition_initfs=""
-        deviceinfo_flash_heimdall_partition_system=""
+        deviceinfo_flash_heimdall_partition_rootfs=""
         """
 
     content_0xffff = """\
@@ -205,9 +212,9 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
         """
 
     if flash_method == "fastboot":
-        content += generate_deviceinfo_fastboot_content(args, bootimg)
+        content += generate_deviceinfo_fastboot_content(bootimg)
     elif flash_method == "heimdall-bootimg":
-        content += generate_deviceinfo_fastboot_content(args, bootimg)
+        content += generate_deviceinfo_fastboot_content(bootimg)
         content += content_heimdall_bootimg
     elif flash_method == "heimdall-isorec":
         content += content_heimdall_isorec
@@ -224,6 +231,24 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
             handle.write(line.lstrip() + "\n")
 
 
+def generate_modules_initfs(args):
+    content = """\
+    # Remove this comment after reading, or the file if unnecessary (CHANGEME!)
+    # This file can contain a list of modules to be included in the initramfs,
+    # so that they are available in early boot stages. It should have one
+    # module name per line. If there are multiple kernel variants with different
+    # requirements for modules into the initramfs, one modules-initfs.$variant
+    # file should be created for each of them.
+    """
+
+    # Write to file
+    pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"])
+    path = args.work + "/aportgen/modules-initfs"
+    with open(path, "w", encoding="utf-8") as handle:
+        for line in content.rstrip().split("\n"):
+            handle.write(line.lstrip() + "\n")
+
+
 def generate_apkbuild(args, pkgname, name, arch, flash_method):
     # Dependencies
     depends = ["postmarketos-base",
@@ -232,7 +257,6 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
         depends.append("mkbootimg")
     if flash_method == "0xffff":
         depends.append("uboot-tools")
-    depends.append("mesa-dri-gallium")
 
     # Whole APKBUILD
     depends.sort()
@@ -251,7 +275,10 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
             {depends}
         "
         makedepends="devicepkg-dev"
-        source="deviceinfo"
+        source="
+            deviceinfo
+            modules-initfs
+        "
 
         build() {{
             devicepkg_build $startdir $pkgname
@@ -273,14 +300,14 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
 
 
 def generate(args, pkgname):
-    arch = ask_for_architecture(args)
-    manufacturer = ask_for_manufacturer(args)
-    name = ask_for_name(args, manufacturer)
-    year = ask_for_year(args)
-    chassis = ask_for_chassis(args)
+    arch = ask_for_architecture()
+    manufacturer = ask_for_manufacturer()
+    name = ask_for_name(manufacturer)
+    year = ask_for_year()
+    chassis = ask_for_chassis()
     has_keyboard = ask_for_keyboard(args)
     has_external_storage = ask_for_external_storage(args)
-    flash_method = ask_for_flash_method(args)
+    flash_method = ask_for_flash_method()
     bootimg = None
     if flash_method in ["fastboot", "heimdall-bootimg"]:
         bootimg = ask_for_bootimg(args)
@@ -288,4 +315,5 @@ def generate(args, pkgname):
     generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
                         chassis, has_keyboard, has_external_storage,
                         flash_method, bootimg)
+    generate_modules_initfs(args)
     generate_apkbuild(args, pkgname, name, arch, flash_method)

pmb/aportgen/gcc.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.helpers.git
@@ -28,17 +28,17 @@ def generate(args, pkgname):
     fields = {
         "pkgname": pkgname,
         "pkgdesc": f"Stage2 cross-compiler for {arch}",
-        "arch": args.arch_native,
-        "depends": f"isl binutils-{arch} mpc1",
+        "arch": pmb.config.arch_native,
+        "depends": f"binutils-{arch} mpc1",
         "makedepends_build": "gcc g++ bison flex texinfo gawk zip"
                              " gmp-dev mpfr-dev mpc1-dev zlib-dev",
         "makedepends_host": "linux-headers gmp-dev mpfr-dev mpc1-dev isl-dev"
                             f" zlib-dev musl-dev-{arch} binutils-{arch}",
-        "subpackages": f"g++-{arch}:gpp" if prefix == "gcc" else "",
+        "subpackages": "",
 
         # gcc6: options is already there, so we need to replace it and not only
         # set it below the header like done below.
-        "options": "!strip !tracedeps",
+        "options": "!strip",
 
         "LIBGOMP": "false",
         "LIBGCC": "false",
@@ -46,6 +46,11 @@ def generate(args, pkgname):
         "LIBITM": "false",
     }
 
+    # Latest gcc only, not gcc4 and gcc6
+    if prefix == "gcc":
+        fields["subpackages"] = f"g++-{arch}:gpp" \
+                                f" libstdc++-dev-{arch}:libcxx_dev"
+
     below_header = "CTARGET_ARCH=" + arch + """
         CTARGET="$(arch_to_hostspec ${CTARGET_ARCH})"
         LANG_D=false
@@ -54,7 +59,7 @@ def generate(args, pkgname):
         LANG_GO=false
         LANG_FORTRAN=false
         LANG_ADA=false
-        options="!strip !tracedeps"
+        options="!strip"
 
         # abuild doesn't try to tries to install "build-base-$CTARGET_ARCH"
         # when this variable matches "no*"
@@ -78,6 +83,9 @@ def generate(args, pkgname):
         # use CBUILDROOT as sysroot. In the original APKBUILD this is a local
         # variable, but we make it a global one.
         '*_cross_configure=*': None,
+
+        # Do not build foreign arch libgcc, we use the one from Alpine (#2168)
+        '_libgcc=true*': '_libgcc=false',
     }
 
     pmb.aportgen.core.rewrite(args, pkgname, based_on, fields,

pmb/aportgen/grub_efi.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Nick Reitemeyer, Oliver Smith
+# Copyright 2023 Nick Reitemeyer, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -43,7 +43,7 @@ def generate(args, pkgname):
             pkgdesc="GRUB $_arch EFI files for every architecture"
             url="https://www.gnu.org/software/grub/"
             license="GPL-3.0-or-later"
-            arch="{args.arch_native}"
+            arch="{pmb.config.arch_native}"
             source="grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk::$_mirror/{mirrordir}/main/$_arch/grub-efi-$pkgver-r$pkgrel.apk"
 
             package() {{
@@ -57,7 +57,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/linux.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.helpers.run
 import pmb.aportgen.core
@@ -10,7 +10,7 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
     device = "-".join(pkgname.split("-")[1:])
     carch = pmb.parse.arch.alpine_to_kernel(deviceinfo["arch"])
 
-    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "flex",
+    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex",
                    "openssl-dev", "perl"]
 
     build = """
@@ -22,11 +22,17 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
             downstreamkernel_package "$builddir" "$pkgdir" "$_carch\" \\
                 "$_flavor" "$_outdir\""""
 
+    if deviceinfo.get("header_version") == "2":
+        package += """
+
+            make dtbs_install O="$_outdir" ARCH="$_carch" \\
+                INSTALL_DTBS_PATH="$pkgdir\"/boot/dtbs"""
+
     if deviceinfo["bootimg_qcdt"] == "true":
         build += """\n
             # Master DTB (deviceinfo_bootimg_qcdt)"""
         vendors = ["spreadtrum", "exynos", "other"]
-        soc_vendor = pmb.helpers.cli.ask(args, "SoC vendor", vendors,
+        soc_vendor = pmb.helpers.cli.ask("SoC vendor", vendors,
                                          vendors[-1], complete=vendors)
         if soc_vendor == "spreadtrum":
             makedepends.append("dtbtool-sprd")

pmb/aportgen/musl.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -42,7 +42,7 @@ def generate(args, pkgname):
             pkgname={pkgname}
             pkgver={pkgver}
             pkgrel={pkgrel}
-            arch="{args.arch_native}"
+            arch="{pmb.config.arch_native}"
             subpackages="musl-dev-{arch}:package_dev"
 
             _arch="{arch}"
@@ -93,7 +93,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/build/__init__.py

@@ -1,8 +1,8 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-from pmb.build.init import init, init_compiler
+from pmb.build.init import init, init_abuild_minimal, init_compiler
 from pmb.build.envkernel import package_kernel
-from pmb.build.menuconfig import menuconfig
+from pmb.build.kconfig import menuconfig
 from pmb.build.newapkbuild import newapkbuild
 from pmb.build.other import copy_to_buildpath, is_necessary, \
     index_repo

pmb/build/_package.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import logging
@@ -8,27 +8,26 @@ import pmb.build
 import pmb.build.autodetect
 import pmb.chroot
 import pmb.chroot.apk
-import pmb.chroot.distccd
 import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.parse
 import pmb.parse.arch
 
 
-def skip_already_built(args, pkgname, arch):
+def skip_already_built(pkgname, arch):
     """
     Check if the package was already built in this session, and add it
     to the cache in case it was not built yet.
 
     :returns: True when it can be skipped or False
     """
-    if arch not in args.cache["built"]:
-        args.cache["built"][arch] = []
-    if pkgname in args.cache["built"][arch]:
+    if arch not in pmb.helpers.other.cache["built"]:
+        pmb.helpers.other.cache["built"][arch] = []
+    if pkgname in pmb.helpers.other.cache["built"][arch]:
         logging.verbose(pkgname + ": already checked this session,"
                         " no need to build it or its dependencies")
         return True
-    args.cache["built"][arch].append(pkgname)
+    pmb.helpers.other.cache["built"][arch].append(pkgname)
     return False
 
 
@@ -130,6 +129,9 @@ def build_depends(args, apkbuild, arch, strict):
     if "no_depends" in args and args.no_depends:
         pmb.helpers.repo.update(args, arch)
         for depend in depends:
+            # Ignore conflicting dependencies
+            if depend.startswith("!"):
+                continue
             # Check if binary package is missing
             if not pmb.parse.apkindex.package(args, depend, arch, False):
                 raise RuntimeError("Missing binary package for dependency '" +
@@ -147,6 +149,8 @@ def build_depends(args, apkbuild, arch, strict):
     else:
         # Build the dependencies
         for depend in depends:
+            if depend.startswith("!"):
+                continue
             if package(args, depend, arch, strict=strict):
                 depends_built += [depend]
         logging.verbose(pkgname + ": build dependencies: done, built: " +
@@ -185,7 +189,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     just initialized the build environment for nothing) and then setup the
     whole build environment (abuild, gcc, dependencies, cross-compiler).
 
-    :param cross: None, "native", "distcc", or "crossdirect"
+    :param cross: None, "native", or "crossdirect"
     :param skip_init_buildenv: can be set to False to avoid initializing the
                                build environment. Use this when building
                                something during initialization of the build
@@ -196,7 +200,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
 
     depends_arch = arch
     if cross == "native":
-        depends_arch = args.arch_native
+        depends_arch = pmb.config.arch_native
 
     # Build dependencies
     depends, built = build_depends(args, apkbuild, depends_arch, strict)
@@ -209,7 +213,10 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     if not skip_init_buildenv:
         pmb.build.init(args, suffix)
         pmb.build.other.configure_abuild(args, suffix)
-        pmb.build.other.configure_ccache(args, suffix)
+        if args.ccache:
+            pmb.build.other.configure_ccache(args, suffix)
+            if "rust" in depends or "cargo" in depends:
+                pmb.chroot.apk.install(args, ["sccache"], suffix)
     if not strict and "pmb:strict" not in apkbuild["options"] and len(depends):
         pmb.chroot.apk.install(args, depends, suffix)
     if src:
@@ -218,27 +225,12 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     # Cross-compiler init
     if cross:
         pmb.build.init_compiler(args, depends, cross, arch)
-    if cross == "distcc":
-        pmb.chroot.distccd.start(args, arch)
     if cross == "crossdirect":
         pmb.chroot.mount_native_into_foreign(args, suffix)
 
     return True
 
 
-def get_gcc_version(args, arch):
-    """
-    Get the GCC version for a specific arch from parsing the right APKINDEX.
-    We feed this to ccache, so it knows the right GCC version, when
-    cross-compiling in a foreign arch chroot with distcc. See the "using
-    ccache with other compiler wrappers" section of their man page:
-    <https://linux.die.net/man/1/ccache>
-    :returns: a string like "6.4.0-r5"
-    """
-    return pmb.parse.apkindex.package(args, "gcc-" + arch,
-                                      args.arch_native)["version"]
-
-
 def get_pkgver(original_pkgver, original_source=False, now=None):
     """
     Get the original pkgver when using the original source. Otherwise, get the
@@ -271,7 +263,7 @@ def override_source(args, apkbuild, pkgver, src, suffix="native"):
         return
 
     # Mount source in chroot
-    mount_path = "/mnt/pmbootstrap-source-override/"
+    mount_path = "/mnt/pmbootstrap/source-override/"
     mount_path_outside = args.work + "/chroot_" + suffix + mount_path
     pmb.helpers.mount.bind(args, src, mount_path_outside, umount=True)
 
@@ -376,7 +368,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
     depending on the cross-compiler method and target architecture), copy
     the aport to the chroot and execute abuild.
 
-    :param cross: None, "native", "distcc", or "crossdirect"
+    :param cross: None, "native", or "crossdirect"
     :param src: override source used to build the package with a local folder
     :returns: (output, cmd, env), output is the destination apk path relative
               to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
@@ -405,24 +397,28 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
         hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
         env["CROSS_COMPILE"] = hostspec + "-"
         env["CC"] = hostspec + "-gcc"
-    if cross == "distcc":
-        env["CCACHE_PREFIX"] = "distcc"
-        env["CCACHE_PATH"] = f"/usr/lib/arch-bin-masquerade/{arch}:/usr/bin"
-        env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
-        env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
-        env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
-                             args.port_distccd)
-        env["DISTCC_BACKOFF_PERIOD"] = "0"
-        if not args.distcc_fallback:
-            env["DISTCC_FALLBACK"] = "0"
-        if args.verbose:
-            env["DISTCC_VERBOSE"] = "1"
     if cross == "crossdirect":
         env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
                                 pmb.config.chroot_path])
     if not args.ccache:
         env["CCACHE_DISABLE"] = "1"
 
+    # Use sccache without crossdirect (crossdirect uses it via rustc.sh)
+    if args.ccache and cross != "crossdirect":
+        env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
+
+    # Cache binary objects from go in this path (like ccache)
+    env["GOCACHE"] = "/home/pmos/.cache/go-build"
+
+    # Cache go modules (git repositories). Usually these should be bundled and
+    # it should not be required to download them at build time, in that case
+    # the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But
+    # e.g. when using --src they are not bundled, in that case it makes sense
+    # to point GOMODCACHE at pmbootstrap's work dir so the modules are only
+    # downloaded once.
+    if args.go_mod_cache:
+        env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
+
     # Build the abuild command
     cmd = ["abuild", "-D", "postmarketOS"]
     if strict or "pmb:strict" in apkbuild["options"]:
@@ -456,8 +452,8 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
     # Clear APKINDEX cache (we only parse APKINDEX files once per session and
     # cache the result for faster dependency resolving, but after we built a
     # package we need to parse it again)
-    pmb.parse.apkindex.clear_cache(args, f"{args.work}/packages/{channel}"
-                                         f"/{arch}/APKINDEX.tar.gz")
+    pmb.parse.apkindex.clear_cache(f"{args.work}/packages/{channel}"
+                                   f"/{arch}/APKINDEX.tar.gz")
 
     # Uninstall build dependencies (strict mode)
     if strict or "pmb:strict" in apkbuild["options"]:
@@ -493,8 +489,8 @@ def package(args, pkgname, arch=None, force=False, strict=False,
               output path relative to the packages folder ("armhf/ab-1-r2.apk")
     """
     # Once per session is enough
-    arch = arch or args.arch_native
-    if skip_already_built(args, pkgname, arch):
+    arch = arch or pmb.config.arch_native
+    if skip_already_built(pkgname, arch):
         return
 
     # Only build when APKBUILD exists
@@ -505,7 +501,7 @@ def package(args, pkgname, arch=None, force=False, strict=False,
     # Detect the build environment (skip unnecessary builds)
     if not check_build_for_arch(args, pkgname, arch):
         return
-    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
+    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
     cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
     if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
                          skip_init_buildenv, src):

pmb/build/autodetect.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -40,7 +40,7 @@ def arch(args, pkgname):
     :returns: arch string like "x86_64" or "armhf". Preferred order, depending
               on what is supported by the APKBUILD:
               * native arch
-              * device arch
+              * device arch (this will be preferred instead if build_default_device_arch is true)
               * first arch in the APKBUILD
     """
     aport = pmb.helpers.pmaports.find(args, pkgname)
@@ -48,14 +48,21 @@ def arch(args, pkgname):
     if ret:
         return ret
 
-    apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
+    apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
     arches = apkbuild["arch"]
-    if "noarch" in arches or "all" in arches or args.arch_native in arches:
-        return args.arch_native
 
-    arch_device = args.deviceinfo["arch"]
-    if arch_device in arches:
-        return arch_device
+    if args.build_default_device_arch:
+        preferred_arch = args.deviceinfo["arch"]
+        preferred_arch_2nd = pmb.config.arch_native
+    else:
+        preferred_arch = pmb.config.arch_native
+        preferred_arch_2nd = args.deviceinfo["arch"]
+
+    if "noarch" in arches or "all" in arches or preferred_arch in arches:
+        return preferred_arch
+
+    if preferred_arch_2nd in arches:
+        return preferred_arch_2nd
 
     try:
         return apkbuild["arch"][0]
@@ -63,8 +70,8 @@ def arch(args, pkgname):
         return None
 
 
-def suffix(args, apkbuild, arch):
-    if arch == args.arch_native:
+def suffix(apkbuild, arch):
+    if arch == pmb.config.arch_native:
         return "native"
 
     if "pmb:cross-native" in apkbuild["options"]:
@@ -75,14 +82,14 @@ def suffix(args, apkbuild, arch):
 
 def crosscompile(args, apkbuild, arch, suffix):
     """
-        :returns: None, "native", "crossdirect" or "distcc"
+        :returns: None, "native", "crossdirect"
     """
     if not args.cross:
         return None
-    if not pmb.parse.arch.cpu_emulation_required(args, arch):
+    if not pmb.parse.arch.cpu_emulation_required(arch):
         return None
     if suffix == "native":
         return "native"
-    if args.no_crossdirect or "!pmb:crossdirect" in apkbuild["options"]:
-        return "distcc"
+    if "!pmb:crossdirect" in apkbuild["options"]:
+        return None
     return "crossdirect"

pmb/build/checksum.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 
@@ -10,7 +10,7 @@ import pmb.helpers.pmaports
 
 def update(args, pkgname):
     """ Fetch all sources and update the checksums in the APKBUILD. """
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) generate checksums for " + pkgname)
     pmb.chroot.user(args, ["abuild", "checksum"],
@@ -24,7 +24,7 @@ def update(args, pkgname):
 
 def verify(args, pkgname):
     """ Fetch all sources and verify their checksums. """
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) verify checksums for " + pkgname)
 

pmb/build/envkernel.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Robert Yang
+# Copyright 2023 Robert Yang
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -12,7 +12,7 @@ import pmb.helpers.pmaports
 import pmb.parse
 
 
-def match_kbuild_out(args, word):
+def match_kbuild_out(word):
     """
     Look for paths in the following formats:
       "<prefix>/<kbuild_out>/arch/<arch>/boot"
@@ -47,7 +47,7 @@ def match_kbuild_out(args, word):
     return "" if out_dir is None else out_dir.strip("/")
 
 
-def find_kbuild_output_dir(args, function_body):
+def find_kbuild_output_dir(function_body):
     """
     Guess what the kernel build output directory is. Parses each line of the
     function word by word, looking for paths which contain the kbuild output
@@ -61,7 +61,7 @@ def find_kbuild_output_dir(args, function_body):
     guesses = []
     for line in function_body:
         for item in line.split():
-            kbuild_out = match_kbuild_out(args, item)
+            kbuild_out = match_kbuild_out(item)
             if kbuild_out is not None:
                 guesses.append(kbuild_out)
                 break
@@ -86,7 +86,7 @@ def modify_apkbuild(args, pkgname, aport):
     Modify kernel APKBUILD to package build output from envkernel.sh
     """
     apkbuild_path = aport + "/APKBUILD"
-    apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
+    apkbuild = pmb.parse.apkbuild(apkbuild_path)
     if os.path.exists(args.work + "/aportgen"):
         pmb.helpers.run.user(args, ["rm", "-r", args.work + "/aportgen"])
 
@@ -104,6 +104,23 @@ def modify_apkbuild(args, pkgname, aport):
     pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields)
 
 
+def host_build_bindmount(args, chroot, flag_file, mount=False):
+    """
+    Check if the bind mount already exists and unmount it.
+    Then bindmount the current directory into the chroot as
+    /mnt/linux so it can be used by the envkernel abuild wrapper
+    """
+    flag_path = f"{chroot}/{flag_file}"
+    if os.path.exists(flag_path):
+        logging.info("Cleaning up kernel sources bind-mount")
+        pmb.helpers.run.root(args, ["umount", chroot + "/mnt/linux"], check=False)
+        pmb.helpers.run.root(args, ["rm", flag_path])
+
+    if mount:
+        pmb.helpers.mount.bind(args, ".", f"{chroot}/mnt/linux")
+        pmb.helpers.run.root(args, ["touch", flag_path])
+
+
 def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     """
     Prepare build environment and run abuild.
@@ -117,10 +134,30 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     build_path = "/home/pmos/build"
     kbuild_out_source = "/mnt/linux/.output"
 
+    # If the kernel was cross-compiled on the host rather than with the envkernel
+    # helper, we can still use the envkernel logic to package the artifacts for
+    # development, making it easy to quickly sideload a new kernel or pmbootstrap
+    # to create a boot image
+    # This handles bind mounting the current directory (assumed to be kernel sources)
+    # into the chroot so we can run abuild against it for the currently selected
+    # devices kernel package.
+    flag_file = "envkernel-bind-mounted"
+    host_build = False
+
+    if not pmb.helpers.mount.ismount(chroot + "/mnt/linux"):
+        logging.info("envkernel.sh hasn't run, assuming the kernel was cross compiled"
+                     "on host and using current dir as source")
+        host_build = True
+
+    host_build_bindmount(args, chroot, flag_file, mount=host_build)
+
     if not os.path.exists(chroot + kbuild_out_source):
-        raise RuntimeError("No '.output' dir found in your kernel source dir."
-                           "Compile the " + args.device + " kernel with "
-                           "envkernel.sh first, then try again.")
+        raise RuntimeError("No '.output' dir found in your kernel source dir. "
+                           "Compile the " + args.device + " kernel first and "
+                           "then try again. See https://postmarketos.org/envkernel"
+                           "for details. If building on your host and only using "
+                           "--envkernel for packaging, make sure you have O=.output "
+                           "as an argument to make.")
 
     # Create working directory for abuild
     pmb.build.copy_to_buildpath(args, pkgname)
@@ -142,11 +179,14 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     # Create the apk package
     env = {"CARCH": arch,
            "CHOST": arch,
-           "CBUILD": args.arch_native,
+           "CBUILD": pmb.config.arch_native,
            "SUDO_APK": "abuild-apk --no-progress"}
     cmd = ["abuild", "rootpkg"]
     pmb.chroot.user(args, cmd, working_dir=build_path, env=env)
 
+    # Clean up bindmount if needed
+    host_build_bindmount(args, chroot, flag_file)
+
     # Clean up symlinks
     if build_output != "":
         if os.path.islink(chroot + "/mnt/linux/" + build_output) and \
@@ -171,18 +211,20 @@ def package_kernel(args):
     apkbuild_path = args.work + "/aportgen/APKBUILD"
 
     arch = args.deviceinfo["arch"]
-    apkbuild = pmb.parse.apkbuild(args, apkbuild_path, check_pkgname=False)
+    apkbuild = pmb.parse.apkbuild(apkbuild_path, check_pkgname=False)
     if apkbuild["_outdir"]:
         kbuild_out = apkbuild["_outdir"]
     else:
         function_body = pmb.parse.function_body(aport + "/APKBUILD", "package")
-        kbuild_out = find_kbuild_output_dir(args, function_body)
-    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
+        kbuild_out = find_kbuild_output_dir(function_body)
+    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
 
     # Install package dependencies
     depends, _ = pmb.build._package.build_depends(
-        args, apkbuild, args.arch_native, strict=False)
+        args, apkbuild, pmb.config.arch_native, strict=False)
     pmb.build.init(args, suffix)
+    if pmb.parse.arch.cpu_emulation_required(arch):
+        depends.append("binutils-" + arch)
     pmb.chroot.apk.install(args, depends, suffix)
 
     output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +

pmb/build/init.py

@@ -1,8 +1,9 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import os
-import logging
 import glob
+import logging
+import os
+import pathlib
 
 import pmb.build
 import pmb.config
@@ -11,15 +12,14 @@ import pmb.chroot.apk
 import pmb.helpers.run
 
 
-def init(args, suffix="native"):
-    # Check if already initialized
-    marker = "/var/local/pmbootstrap_chroot_build_init_done"
-    if os.path.exists(args.work + "/chroot_" + suffix + marker):
+def init_abuild_minimal(args, suffix="native"):
+    """ Initialize a minimal chroot with abuild where one can do
+        'abuild checksum'. """
+    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_abuild_init_done"
+    if os.path.exists(marker):
         return
 
-    # Initialize chroot, install packages
-    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
-                           build=False)
+    pmb.chroot.apk.install(args, ["abuild"], suffix, build=False)
 
     # Fix permissions
     pmb.chroot.root(args, ["chown", "root:abuild",
@@ -27,6 +27,24 @@ def init(args, suffix="native"):
     pmb.chroot.root(args, ["chmod", "g+w",
                            "/var/cache/distfiles"], suffix)
 
+    # Add user to group abuild
+    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
+
+    pathlib.Path(marker).touch()
+
+
+def init(args, suffix="native"):
+    """ Initialize a chroot for building packages with abuild. """
+    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_build_init_done"
+    if os.path.exists(marker):
+        return
+
+    init_abuild_minimal(args, suffix)
+
+    # Initialize chroot, install packages
+    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
+                           build=False)
+
     # Generate package signing keys
     chroot = args.work + "/chroot_" + suffix
     if not os.path.exists(args.work + "/config_abuild/abuild.conf"):
@@ -36,7 +54,7 @@ def init(args, suffix="native"):
 
         # Copy package signing key to /etc/apk/keys
         for key in glob.glob(chroot +
-                             "/mnt/pmbootstrap-abuild-config/*.pub"):
+                             "/mnt/pmbootstrap/abuild-config/*.pub"):
             key = key[len(chroot):]
             pmb.chroot.root(args, ["cp", key, "/etc/apk/keys/"], suffix)
 
@@ -63,9 +81,6 @@ def init(args, suffix="native"):
                                "/usr/local/bin/gzip"], suffix)
         pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix)
 
-    # Add user to group abuild
-    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
-
     # abuild.conf: Don't clean the build folder after building, so we can
     # inspect it afterwards for debugging
     pmb.chroot.root(args, ["sed", "-i", "-e", "s/^CLEANUP=.*/CLEANUP=''/",
@@ -77,8 +92,7 @@ def init(args, suffix="native"):
                            "s/^ERROR_CLEANUP=.*/ERROR_CLEANUP=''/",
                            "/etc/abuild.conf"], suffix)
 
-    # Mark the chroot as initialized
-    pmb.chroot.root(args, ["touch", marker], suffix)
+    pathlib.Path(marker).touch()
 
 
 def init_compiler(args, depends, cross, arch):
@@ -94,6 +108,11 @@ def init_compiler(args, depends, cross, arch):
     if cross == "crossdirect":
         cross_pkgs += ["crossdirect"]
         if "rust" in depends or "cargo" in depends:
-            cross_pkgs += ["rust"]
+            if args.ccache:
+                cross_pkgs += ["sccache"]
+            # crossdirect for rust installs all build dependencies in the
+            # native chroot too, as some of them can be required for building
+            # native macros / build scripts
+            cross_pkgs += depends
 
     pmb.chroot.apk.install(args, cross_pkgs)

pmb/build/kconfig.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -14,7 +14,7 @@ import pmb.helpers.run
 import pmb.parse
 
 
-def get_arch(args, apkbuild):
+def get_arch(apkbuild):
     """
     Take the architecture from the APKBUILD or complain if it's ambiguous. This
     function only gets called if --arch is not set.
@@ -78,46 +78,7 @@ def get_outputdir(args, pkgname, apkbuild):
                        " template with: pmbootstrap aportgen " + pkgname)
 
 
-def menuconfig(args, pkgname):
-    # Pkgname: allow omitting "linux-" prefix
-    if pkgname.startswith("linux-"):
-        pkgname_ = pkgname.split("linux-")[1]
-        logging.info("PROTIP: You can simply do 'pmbootstrap kconfig edit " +
-                     pkgname_ + "'")
-    else:
-        pkgname = "linux-" + pkgname
-
-    # Read apkbuild
-    aport = pmb.helpers.pmaports.find(args, pkgname)
-    apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
-    arch = args.arch or get_arch(args, apkbuild)
-    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
-    cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
-    hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
-
-    # Set up build tools and makedepends
-    pmb.build.init(args, suffix)
-    if cross:
-        pmb.build.init_compiler(args, [], cross, arch)
-    depends = apkbuild["makedepends"]
-    kopt = "menuconfig"
-    copy_xauth = False
-    if args.xconfig:
-        depends += ["qt5-qtbase-dev", "font-noto"]
-        kopt = "xconfig"
-        copy_xauth = True
-    elif args.nconfig:
-        kopt = "nconfig"
-        depends += ["ncurses-dev"]
-    else:
-        depends += ["ncurses-dev"]
-    pmb.chroot.apk.install(args, depends)
-
-    # Copy host's .xauthority into native
-    if copy_xauth:
-        pmb.chroot.other.copy_xauthority(args)
-
-    # Patch and extract sources
+def extract_and_patch_sources(args, pkgname, arch):
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) extract kernel source")
     pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
@@ -126,6 +87,53 @@ def menuconfig(args, pkgname):
                     "/home/pmos/build", output="interactive",
                     env={"CARCH": arch})
 
+
+def menuconfig(args, pkgname, use_oldconfig):
+    # Pkgname: allow omitting "linux-" prefix
+    if not pkgname.startswith("linux-"):
+        pkgname = "linux-" + pkgname
+
+    # Read apkbuild
+    aport = pmb.helpers.pmaports.find(args, pkgname)
+    apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
+    arch = args.arch or get_arch(apkbuild)
+    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
+    cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
+    hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
+
+    # Set up build tools and makedepends
+    pmb.build.init(args, suffix)
+    if cross:
+        pmb.build.init_compiler(args, [], cross, arch)
+
+    depends = apkbuild["makedepends"]
+    copy_xauth = False
+
+    if use_oldconfig:
+        kopt = "oldconfig"
+    else:
+        kopt = "menuconfig"
+        if args.xconfig:
+            depends += ["qt5-qtbase-dev", "font-noto"]
+            kopt = "xconfig"
+            copy_xauth = True
+        elif args.nconfig:
+            kopt = "nconfig"
+            depends += ["ncurses-dev"]
+        else:
+            depends += ["ncurses-dev"]
+
+    pmb.chroot.apk.install(args, depends)
+
+    # Copy host's .xauthority into native
+    if copy_xauth:
+        pmb.chroot.other.copy_xauthority(args)
+
+    extract_and_patch_sources(args, pkgname, arch)
+
+    # Check for background color variable
+    color = os.environ.get("MENUCONFIG_COLOR")
+
     # Run make menuconfig
     outputdir = get_outputdir(args, pkgname, apkbuild)
     logging.info("(native) make " + kopt)
@@ -135,6 +143,8 @@ def menuconfig(args, pkgname):
     if cross:
         env["CROSS_COMPILE"] = f"{hostspec}-"
         env["CC"] = f"{hostspec}-gcc"
+    if color:
+        env["MENUCONFIG_COLOR"] = color
     pmb.chroot.user(args, ["make", kopt], "native",
                     outputdir, output="tui", env=env)
 
@@ -151,8 +161,4 @@ def menuconfig(args, pkgname):
     pmb.build.checksum.update(args, pkgname)
 
     # Check config
-    pmb.parse.kconfig.check(args, apkbuild["_flavor"], force_anbox_check=False,
-                            force_nftables_check=False,
-                            force_containers_check=False,
-                            force_zram_check=False,
-                            details=True)
+    pmb.parse.kconfig.check(args, apkbuild["_flavor"], details=True)

pmb/build/newapkbuild.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import os
@@ -25,7 +25,7 @@ def newapkbuild(args, folder, args_passed, force=False):
 
     # Paths for copying
     source_apkbuild = glob_result[0]
-    pkgname = pmb.parse.apkbuild(args, source_apkbuild, False)["pkgname"]
+    pkgname = pmb.parse.apkbuild(source_apkbuild, False)["pkgname"]
     target = args.aports + "/" + folder + "/" + pkgname
 
     # Move /home/pmos/build/$pkgname/* to /home/pmos/build/*

pmb/build/other.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -28,7 +28,16 @@ def copy_to_buildpath(args, package, suffix="native"):
         pmb.chroot.root(args, ["rm", "-rf", "/home/pmos/build"], suffix)
 
     # Copy aport contents with resolved symlinks
-    pmb.helpers.run.root(args, ["cp", "-rL", aport + "/", build])
+    pmb.helpers.run.root(args, ["mkdir", "-p", build])
+    for entry in os.listdir(aport):
+        # Don't copy those dirs, as those have probably been generated by running `abuild`
+        # on the host system directly and not cleaning up after itself.
+        # Those dirs might contain broken symlinks and cp fails resolving them.
+        if entry in ["src", "pkg"]:
+            logging.warn(f"WARNING: Not copying {entry}, looks like a leftover from abuild")
+            continue
+        pmb.helpers.run.root(args, ["cp", "-rL", f"{aport}/{entry}", f"{build}/{entry}"])
+
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos",
                            "/home/pmos/build"], suffix)
 
@@ -117,7 +126,7 @@ def index_repo(args, arch=None):
                 pmb.chroot.user(args, command, working_dir=path_repo_chroot)
         else:
             logging.debug("NOTE: Can't build index for: " + path)
-        pmb.parse.apkindex.clear_cache(args, path + "/APKINDEX.tar.gz")
+        pmb.parse.apkindex.clear_cache(f"{path}/APKINDEX.tar.gz")
 
 
 def configure_abuild(args, suffix, verify=False):
@@ -143,7 +152,7 @@ def configure_abuild(args, suffix, verify=False):
                                 suffix)
                 configure_abuild(args, suffix, True)
             return
-    raise RuntimeError("Could not find " + prefix + " line in " + path)
+    pmb.chroot.root(args, ["sed", "-i", f"$ a\\{prefix}{args.jobs}", "/etc/abuild.conf"], suffix)
 
 
 def configure_ccache(args, suffix="native", verify=False):

pmb/chroot/__init__.py

@@ -1,7 +1,7 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.chroot.init import init, init_keys
-from pmb.chroot.mount import mount, mount_native_into_foreign
+from pmb.chroot.mount import mount, mount_native_into_foreign, remove_mnt_pmbootstrap
 from pmb.chroot.root import root
 from pmb.chroot.user import user
 from pmb.chroot.user import exists as user_exists

pmb/chroot/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -25,7 +25,7 @@ def update_repository_list(args, suffix="native", check=False):
                   True.
     """
     # Skip if we already did this
-    if suffix in args.cache["apk_repository_list_updated"]:
+    if suffix in pmb.helpers.other.cache["apk_repository_list_updated"]:
         return
 
     # Read old entries or create folder structure
@@ -43,7 +43,7 @@ def update_repository_list(args, suffix="native", check=False):
     # Up to date: Save cache, return
     lines_new = pmb.helpers.repo.urls(args)
     if lines_old == lines_new:
-        args.cache["apk_repository_list_updated"].append(suffix)
+        pmb.helpers.other.cache["apk_repository_list_updated"].append(suffix)
         return
 
     # Check phase: raise error when still outdated
@@ -67,7 +67,7 @@ def check_min_version(args, suffix="native"):
     """
 
     # Skip if we already did this
-    if suffix in args.cache["apk_min_version_checked"]:
+    if suffix in pmb.helpers.other.cache["apk_min_version_checked"]:
         return
 
     # Skip if apk is not installed yet
@@ -84,153 +84,113 @@ def check_min_version(args, suffix="native"):
         " 'pmbootstrap zap -hc'")
 
     # Mark this suffix as checked
-    args.cache["apk_min_version_checked"].append(suffix)
+    pmb.helpers.other.cache["apk_min_version_checked"].append(suffix)
 
 
-def install_is_necessary(args, build, arch, package, packages_installed):
+def install_build(args, package, arch):
     """
-    This function optionally builds an out of date package, and checks if the
-    version installed inside a chroot is up to date.
-    :param build: Set to true to build the package, if the binary packages are
-                  out of date, and it is in the aports folder.
-    :param packages_installed: Return value from installed().
-    :returns: True if the package needs to be installed/updated,
-              False otherwise.
+    Build an outdated package unless pmbootstrap was invoked with
+    "pmbootstrap install" and the option to build packages during pmb install
+    is disabled.
+
+    :param package: name of the package to build
+    :param arch: architecture of the package to build
     """
-    # User may have disabled buiding packages during "pmbootstrap install"
-    build_disabled = False
+    # User may have disabled building packages during "pmbootstrap install"
     if args.action == "install" and not args.build_pkgs_on_install:
-        build_disabled = True
-
-    # Build package
-    if build and not build_disabled:
-        pmb.build.package(args, package, arch)
-
-    # No further checks when not installed
-    if package not in packages_installed:
-        return True
-
-    # Make sure that we really have a binary package
-    data_repo = pmb.parse.apkindex.package(args, package, arch, False)
-    if not data_repo:
-        if build_disabled:
+        if not pmb.parse.apkindex.package(args, package, arch, False):
             raise RuntimeError(f"{package}: no binary package found for"
                                f" {arch}, and compiling packages during"
                                " 'pmbootstrap install' has been disabled."
                                " Consider changing this option in"
                                " 'pmbootstrap init'.")
-        logging.warning("WARNING: Internal error in pmbootstrap,"
-                        f" package '{package}' for {arch}"
-                        " has not been built yet, but it should have"
-                        " been. Rebuilding it with force. Please "
-                        " report this, if there is no ticket about this"
-                        " yet!")
-        pmb.build.package(args, package, arch, True)
-        return install_is_necessary(args, build, arch, package,
-                                    packages_installed)
+        # Use the existing binary package
+        return
 
-    # Compare the installed version vs. the version in the repos
-    data_installed = packages_installed[package]
-    compare = pmb.parse.version.compare(data_installed["version"],
-                                        data_repo["version"])
-    # a) Installed newer (should not happen normally)
-    if compare == 1:
-        logging.info(f"WARNING: {arch} package '{package}'"
-                     f" installed version {data_installed['version']}"
-                     " is newer, than the version in the repositories:"
-                     f" {data_repo['version']}"
-                     " See also: <https://postmarketos.org/warning-repo>")
-        return False
-
-    # b) Repo newer
-    elif compare == -1:
-        return True
-
-    # c) Same version, look at last modified
-    elif compare == 0:
-        time_installed = float(data_installed["timestamp"])
-        time_repo = float(data_repo["timestamp"])
-        return time_repo > time_installed
+    # Build the package if it's in pmaports and there is no binary package
+    # with the same pkgver and pkgrel. This check is done in
+    # pmb.build.is_necessary, which gets called in pmb.build.package.
+    return pmb.build.package(args, package, arch)
 
 
-def replace_aports_packages_with_path(args, packages, suffix, arch):
+def packages_split_to_add_del(packages):
     """
-    apk will only re-install packages with the same pkgname,
-    pkgver and pkgrel, when you give it the absolute path to the package.
-    This function replaces all packages that were built locally,
-    with the absolute path to the package.
+    Sort packages into "to_add" and "to_del" lists depending on their pkgname
+    starting with an exclamation mark.
+
+    :param packages: list of pkgnames
+    :returns: (to_add, to_del) - tuple of lists of pkgnames, e.g.
+              (["hello-world", ...], ["some-conflict-pkg", ...])
     """
-    ret = []
+    to_add = []
+    to_del = []
+
     for package in packages:
-        aport = pmb.helpers.pmaports.find(args, package, False)
-        if aport:
-            data_repo = pmb.parse.apkindex.package(args, package, arch, False)
-            if not data_repo:
-                raise RuntimeError(f"{package}: could not find binary"
-                                   " package, although it should exist for"
-                                   " sure at this point in the code."
-                                   " Probably an APKBUILD subpackage parsing"
-                                   " bug. Related: https://gitlab.com/"
-                                   "postmarketOS/build.postmarketos.org/"
-                                   "issues/61")
-            apk_path = (f"/mnt/pmbootstrap-packages/{arch}/"
-                        f"{package}-{data_repo['version']}.apk")
-            if os.path.exists(f"{args.work}/chroot_{suffix}{apk_path}"):
-                package = apk_path
-        ret.append(package)
+        if package.startswith("!"):
+            to_del.append(package.lstrip("!"))
+        else:
+            to_add.append(package)
+
+    return (to_add, to_del)
+
+
+def packages_get_locally_built_apks(args, packages, arch):
+    """
+    Iterate over packages and if existing, get paths to locally built packages.
+    This is used to force apk to upgrade packages to newer local versions, even
+    if the pkgver and pkgrel did not change.
+
+    :param packages: list of pkgnames
+    :param arch: architecture that the locally built packages should have
+    :returns: list of apk file paths that are valid inside the chroots, e.g.
+              ["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...]
+    """
+    channel = pmb.config.pmaports.read_config(args)["channel"]
+    ret = []
+
+    for package in packages:
+        data_repo = pmb.parse.apkindex.package(args, package, arch, False)
+        if not data_repo:
+            continue
+
+        apk_file = f"{package}-{data_repo['version']}.apk"
+        if not os.path.exists(f"{args.work}/packages/{channel}/{arch}/{apk_file}"):
+            continue
+
+        ret.append(f"/mnt/pmbootstrap/packages/{arch}/{apk_file}")
+
     return ret
 
 
-def install(args, packages, suffix="native", build=True):
+def install_run_apk(args, to_add, to_add_local, to_del, suffix):
     """
-    :param build: automatically build the package, when it does not exist yet
-                  or needs to be updated, and it is inside the pm-aports
-                  folder. Checking this is expensive - if you know that all
-                  packages are provides by upstream repos, set this to False!
+    Run apk to add packages, and ensure only the desired packages get
+    explicitly marked as installed.
+
+    :param to_add: list of pkgnames to install, without their dependencies
+    :param to_add_local: return of packages_get_locally_built_apks()
+    :param to_del: list of pkgnames to be deleted, this should be set to
+                   conflicting dependencies in any of the packages to be
+                   installed or their dependencies (e.g. ["osk-sdl"])
+    :param suffix: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64"
     """
-    # Initialize chroot
-    check_min_version(args, suffix)
-    pmb.chroot.init(args, suffix)
-
-    # Add depends to packages
-    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
-    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
-
-    # Filter outdated packages (build them if required)
-    packages_installed = installed(args, suffix)
-    packages_todo = []
-    for package in packages_with_depends:
-        if install_is_necessary(
-                args, build, arch, package, packages_installed):
-            packages_todo.append(package)
-    if not len(packages_todo):
-        return
-
     # Sanitize packages: don't allow '--allow-untrusted' and other options
     # to be passed to apk!
-    for package in packages_todo:
+    for package in to_add + to_add_local + to_del:
         if package.startswith("-"):
             raise ValueError(f"Invalid package name: {package}")
 
-    # Readable install message without dependencies
-    message = f"({suffix}) install"
-    for pkgname in packages:
-        if pkgname not in packages_installed:
-            message += f" {pkgname}"
-    logging.info(message)
-
-    # Local packages: Using the path instead of pkgname makes apk update
-    # packages of the same version if the build date is different
-    packages_todo = replace_aports_packages_with_path(args, packages_todo,
-                                                      suffix, arch)
+    commands = [["add"] + to_add]
 
     # Use a virtual package to mark only the explicitly requested packages as
-    # explicitly installed, not their dependencies or specific paths (#1212)
-    commands = [["add"] + packages]
-    if packages != packages_todo:
-        commands = [["add", "-u", "--virtual", ".pmbootstrap"] + packages_todo,
-                    ["add"] + packages,
-                    ["del", ".pmbootstrap"]]
+    # explicitly installed, not the ones in to_add_local
+    if to_add_local:
+        commands += [["add", "-u", "--virtual", ".pmbootstrap"] + to_add_local,
+                     ["del", ".pmbootstrap"]]
+
+    if to_del:
+        commands += [["del"] + to_del]
+
     for (i, command) in enumerate(commands):
         if args.offline:
             command = ["--no-network"] + command
@@ -245,6 +205,44 @@ def install(args, packages, suffix="native", build=True):
                             suffix=suffix)
 
 
+def install(args, packages, suffix="native", build=True):
+    """
+    Install packages from pmbootstrap's local package index or the pmOS/Alpine
+    binary package mirrors. Iterate over all dependencies recursively, and
+    build missing packages as necessary.
+
+    :param packages: list of pkgnames to be installed
+    :param suffix: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64"
+    :param build: automatically build the package, when it does not exist yet
+                  or needs to be updated, and it is inside pmaports. For the
+                  special case that all packages are expected to be in Alpine's
+                  repositories, set this to False for performance optimization.
+    """
+    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
+
+    if not packages:
+        logging.verbose("pmb.chroot.apk.install called with empty packages list,"
+                        " ignoring")
+        return
+
+    # Initialize chroot
+    check_min_version(args, suffix)
+    pmb.chroot.init(args, suffix)
+
+    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
+    to_add, to_del = packages_split_to_add_del(packages_with_depends)
+
+    if build:
+        for package in to_add:
+            install_build(args, package, arch)
+
+    to_add_local = packages_get_locally_built_apks(args, to_add, arch)
+    to_add_no_deps, _ = packages_split_to_add_del(packages)
+
+    logging.info(f"({suffix}) install {' '.join(to_add_no_deps)}")
+    install_run_apk(args, to_add_no_deps, to_add_local, to_del, suffix)
+
+
 def installed(args, suffix="native"):
     """
     Read the list of installed packages (which has almost the same format, as
@@ -261,4 +259,4 @@ def installed(args, suffix="native"):
               }
     """
     path = f"{args.work}/chroot_{suffix}/lib/apk/db/installed"
-    return pmb.parse.apkindex.parse(args, path, False)
+    return pmb.parse.apkindex.parse(path, False)

pmb/chroot/apk_static.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -141,7 +141,7 @@ def download(args, file):
     """
     channel_cfg = pmb.config.pmaports.read_config_channel(args)
     mirrordir = channel_cfg["mirrordir_alpine"]
-    base_url = f"{args.mirror_alpine}{mirrordir}/main/{args.arch_native}"
+    base_url = f"{args.mirror_alpine}{mirrordir}/main/{pmb.config.arch_native}"
     return pmb.helpers.http.download(args, f"{base_url}/{file}", file)
 
 

pmb/chroot/binfmt.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -33,7 +33,7 @@ def register(args, arch):
     if is_registered(arch_qemu):
         return
 
-    info = pmb.parse.binfmt_info(args, arch_qemu)
+    info = pmb.parse.binfmt_info(arch_qemu)
 
     # Build registration string
     # https://en.wikipedia.org/wiki/Binfmt_misc

pmb/chroot/distccd.py

@@ -1,250 +0,0 @@
-# Copyright 2021 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-import errno
-import json
-import logging
-import os
-import pmb.chroot
-import pmb.config
-import pmb.chroot.apk
-
-""" Packages for foreign architectures (e.g. armhf) get built in chroots
-    running with QEMU. While this works, it is painfully slow. So we speed it
-    up by using distcc to let cross compilers running in the native chroots do
-    the heavy lifting.
-
-    This file sets up an SSH server in the native chroot, which will then be
-    used by the foreign arch chroot to communicate with the distcc daemon. We
-    make sure that only the foreign arch chroot can connect to the sshd by only
-    listening on localhost, as well as generating dedicated ssh keys.
-
-    Using the SSH server instead of running distccd directly is a security
-    measure. Distccd does not authenticate its clients and would therefore
-    allow any process of the host system (not related to pmbootstrap) to
-    execute compilers in the native chroot. By modifying the compiler's options
-    or sending malicious data to the compiler, it is likely that the process
-    can gain remote code execution [1]. That way, a compromised, but sandboxed
-    process could gain privilege escalation.
-
-    [1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
-"""
-
-
-def init_server(args):
-    """
-    Install dependencies and generate keys for the server.
-    """
-    # Install dependencies
-    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
-                                  "openssh-server"])
-
-    # Config folder (nothing to do if existing)
-    dir = "/home/pmos/.distcc-sshd"
-    dir_outside = args.work + "/chroot_native" + dir
-    if os.path.exists(dir_outside):
-        return
-
-    # Generate keys
-    logging.info("(native) generate distcc-sshd server keys")
-    pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
-    pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
-
-
-def init_client(args, suffix):
-    """
-    Install dependencies and generate keys for the client.
-    """
-    # Install dependencies
-    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
-                                  "openssh-client"], suffix)
-
-    # Public key path (nothing to do if existing)
-    pub = "/home/pmos/id_ed25519.pub"
-    pub_outside = args.work + "/chroot_" + suffix + pub
-    if os.path.exists(pub_outside):
-        return
-
-    # Generate keys
-    logging.info("(" + suffix + ") generate distcc-sshd client keys")
-    pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
-                           "-f", "/home/pmos/.ssh/id_ed25519"], suffix)
-    pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
-                    suffix)
-
-
-def configure_authorized_keys(args, suffix):
-    """
-    Exclusively allow one foreign arch chroot to access the sshd.
-    """
-    auth = "/home/pmos/.distcc-sshd/authorized_keys"
-    auth_outside = args.work + "/chroot_native/" + auth
-    pub = "/home/pmos/id_ed25519.pub"
-    pub_outside = args.work + "/chroot_" + suffix + pub
-    pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
-
-
-def configure_cmdlist(args, arch):
-    """
-    Create a whitelist of all the cross compiler wrappers.
-
-    Distcc 3.3 and above requires such a whitelist, or else it will only run
-    with the --make-me-a-botnet parameter (even in ssh mode).
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
-        for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
-            cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
-            handle.write(cmd_full + "\n")
-    pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
-    pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
-
-
-def configure_distccd_wrapper(args):
-    """
-    Wrap distccd in a shell script, so we can pass the compiler whitelist and
-    set the verbose flag (when pmbootstrap is running with --verbose).
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
-        handle.write("#!/bin/sh\n"
-                     "export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
-                     "distccd --log-file /home/pmos/distccd.log --nice 19")
-        if args.verbose:
-            handle.write(" --verbose")
-        handle.write(" \"$@\"\n")
-    pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
-    pmb.chroot.user(args, ["cat", dir + "/distccd"])
-    pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
-
-
-def configure_sshd(args):
-    """
-    Configure the SSH daemon in the native chroot.
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    config = """AllowAgentForwarding no
-                AllowTcpForwarding no
-                AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
-                HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
-                ListenAddress 127.0.0.1
-                PasswordAuthentication no
-                PidFile /home/pmos/.distcc-sshd/sshd.pid
-                Port """ + args.port_distccd + """
-                X11Forwarding no"""
-
-    with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
-        for line in config.split("\n"):
-            handle.write(line.lstrip() + "\n")
-    pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
-    pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
-
-
-def get_running_pid(args):
-    """
-    :returns: the running distcc-sshd's pid as integer or None
-    """
-    # PID file must exist
-    pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
-    pidfile_outside = args.work + "/chroot_native" + pidfile
-    if not os.path.exists(pidfile_outside):
-        return None
-
-    # Verify, if it still exists by sending a kill signal
-    with open(pidfile_outside, "r") as handle:
-        pid = int(handle.read()[:-1])
-    try:
-        os.kill(pid, 0)
-    except OSError as err:
-        if err.errno == errno.ESRCH:  # no such process
-            pmb.helpers.run.root(args, ["rm", pidfile_outside])
-            return None
-    return pid
-
-
-def get_running_parameters(args):
-    """
-    Get the parameters of the currently running distcc-sshd instance.
-
-    :returns: a dictionary in the form of
-              {"arch": "armhf", "port": 1234, "verbose": False}
-              If the information can not be read, "arch" is set to "unknown"
-    """
-    # Return defaults
-    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-    if not os.path.exists(path):
-        return {"arch": "unknown", "port": 0, "verbose": False}
-
-    # Parse the file as JSON
-    with open(path, "r") as handle:
-        return json.loads(handle.read())
-
-
-def set_running_parameters(args, arch):
-    """
-    Set the parameters of the currently running distcc-sshd instance.
-    """
-    parameters = {"arch": arch,
-                  "port": args.port_distccd,
-                  "verbose": args.verbose}
-
-    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-    with open(path, "w") as handle:
-        json.dump(parameters, handle)
-
-
-def is_running_with_same_parameters(args, arch):
-    """
-    Check whether we can use the already running distcc-sshd instance with our
-    current set of parameters. In case we can use it directly, we save some
-    time, otherwise we need to stop it, configure it again, and start it once
-    more.
-    """
-    if not get_running_pid(args):
-        return False
-
-    parameters = get_running_parameters(args)
-    return (parameters["arch"] == arch and
-            parameters["port"] == args.port_distccd and
-            parameters["verbose"] == args.verbose)
-
-
-def stop(args):
-    """
-    Kill the sshd process (by using its pid).
-    """
-    pid = get_running_pid(args)
-    if not pid:
-        return
-
-    parameters = get_running_parameters(args)
-    logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
-    pmb.chroot.user(args, ["kill", str(pid)])
-
-
-def start(args, arch):
-    """
-    Set up a new distcc-sshd instance or use an already running one.
-    """
-    if is_running_with_same_parameters(args, arch):
-        return
-    stop(args)
-
-    # Initialize server and client
-    suffix = "buildroot_" + arch
-    init_server(args)
-    init_client(args, suffix)
-
-    logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
-                 args.port_distccd)
-
-    # Configure server parameters (arch, port, verbose)
-    configure_authorized_keys(args, suffix)
-    configure_distccd_wrapper(args)
-    configure_cmdlist(args, arch)
-    configure_sshd(args)
-
-    # Run
-    dir = "/home/pmos/.distcc-sshd"
-    pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
-                           "-E", dir + "/log.txt"])
-    set_running_parameters(args, arch)

pmb/chroot/init.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -43,7 +43,7 @@ def mark_in_chroot(args, suffix="native"):
 
 def setup_qemu_emulation(args, suffix):
     arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
-    if not pmb.parse.arch.cpu_emulation_required(args, arch):
+    if not pmb.parse.arch.cpu_emulation_required(arch):
         return
 
     chroot = f"{args.work}/chroot_{suffix}"

pmb/chroot/initfs.py

@@ -1,10 +1,11 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
 import pmb.chroot.initfs_hooks
 import pmb.chroot.other
 import pmb.chroot.apk
+import pmb.config.pmaports
 import pmb.helpers.cli
 
 
@@ -35,9 +36,16 @@ def extract(args, flavor, suffix, extra=False):
     """
     # Extraction folder
     inside = "/tmp/initfs-extracted"
+
+    pmaports_cfg = pmb.config.pmaports.read_config(args)
+    if pmaports_cfg.get("supported_mkinitfs_without_flavors", False):
+        initfs_file = "/boot/initramfs"
+    else:
+        initfs_file = f"/boot/initramfs-${flavor}"
     if extra:
         inside = "/tmp/initfs-extra-extracted"
-        flavor += "-extra"
+        initfs_file += "-extra"
+
     outside = f"{args.work}/chroot_{suffix}{inside}"
     if os.path.exists(outside):
         if not pmb.helpers.cli.confirm(args, f"Extraction folder {outside}"
@@ -55,7 +63,7 @@ def extract(args, flavor, suffix, extra=False):
 
     # Extract
     commands = [["mkdir", "-p", inside],
-                ["cp", f"/boot/initramfs-{flavor}", f"{inside}/_initfs.gz"],
+                ["cp", initfs_file, f"{inside}/_initfs.gz"],
                 ["gzip", "-d", f"{inside}/_initfs.gz"],
                 ["cat", "/tmp/_extract.sh"],  # for the log
                 ["sh", "/tmp/_extract.sh"],

pmb/chroot/initfs_hooks.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/chroot/mount.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -72,6 +72,9 @@ def mount_dev_tmpfs(args, suffix="native"):
                                 "tmpfs", dev + "/shm"])
     create_device_nodes(args, suffix)
 
+    # Setup /dev/fd as a symlink
+    pmb.helpers.run.root(args, ["ln", "-sf", "/proc/self/fd", f"{dev}/"])
+
 
 def mount(args, suffix="native"):
     # Mount tmpfs as the chroot's /dev
@@ -103,3 +106,18 @@ def mount_native_into_foreign(args, suffix):
     if not os.path.lexists(musl_link):
         pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,
                                     musl_link])
+
+def remove_mnt_pmbootstrap(args, suffix):
+    """ Safely remove /mnt/pmbootstrap directories from the chroot, without
+        running rm -r as root and potentially removing data inside the
+        mountpoint in case it was still mounted (bug in pmbootstrap, or user
+        ran pmbootstrap 2x in parallel). This is similar to running 'rm -r -d',
+        but we don't assume that the host's rm has the -d flag (busybox does
+        not). """
+    mnt_dir = f"{args.work}/chroot_{suffix}/mnt/pmbootstrap"
+
+    if not os.path.exists(mnt_dir):
+        return
+
+    for path in glob.glob(f"{mnt_dir}/*") + [mnt_dir]:
+        pmb.helpers.run.root(args, ["rmdir", path])

pmb/chroot/other.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/chroot/root.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import shutil
@@ -55,10 +55,11 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
 
     # Merge env with defaults into env_all
     env_all = {"CHARSET": "UTF-8",
-               "LANG": "UTF-8",
                "HISTFILE": "~/.ash_history",
                "HOME": "/root",
+               "LANG": "UTF-8",
                "PATH": pmb.config.chroot_path,
+               "PYTHONUNBUFFERED": "1",
                "SHELL": "/bin/ash",
                "TERM": "xterm"}
     for key, value in env.items():
@@ -70,9 +71,11 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
     # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
     executables = executables_absolute_path()
     cmd_chroot = [executables["chroot"], chroot, "/bin/sh", "-c",
-                  pmb.helpers.run.flat_cmd(cmd, working_dir)]
-    cmd_sudo = ["sudo", "env", "-i", executables["sh"], "-c",
-                pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)]
+                  pmb.helpers.run_core.flat_cmd(cmd, working_dir)]
+    cmd_sudo = pmb.config.sudo([
+        "env", "-i", executables["sh"], "-c",
+        pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)]
+    )
     return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output,
                                      output_return, check, True,
                                      disable_timeout)

pmb/chroot/shutdown.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -7,7 +7,6 @@ import socket
 from contextlib import closing
 
 import pmb.chroot
-import pmb.chroot.distccd
 import pmb.helpers.mount
 import pmb.install.losetup
 import pmb.parse.arch
@@ -23,6 +22,17 @@ def kill_adb(args):
             pmb.chroot.root(args, ["adb", "-P", str(port), "kill-server"])
 
 
+def kill_sccache(args):
+    """
+    Kill sccache daemon if it's running. Unlike ccache it automatically spawns
+    a daemon when you call it and exits after some time of inactivity.
+    """
+    port = 4226
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+        if sock.connect_ex(("127.0.0.1", port)) == 0:
+            pmb.chroot.root(args, ["sccache", "--stop-server"])
+
+
 def shutdown_cryptsetup_device(args, name):
     """
     :param name: cryptsetup device name, usually "pm_crypt" in pmbootstrap
@@ -49,10 +59,9 @@ def shutdown_cryptsetup_device(args, name):
 
 
 def shutdown(args, only_install_related=False):
-    pmb.chroot.distccd.stop(args)
-
-    # Stop adb server
+    # Stop daemons
     kill_adb(args)
+    kill_sccache(args)
 
     # Umount installation-related paths (order is important!)
     pmb.helpers.mount.umount_all(args, args.work +
@@ -90,6 +99,6 @@ def shutdown(args, only_install_related=False):
 
         # Clean up the rest
         for arch in pmb.config.build_device_architectures:
-            if pmb.parse.arch.cpu_emulation_required(args, arch):
+            if pmb.parse.arch.cpu_emulation_required(arch):
                 pmb.chroot.binfmt.unregister(args, arch)
         logging.debug("Shutdown complete")

pmb/chroot/user.py

@@ -1,7 +1,8 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.chroot.root
 import pmb.helpers.run
+import pmb.helpers.run_core
 
 
 def user(args, cmd, suffix="native", working_dir="/", output="log",
@@ -21,7 +22,7 @@ def user(args, cmd, suffix="native", working_dir="/", output="log",
     if "HOME" not in env:
         env["HOME"] = "/home/pmos"
 
-    flat_cmd = pmb.helpers.run.flat_cmd(cmd, env=env)
+    flat_cmd = pmb.helpers.run_core.flat_cmd(cmd, env=env)
     cmd = ["busybox", "su", "pmos", "-c", flat_cmd]
     return pmb.chroot.root(args, cmd, suffix, working_dir, output,
                            output_return, check, {}, auto_init)

pmb/chroot/zap.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -15,9 +15,9 @@ import pmb.parse.apkindex
 
 def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False,
-        rust=False):
+        rust=False, netboot=False):
     """
-    Shutdown everything inside the chroots (e.g. distccd, adb), umount
+    Shutdown everything inside the chroots (e.g. adb), umount
     everything and then safely remove folders from the work-directory.
 
     :param dry: Only show what would be deleted, do not delete for real
@@ -29,6 +29,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         downloaded from mirrors (e.g. from Alpine)
     :param distfiles: Clear the downloaded files cache
     :param rust: Remove rust related caches
+    :param netboot: Remove images for netboot
 
     NOTE: This function gets called in pmb/config/init.py, with only args.work
     and args.device set!
@@ -65,6 +66,8 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         patterns += ["cache_distfiles"]
     if rust:
         patterns += ["cache_rust"]
+    if netboot:
+        patterns += ["images_netboot"]
 
     # Delete everything matching the patterns
     for pattern in patterns:
@@ -81,7 +84,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
     pmb.config.workdir.clean(args)
 
     # Chroots were zapped, so no repo lists exist anymore
-    args.cache["apk_repository_list_updated"].clear()
+    pmb.helpers.other.cache["apk_repository_list_updated"].clear()
 
     # Print amount of cleaned up space
     if dry:
@@ -106,7 +109,7 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
     pattern = f"{args.work}/packages/{channel}/*/APKINDEX.tar.gz"
     for apkindex_path in glob.glob(pattern):
         # Delete packages without same version in aports
-        blocks = pmb.parse.apkindex.parse_blocks(args, apkindex_path)
+        blocks = pmb.parse.apkindex.parse_blocks(apkindex_path)
         for block in blocks:
             pkgname = block["pkgname"]
             origin = block["origin"]
@@ -132,7 +135,7 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
                 continue
 
             # Clear out any binary apks that do not match what is in aports
-            apkbuild = pmb.parse.apkbuild(args, f"{aport_path}/APKBUILD")
+            apkbuild = pmb.parse.apkbuild(f"{aport_path}/APKBUILD")
             version_aport = f"{apkbuild['pkgver']}-r{apkbuild['pkgrel']}"
             if version != version_aport:
                 logging.info(f"% rm {apk_path_short}"
@@ -158,7 +161,9 @@ def zap_pkgs_online_mismatch(args, confirm=True, dry=False):
     # Iterate over existing apk caches
     for path in paths:
         arch = os.path.basename(path).split("_", 2)[2]
-        suffix = "native" if arch == args.arch_native else f"buildroot_{arch}"
+        suffix = f"buildroot_{arch}"
+        if arch == pmb.config.arch_native:
+            suffix = "native"
 
         # Clean the cache with apk
         logging.info(f"({suffix}) apk -v cache clean")

pmb/ci/__init__.py

@@ -0,0 +1,169 @@
+# Copyright 2023 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+import collections
+import glob
+import logging
+import os
+import shlex
+import pmb.chroot
+import pmb.helpers.cli
+
+
+def get_ci_scripts(topdir):
+    """ Find 'pmbootstrap ci'-compatible scripts inside a git repository, and
+        parse their metadata (description, options). The reference is at:
+        https://postmarketos.org/pmb-ci
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir()
+        :returns: a dict of CI scripts found in the git repository, e.g.
+                  {"ruff": {"description": "lint all python scripts",
+                              "options": []},
+                   ...} """
+    ret = {}
+    for script in glob.glob(f"{topdir}/.ci/*.sh"):
+        is_pmb_ci_script = False
+        description = ""
+        options = []
+
+        with open(script) as handle:
+            for line in handle:
+                if line.startswith("# https://postmarketos.org/pmb-ci"):
+                    is_pmb_ci_script = True
+                elif line.startswith("# Description: "):
+                    description = line.split(": ", 1)[1].rstrip()
+                elif line.startswith("# Options: "):
+                    options = line.split(": ", 1)[1].rstrip().split(" ")
+                elif not line.startswith("#"):
+                    # Stop parsing after the block of comments on top
+                    break
+
+        if not is_pmb_ci_script:
+            continue
+
+        if not description:
+            logging.error(f"ERROR: {script}: missing '# Description: …' line")
+            exit(1)
+
+        for option in options:
+            if option not in pmb.config.ci_valid_options:
+                raise RuntimeError(f"{script}: unsupported option '{option}'."
+                                   " Typo in script or pmbootstrap too old?")
+
+        short_name = os.path.basename(script).split(".", -1)[0]
+        ret[short_name] = {"description": description,
+                           "options": options}
+    return ret
+
+
+def sort_scripts_by_speed(scripts):
+    """ Order the scripts, so fast scripts run before slow scripts. Whether a
+        script is fast or not is determined by the '# Options: slow' comment in
+        the file.
+        :param scripts: return of get_ci_scripts()
+        :returns: same format as get_ci_scripts(), but as ordered dict with
+                  fast scripts before slow scripts """
+    ret = collections.OrderedDict()
+
+    # Fast scripts first
+    for script_name, script in scripts.items():
+        if "slow" in script["options"]:
+            continue
+        ret[script_name] = script
+
+    # Then slow scripts
+    for script_name, script in scripts.items():
+        if "slow" not in script["options"]:
+            continue
+        ret[script_name] = script
+    return ret
+
+
+def ask_which_scripts_to_run(scripts_available):
+    """ Display an interactive prompt about which of the scripts the user
+        wishes to run, or all of them.
+        :param scripts_available: same format as get_ci_scripts()
+        :returns: either full scripts_available (all selected), or a subset """
+    count = len(scripts_available.items())
+    choices = ["all"]
+
+    logging.info(f"Available CI scripts ({count}):")
+    for script_name, script in scripts_available.items():
+        extra = ""
+        if "slow" in script["options"]:
+            extra += " (slow)"
+        logging.info(f"* {script_name}: {script['description']}{extra}")
+        choices += [script_name]
+
+    selection = pmb.helpers.cli.ask("Which script?", None, "all",
+                                    complete=choices)
+    if selection == "all":
+        return scripts_available
+
+    ret = {}
+    ret[selection] = scripts_available[selection]
+    return ret
+
+
+def copy_git_repo_to_chroot(args, topdir):
+    """ Create a tarball of the git repo (including unstaged changes and new
+        files) and extract it in chroot_native.
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir() """
+    pmb.chroot.init(args)
+    tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz"
+    files = pmb.helpers.git.get_files(args, topdir)
+
+    with open(f"{tarball_path}.files", "w") as handle:
+        for file in files:
+            handle.write(file)
+            handle.write("\n")
+
+    pmb.helpers.run.user(args, ["tar", "-cf", tarball_path, "-T",
+                                f"{tarball_path}.files"], topdir)
+
+    ci_dir = "/home/pmos/ci"
+    pmb.chroot.user(args, ["rm", "-rf", ci_dir])
+    pmb.chroot.user(args, ["mkdir", ci_dir])
+    pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"],
+                    working_dir=ci_dir)
+
+
+def run_scripts(args, topdir, scripts):
+    """ Run one of the given scripts after another, either natively or in a
+        chroot. Display a progress message and stop on error (without printing
+        a python stack trace).
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir()
+        :param scripts: return of get_ci_scripts() """
+    steps = len(scripts)
+    step = 0
+    repo_copied = False
+
+    for script_name, script in scripts.items():
+        step += 1
+
+        where = "pmbootstrap chroot"
+        if "native" in script["options"]:
+            where = "native"
+
+        script_path = f".ci/{script_name}.sh"
+        logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}"
+                     f" [{where}] ***")
+
+        if "native" in script["options"]:
+            rc = pmb.helpers.run.user(args, [script_path], topdir,
+                                      output="tui")
+            continue
+        else:
+            # Run inside pmbootstrap chroot
+            if not repo_copied:
+                copy_git_repo_to_chroot(args, topdir)
+                repo_copied = True
+
+            env = {"TESTUSER": "pmos"}
+            rc = pmb.chroot.root(args, [script_path], check=False, env=env,
+                                 working_dir="/home/pmos/ci",
+                                 output="tui")
+        if rc:
+            logging.error(f"ERROR: CI script failed: {script_name}")
+            exit(1)

pmb/config/__init__.py

@@ -1,8 +1,10 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import multiprocessing
 import os
+import pmb.parse.arch
 import sys
+from typing import List
 
 #
 # Exported functions
@@ -10,24 +12,29 @@ import sys
 from pmb.config.load import load
 from pmb.config.save import save
 from pmb.config.merge_with_args import merge_with_args
+from pmb.config.sudo import which_sudo
 
 
 #
 # Exported variables (internal configuration)
 #
-version = "1.37.0"
 pmb_src = os.path.normpath(os.path.realpath(__file__) + "/../../..")
 apk_keys_path = pmb_src + "/pmb/data/keys"
+arch_native = pmb.parse.arch.alpine_native()
 
 # apk-tools minimum version
 # https://pkgs.alpinelinux.org/packages?name=apk-tools&branch=edge
 # Update this frequently to prevent a MITM attack with an outdated version
 # (which may contain a vulnerable apk/openssl, and allows an attacker to
 # exploit the system!)
-apk_tools_min_version = {"edge": "2.12.7-r0",
+apk_tools_min_version = {"edge": "2.14.0-r5",
+                         "v3.18": "2.14.0-r2",
+                         "v3.17": "2.12.10-r1",
+                         "v3.16": "2.12.9-r3",
+                         "v3.15": "2.12.7-r3",
                          "v3.14": "2.12.7-r0",
                          "v3.13": "2.12.7-r0",
-                         "v3.12": "2.10.8-r0"}
+                         "v3.12": "2.10.8-r1"}
 
 # postmarketOS aports compatibility (checked against "version" in pmaports.cfg)
 pmaports_min_version = "7"
@@ -46,32 +53,53 @@ ondev_min_version = "0.2.0"
 # Programs that pmbootstrap expects to be available from the host system. Keep
 # in sync with README.md, and try to keep the list as small as possible. The
 # idea is to run almost everything in Alpine chroots.
-required_programs = ["git", "openssl", "ps"]
+required_programs = [
+    "git",
+    "openssl",
+    "ps",
+    "tar",
+]
+
+
+def sudo(cmd: List[str]) -> List[str]:
+    """Adapt a command to run as root."""
+    sudo = which_sudo()
+    if sudo:
+        return [sudo, *cmd]
+    else:
+        return cmd
+
 
 # Keys saved in the config file (mostly what we ask in 'pmbootstrap init')
-config_keys = ["aports",
-               "ccache_size",
-               "device",
-               "extra_packages",
-               "hostname",
-               "build_pkgs_on_install",
-               "is_default_channel",
-               "jobs",
-               "kernel",
-               "keymap",
-               "locale",
-               "mirrors_postmarketos",
-               "nonfree_firmware",
-               "nonfree_userland",
-               "ssh_keys",
-               "timezone",
-               "ui",
-               "ui_extras",
-               "user",
-               "work",
-               "boot_size",
-               "extra_space",
-               "sudo_timer"]
+config_keys = [
+    "aports",
+    "boot_size",
+    "build_default_device_arch",
+    "build_pkgs_on_install",
+    "ccache_size",
+    "device",
+    "extra_packages",
+    "extra_space",
+    "hostname",
+    "is_default_channel",
+    "jobs",
+    "kernel",
+    "keymap",
+    "locale",
+    "mirror_alpine",
+    "mirrors_postmarketos",
+    "nonfree_firmware",
+    "nonfree_userland",
+    "qemu_redir_stdio",
+    "ssh_key_glob",
+    "ssh_keys",
+    "sudo_timer",
+    "timezone",
+    "ui",
+    "ui_extras",
+    "user",
+    "work",
+]
 
 # Config file/commandline default values
 # $WORK gets replaced with the actual value for args.work (which may be
@@ -95,7 +123,7 @@ defaults = {
     "jobs": str(multiprocessing.cpu_count() + 1),
     "kernel": "stable",
     "keymap": "",
-    "locale": "C.UTF-8",
+    "locale": "en_US.UTF-8",
     "log": "$WORK/log.txt",
     "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
     # NOTE: mirrors_postmarketos variable type is supposed to be
@@ -103,16 +131,18 @@ defaults = {
     "mirrors_postmarketos": "http://mirror.postmarketos.org/postmarketos/",
     "nonfree_firmware": True,
     "nonfree_userland": False,
-    "port_distccd": "33632",
     "ssh_keys": False,
+    "ssh_key_glob": "~/.ssh/id_*.pub",
     "timezone": "GMT",
-    "ui": "weston",
+    "ui": "console",
     "ui_extras": False,
     "user": "user",
     "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
     "boot_size": "256",
     "extra_space": "0",
-    "sudo_timer": False
+    "sudo_timer": False,
+    "qemu_redir_stdio": False,
+    "build_default_device_arch": False,
 }
 
 
@@ -137,28 +167,9 @@ if "NO_COLOR" in os.environ:
     for style in styles.keys():
         styles[style] = ""
 
-
-# List of available locales taken from musl-locales package; see
-# https://pkgs.alpinelinux.org/contents?name=musl-locales
-locales = [
-    "C.UTF-8",
-    "ch_DE.UTF-8",
-    "de_CH.UTF-8",
-    "de_DE.UTF-8",
-    "en_GB.UTF-8",
-    "en_US.UTF-8",
-    "es_ES.UTF-8",
-    "fr_FR.UTF-8",
-    "it_IT.UTF-8",
-    "nb_NO.UTF-8",
-    "nl_NL.UTF-8",
-    "pt_BR.UTF-8",
-    "ru_RU.UTF-8",
-    "sv_SE.UTF-8"
-]
-
 # Supported filesystems and their fstools packages
-filesystems = {"ext2": "e2fsprogs",
+filesystems = {"btrfs": "btrfs-progs",
+               "ext2": "e2fsprogs",
                "ext4": "e2fsprogs",
                "f2fs": "f2fs-tools",
                "fat16": "dosfstools",
@@ -195,16 +206,21 @@ chroot_host_path = os.environ["PATH"] + ":/usr/sbin/"
 # $WORK gets replaced with args.work
 # $ARCH gets replaced with the chroot architecture (eg. x86_64, armhf)
 # $CHANNEL gets replaced with the release channel (e.g. edge, v21.03)
+# Use no more than one dir after /mnt/pmbootstrap, see remove_mnt_pmbootstrap.
 chroot_mount_bind = {
     "/proc": "/proc",
     "$WORK/cache_apk_$ARCH": "/var/cache/apk",
-    "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap-ccache",
+    "$WORK/cache_appstream/$ARCH/$CHANNEL": "/mnt/appstream-data",
+    "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap/ccache",
     "$WORK/cache_distfiles": "/var/cache/distfiles",
-    "$WORK/cache_git": "/mnt/pmbootstrap-git",
-    "$WORK/cache_rust": "/mnt/pmbootstrap-rust",
-    "$WORK/config_abuild": "/mnt/pmbootstrap-abuild-config",
+    "$WORK/cache_git": "/mnt/pmbootstrap/git",
+    "$WORK/cache_go": "/mnt/pmbootstrap/go",
+    "$WORK/cache_rust": "/mnt/pmbootstrap/rust",
+    "$WORK/config_abuild": "/mnt/pmbootstrap/abuild-config",
     "$WORK/config_apk_keys": "/etc/apk/keys",
-    "$WORK/packages/$CHANNEL": "/mnt/pmbootstrap-packages",
+    "$WORK/cache_sccache": "/mnt/pmbootstrap/sccache",
+    "$WORK/images_netboot": "/mnt/pmbootstrap/netboot",
+    "$WORK/packages/$CHANNEL": "/mnt/pmbootstrap/packages",
 }
 
 # Building chroots (all chroots, except for the rootfs_ chroot) get symlinks in
@@ -215,13 +231,20 @@ chroot_mount_bind = {
 # a no-go, but at least until this is resolved properly, let's cache the
 # dependencies and downloads as suggested in "Caching the Cargo home in CI":
 # https://doc.rust-lang.org/cargo/guide/cargo-home.html
+# Go: cache the directories "go env GOMODCACHE" and "go env GOCACHE" point to,
+# to avoid downloading dependencies over and over (GOMODCACHE, similar to the
+# rust depends caching described above) and to cache build artifacts (GOCACHE,
+# similar to ccache).
 chroot_home_symlinks = {
-    "/mnt/pmbootstrap-abuild-config": "/home/pmos/.abuild",
-    "/mnt/pmbootstrap-ccache": "/home/pmos/.ccache",
-    "/mnt/pmbootstrap-packages": "/home/pmos/packages/pmos",
-    "/mnt/pmbootstrap-rust/registry/index": "/home/pmos/.cargo/registry/index",
-    "/mnt/pmbootstrap-rust/registry/cache": "/home/pmos/.cargo/registry/cache",
-    "/mnt/pmbootstrap-rust/git/db": "/home/pmos/.cargo/git/db",
+    "/mnt/pmbootstrap/abuild-config": "/home/pmos/.abuild",
+    "/mnt/pmbootstrap/ccache": "/home/pmos/.ccache",
+    "/mnt/pmbootstrap/go/gocache": "/home/pmos/.cache/go-build",
+    "/mnt/pmbootstrap/go/gomodcache": "/home/pmos/go/pkg/mod",
+    "/mnt/pmbootstrap/packages": "/home/pmos/packages/pmos",
+    "/mnt/pmbootstrap/rust/git/db": "/home/pmos/.cargo/git/db",
+    "/mnt/pmbootstrap/rust/registry/cache": "/home/pmos/.cargo/registry/cache",
+    "/mnt/pmbootstrap/rust/registry/index": "/home/pmos/.cargo/registry/index",
+    "/mnt/pmbootstrap/sccache": "/home/pmos/.cache/sccache",
 }
 
 # Device nodes to be created in each chroot. Syntax for each entry:
@@ -249,34 +272,59 @@ chroot_outdated = 3600 * 24 * 2
 # specify architectures supported by Alpine here. For cross-compiling,
 # we need to generate the "musl-$ARCH", "binutils-$ARCH" and "gcc-$ARCH"
 # packages (use "pmbootstrap aportgen musl-armhf" etc.).
-build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86"]
+build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"]
 
 # Packages that will be installed in a chroot before it builds packages
 # for the first time
 build_packages = ["abuild", "build-base", "ccache", "git"]
 
+#
+# KCONFIG CHECK
+#
+# Implemented value types:
+# - boolean (e.g. '"ANDROID_PARANOID_NETWORK": False'):
+#   - False: disabled
+#   - True: enabled, either as module or built-in
+# - array (e.g. '"ANDROID_BINDER_DEVICES": ["binder", "hwbinder"]'):
+#   - each element of the array must be contained in the kernel config string,
+#     in any order. The example above would accept the following in the config:
+#       CONFIG_ANDROID_BINDER_DEVICES="hwbinder,vndbinder,binder"
+# - string (e.g. '"LSM": "lockdown,yama,loadpin,safesetid,integrity"'):
+#   - the value in the kernel config must be the same as the given string. Use
+#     this e.g. if the order of the elements is important.
+
 # Necessary kernel config options
-necessary_kconfig_options = {
+kconfig_options = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
             "ANDROID_PARANOID_NETWORK": False,
             "BLK_DEV_INITRD": True,
             "CGROUPS": True,
+            "CRYPTO_AES": True,
             "CRYPTO_XTS": True,
             "DEVTMPFS": True,
             "DM_CRYPT": True,
+            "INPUT_EVDEV": True,
             "EXT4_FS": True,
             "KINETO_GAN": False,
             "PFT": False,
-            "SAMSUNG_TUI": False,
             "SEC_RESTRICT_ROOTING": False,
             "SYSVIPC": True,
             "TMPFS_POSIX_ACL": True,
-            "TZDEV": False,
             "USE_VFB": False,
             "VT": True,
         }
     },
+    ">=2.6.0": {
+        "all": {
+            "BINFMT_ELF": True,
+        },
+    },
+    ">=3.10.0": {
+        "all": {
+            "BINFMT_SCRIPT": True,
+        },
+    },
     ">=4.0.0": {
         "all": {
             "UEVENT_HELPER": True,
@@ -288,6 +336,12 @@ necessary_kconfig_options = {
             "DEVPTS_MULTIPLE_INSTANCES": True,
         }
     },
+    "<4.14.0": {
+        "all": {
+            "SAMSUNG_TUI": False,
+            "TZDEV": False,
+        }
+    },
     "<5.2.0": {
         "armhf armv7 x86": {
             "LBDAF": True
@@ -295,40 +349,82 @@ necessary_kconfig_options = {
     }
 }
 
-# Necessary anbox kernel config options
-necessary_kconfig_options_anbox = {
+# Necessary waydroid kernel config options (android app support)
+kconfig_options_waydroid = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
-            "SQUASHFS": True,
-            "SQUASHFS_XZ": True,
-            "SQUASHFS_XATTR": True,
-            "TMPFS_XATTR": True,
-            "ASHMEM": True,
-            "ANDROID_BINDER_IPC": True,
             "ANDROID_BINDERFS": False,
-            "ANDROID_BINDER_DEVICES": ["binder", "hwbinder"],
+            "ANDROID_BINDER_DEVICES": ["binder", "hwbinder", "vndbinder"],
+            "ANDROID_BINDER_IPC": True,
+            "ANDROID_BINDER_IPC_SELFTEST": False,
+            "BLK_DEV_LOOP": True,
+            "BPF_SYSCALL": True,
+            "BRIDGE": True,
+            "BRIDGE_VLAN_FILTERING": True,
+            "CGROUP_BPF": True,
+            "FUSE_FS": True,
+            "IP_NF_MANGLE": True,
             "NETFILTER_XTABLES": True,
             "NETFILTER_XT_MATCH_COMMENT": True,
-            "IP_NF_MANGLE": True,
-            "FUSE_FS": True,
-            "BLK_DEV_LOOP": True,
+            "PSI": True,
+            "PSI_DEFAULT_DISABLED": False,
+            "SQUASHFS": True,
+            "SQUASHFS_XATTR": True,
+            "SQUASHFS_XZ": True,
+            "TMPFS_XATTR": True,
             "TUN": True,
             "VETH": True,
             "VLAN_8021Q": True,  # prerequisite for bridge
-            "BRIDGE": True,
-            "BRIDGE_VLAN_FILTERING": True,
+        }
+    },
+    ">=3.5": {
+        "all": {
+            "CROSS_MEMORY_ATTACH": True,
         }
     },
     ">=4.20.0": {
         "all": {
-            "PSI": True,  # required by userspace OOM killer in Waydroid
+            "PSI": True,  # required by userspace OOM killer
             "PSI_DEFAULT_DISABLED": False,
         }
+    },
+    "<5.18": {  # option has been dropped
+        "all": {
+            "ASHMEM": True,
+        }
     }
 }
 
-# Necessary nftables kernel config options
-necessary_kconfig_options_nftables = {
+# Necessary iwd kernel config options (inet wireless daemon)
+# Obtained from 'grep ADD_MISSING src/main.c' in iwd.git
+kconfig_options_iwd = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "ASYMMETRIC_KEY_TYPE": True,
+            "ASYMMETRIC_PUBLIC_KEY_SUBTYPE": True,
+            "CRYPTO_AES": True,
+            "CRYPTO_CBC": True,
+            "CRYPTO_CMAC": True,
+            "CRYPTO_DES": True,
+            "CRYPTO_ECB": True,
+            "CRYPTO_HMAC": True,
+            "CRYPTO_MD5": True,
+            "CRYPTO_SHA1": True,
+            "CRYPTO_SHA256": True,
+            "CRYPTO_SHA512": True,
+            "CRYPTO_USER_API_HASH": True,
+            "CRYPTO_USER_API_SKCIPHER": True,
+            "KEYS": True,
+            "KEY_DH_OPERATIONS": True,
+            "PKCS7_MESSAGE_PARSER": True,
+            "PKCS8_PRIVATE_KEY_PARSER": True,
+            "X509_CERTIFICATE_PARSER": True,
+        },
+    },
+}
+
+# Necessary nftables kernel config options (firewall)
+kconfig_options_nftables = {
     ">=3.13.0": {  # nftables support introduced here
         "all": {  # all arches
             "NETFILTER": True,
@@ -336,7 +432,6 @@ necessary_kconfig_options_nftables = {
             "NF_TABLES": True,
             "NF_TABLES_INET": True,
             "NFT_CT": True,
-            "NFT_COUNTER": True,
             "NFT_LOG": True,
             "NFT_LIMIT": True,
             "NFT_MASQ": True,
@@ -356,10 +451,15 @@ necessary_kconfig_options_nftables = {
             "IP6_NF_NAT": True,
         }
     },
+    ">=3.13.0 <5.17": {  # option has been dropped
+        "all": {  # all arches
+            "NFT_COUNTER": True,
+        },
+    },
 }
 
 # Necessary kernel config options for containers (lxc, Docker)
-necessary_kconfig_options_containers = {
+kconfig_options_containers = {
     ">=0.0.0": {  # all versions, more specifically - since >=2.5~2.6
         "all": {  # all arches
             "NAMESPACES": True,
@@ -375,7 +475,7 @@ necessary_kconfig_options_containers = {
             "CPUSETS": True,
             "KEYS": True,
             "VETH": True,
-            "BRIDGE": True,  # (also needed for anbox)
+            "BRIDGE": True,  # (also needed for waydroid)
             "BRIDGE_NETFILTER": True,
             "IP_NF_FILTER": True,
             "IP_NF_TARGET_MASQUERADE": True,
@@ -420,14 +520,19 @@ necessary_kconfig_options_containers = {
     ">=3.6": {
         "all": {  # all arches
             "MEMCG": True,
-            "MEMCG_SWAP": True,
             "DM_THIN_PROVISIONING": True,  # Storage Drivers
+            "SWAP": True,
         },
         "x86 x86_64": {  # only for x86, x86_64 (and sparc64, ia64)
-            "CONFIG_HUGETLB_PAGE": True,
+            "HUGETLB_PAGE": True,
             "CGROUP_HUGETLB": True,  # Optional section
         }
     },
+    ">=3.6 <6.1_rc1": {  # option has been dropped
+        "all": {
+            "MEMCG_SWAP": True,
+        }
+    },
     ">=3.7 <5.0": {
         "all": {
             "NF_NAT_IPV4": True,  # Needed for lxc
@@ -442,7 +547,7 @@ necessary_kconfig_options_containers = {
     },
     ">=3.9": {
         "all": {  # all arches
-            "BRIDGE_VLAN_FILTERING": True,  # Network Drivers (also for anbox)
+            "BRIDGE_VLAN_FILTERING": True,  # Network Drivers (also for waydroid)
             "MACVLAN": True,  # Network Drivers
         }
     },
@@ -469,11 +574,12 @@ necessary_kconfig_options_containers = {
     },
 }
 
-necessary_kconfig_options_zram = {
+# Necessary zram kernel config options (RAM disk with on-the-fly compression)
+kconfig_options_zram = {
     ">=3.14.0": {  # zram support introduced here
         "all": {  # all arches
-            "ZSMALLOC_STAT": True,
-            "ZRAM_MEMORY_TRACKING": True,
+            "ZRAM": True,
+            "ZSMALLOC": True,
             "CRYPTO_LZ4": True,
             "LZ4_COMPRESS": True,
             "SWAP": True,
@@ -481,6 +587,90 @@ necessary_kconfig_options_zram = {
     },
 }
 
+# Necessary netboot kernel config options
+kconfig_options_netboot = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "BLK_DEV_NBD": True,
+        }
+    },
+}
+
+# Necessary wireguard & wg-quick kernel config options
+# From https://gitweb.gentoo.org/repo/gentoo.git/tree/net-vpn/wireguard-tools/wireguard-tools-1.0.20210914.ebuild?id=76aaa1eeb6f001baaa68e6946f917ebb091bbd9d # noqa
+kconfig_options_wireguard = {
+    ">=5.6_rc1": {  # all versions
+        "all": {  # all arches
+            "WIREGUARD": True,
+            "IP_ADVANCED_ROUTER": True,
+            "IP_MULTIPLE_TABLES": True,
+            "IPV6_MULTIPLE_TABLES": True,
+            "NF_TABLES": True,
+            "NF_TABLES_IPV4": True,
+            "NF_TABLES_IPV6": True,
+            "NFT_CT": True,
+            "NFT_FIB": True,
+            "NFT_FIB_IPV4": True,
+            "NFT_FIB_IPV6": True,
+            "NF_CONNTRACK_MARK": True,
+        },
+    },
+}
+
+# Necessary file system config options
+kconfig_options_filesystems = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "BTRFS_FS": True,
+            "EXFAT_FS": True,
+            "EXT4_FS": True,
+            "F2FS_FS": True,
+        },
+    },
+}
+
+kconfig_options_usb_gadgets = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            # disable legacy gadgets
+            "USB_ETH": False,
+            "USB_FUNCTIONFS": False,
+            "USB_MASS_STORAGE": False,
+            "USB_G_SERIAL": False,
+            # enable configfs gadgets
+            "USB_CONFIGFS_RNDIS": True,  # USB networking via RNDIS
+        },
+    },
+}
+
+# Various other kernel config options
+kconfig_options_community = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "INPUT_UINPUT": True,  # buffyboard
+            "LEDS_TRIGGER_TIMER": True,  # hfd-service
+            "NETFILTER_XT_MATCH_TCPMSS": True,  # change MTU, e.g. for Wireguard
+            "NETFILTER_XT_TARGET_TCPMSS": True,  # change MTU, e.g. for Wireguard
+        },
+    },
+}
+
+# Necessary UEFI boot config options
+kconfig_options_uefi = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "EFI_STUB": True,
+            "EFI": True,
+            "DMI": True,
+            "EFI_ESRT": True,
+            "EFI_VARS_PSTORE": True,
+            "EFI_PARAMS_FROM_FDT": True,
+            "EFI_RUNTIME_WRAPPERS": True,
+            "EFI_GENERIC_STUB": True,
+        }
+    },
+}
+
 #
 # PARSE
 #
@@ -490,7 +680,9 @@ apkbuild_package_attributes = {
     "pkgdesc": {},
     "depends": {"array": True},
     "provides": {"array": True},
+    "provider_priority": {"int": True},
     "install": {"array": True},
+    "triggers": {"array": True},
 
     # UI meta-packages can specify apps in "_pmb_recommends" to be explicitly
     # installed by default, and not implicitly as dependency of the UI meta-
@@ -502,6 +694,11 @@ apkbuild_package_attributes = {
     # UI meta-packages can specify groups to which the user must be added
     # to access specific hardware such as LED indicators.
     "_pmb_groups": {"array": True},
+
+    # postmarketos-base, UI and device packages can use _pmb_select to provide
+    # additional configuration options in "pmbootstrap init" that allow
+    # selecting alternative providers for a virtual APK package.
+    "_pmb_select": {"array": True},
 }
 
 # Variables in APKBUILD files that get parsed
@@ -531,6 +728,9 @@ apkbuild_attributes = {
     "_outdir": {},
     "_config": {},
 
+    # linux-edge
+    "_depends_dev": {"array": True},
+
     # mesa
     "_llvmver": {},
 
@@ -541,15 +741,24 @@ apkbuild_attributes = {
     # git commit
     "_commit": {},
     "source": {"array": True},
+
+    # gcc
+    "_pkgbase": {},
+    "_pkgsnap": {}
 }
 
 # Reference: https://postmarketos.org/apkbuild-options
 apkbuild_custom_valid_options = [
     "!pmb:crossdirect",
     "!pmb:kconfig-check",
-    "pmb:kconfigcheck-anbox",
+    "pmb:kconfigcheck-community",
     "pmb:kconfigcheck-containers",
+    "pmb:kconfigcheck-iwd",
+    "pmb:kconfigcheck-netboot",
     "pmb:kconfigcheck-nftables",
+    "pmb:kconfigcheck-uefi",
+    "pmb:kconfigcheck-waydroid",
+    "pmb:kconfigcheck-zram",
     "pmb:cross-native",
     "pmb:gpu-accel",
     "pmb:strict",
@@ -564,7 +773,6 @@ deviceinfo_attributes = [
     "codename",
     "year",
     "dtb",
-    "modules_initfs",
     "arch",
 
     # device
@@ -584,20 +792,32 @@ deviceinfo_attributes = [
     # flash
     "flash_heimdall_partition_kernel",
     "flash_heimdall_partition_initfs",
-    "flash_heimdall_partition_system",
+    "flash_heimdall_partition_rootfs",
+    "flash_heimdall_partition_system", # deprecated
     "flash_heimdall_partition_vbmeta",
     "flash_heimdall_partition_dtbo",
     "flash_fastboot_partition_kernel",
-    "flash_fastboot_partition_system",
+    "flash_fastboot_partition_rootfs",
+    "flash_fastboot_partition_system", # deprecated
     "flash_fastboot_partition_vbmeta",
     "flash_fastboot_partition_dtbo",
+    "flash_rk_partition_kernel",
+    "flash_rk_partition_rootfs",
+    "flash_rk_partition_system", # deprecated
+    "flash_mtkclient_partition_kernel",
+    "flash_mtkclient_partition_rootfs",
+    "flash_mtkclient_partition_vbmeta",
+    "flash_mtkclient_partition_dtbo",
     "generate_legacy_uboot_initfs",
     "kernel_cmdline",
     "generate_bootimg",
+    "header_version",
     "bootimg_qcdt",
     "bootimg_mtk_mkimage",
     "bootimg_dtb_second",
+    "bootimg_custom_args",
     "flash_offset_base",
+    "flash_offset_dtb",
     "flash_offset_kernel",
     "flash_offset_ramdisk",
     "flash_offset_second",
@@ -605,13 +825,18 @@ deviceinfo_attributes = [
     "flash_pagesize",
     "flash_fastboot_max_size",
     "flash_sparse",
+    "flash_sparse_samsung_format",
     "rootfs_image_sector_size",
     "sd_embed_firmware",
     "sd_embed_firmware_step_size",
     "partition_blacklist",
     "boot_part_start",
+    "partition_type",
     "root_filesystem",
     "flash_kernel_on_update",
+    "cgpt_kpart",
+    "cgpt_kpart_start",
+    "cgpt_kpart_size",
 
     # weston
     "weston_pixman_type",
@@ -620,7 +845,7 @@ deviceinfo_attributes = [
     "keymaps",
 ]
 
-# Valid types for the 'chassis' atribute in deviceinfo
+# Valid types for the 'chassis' attribute in deviceinfo
 # See https://www.freedesktop.org/software/systemd/man/machine-info.html
 deviceinfo_chassis_types = [
     "desktop",
@@ -651,14 +876,19 @@ default_ip = "172.16.42.1"
 install_native_packages = ["cryptsetup", "util-linux", "parted"]
 install_device_packages = ["postmarketos-base"]
 
-# Groups for the default user
-install_user_groups = ["wheel", "video", "audio", "input", "plugdev", "netdev"]
-
 #
 # FLASH
 #
 
-flash_methods = ["fastboot", "heimdall", "0xffff", "uuu", "none"]
+flash_methods = [
+    "0xffff",
+    "fastboot",
+    "heimdall",
+    "mtkclient",
+    "none",
+    "rkdeveloptool",
+    "uuu",
+]
 
 # These folders will be mounted at the same location into the native
 # chroot, before the flash programs get started.
@@ -679,7 +909,7 @@ $IMAGE: Path to the combined boot/rootfs image
 $IMAGE_SPLIT_BOOT: Path to the (split) boot image
 $IMAGE_SPLIT_ROOT: Path to the (split) rootfs image
 $PARTITION_KERNEL: Partition to flash the kernel/boot.img to
-$PARTITION_SYSTEM: Partition to flash the rootfs to
+$PARTITION_ROOTFS: Partition to flash the rootfs to
 
 Fastboot specific: $KERNEL_CMDLINE
 Heimdall specific: $PARTITION_INITFS
@@ -687,10 +917,10 @@ uuu specific: $UUU_SCRIPT
 """
 flashers = {
     "fastboot": {
-        "depends": ["android-tools", "avbtool"],
+        "depends": [],  # pmaports.cfg: supported_fastboot_depends
         "actions": {
             "list_devices": [["fastboot", "devices", "-l"]],
-            "flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
+            "flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
                               "$IMAGE"]],
             "flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
                               "$BOOT/boot.img$FLAVOR"]],
@@ -706,6 +936,8 @@ flashers = {
                             "$BOOT/dtbo.img"]],
             "boot": [["fastboot", "--cmdline", "$KERNEL_CMDLINE",
                       "boot", "$BOOT/boot.img$FLAVOR"]],
+            "flash_lk2nd": [["fastboot", "flash", "$PARTITION_KERNEL",
+                             "$BOOT/lk2nd.img"]]
         },
     },
     # Some devices provide Fastboot but using Android boot images is not
@@ -719,7 +951,7 @@ flashers = {
         "depends": ["android-tools"],
         "actions": {
             "list_devices": [["fastboot", "devices", "-l"]],
-            "flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
+            "flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
                               "$IMAGE_SPLIT_ROOT"]],
             "flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
                               "$IMAGE_SPLIT_BOOT"]],
@@ -737,7 +969,7 @@ flashers = {
             "list_devices": [["heimdall", "detect"]],
             "flash_rootfs": [
                 ["heimdall_wait_for_device.sh"],
-                ["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
+                ["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
             "flash_kernel": [["heimdall_flash_kernel.sh",
                               "$BOOT/initramfs$FLAVOR", "$PARTITION_INITFS",
                               "$BOOT/vmlinuz$FLAVOR$DTB",
@@ -747,12 +979,12 @@ flashers = {
     # Some Samsung devices need a 'boot.img' file, just like the one generated
     # fastboot compatible devices. Example: s7562, n7100
     "heimdall-bootimg": {
-        "depends": ["heimdall", "avbtool"],
+        "depends": [],  # pmaports.cfg: supported_heimdall_depends
         "actions": {
             "list_devices": [["heimdall", "detect"]],
             "flash_rootfs": [
                 ["heimdall_wait_for_device.sh"],
-                ["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
+                ["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
             "flash_kernel": [
                 ["heimdall_wait_for_device.sh"],
                 ["heimdall", "flash", "--$PARTITION_KERNEL",
@@ -762,7 +994,10 @@ flashers = {
                     "--padding_size", "$FLASH_PAGESIZE",
                     "--output", "/vbmeta.img"],
                 ["heimdall", "flash", "--$PARTITION_VBMETA", "/vbmeta.img"],
-                ["rm", "-f", "/vbmeta.img"]]
+                ["rm", "-f", "/vbmeta.img"]],
+            "flash_lk2nd": [
+                ["heimdall_wait_for_device.sh"],
+                ["heimdall", "flash", "--$PARTITION_KERNEL", "$BOOT/lk2nd.img"]]
         },
     },
     "adb": {
@@ -785,6 +1020,42 @@ flashers = {
                 ["uuu", "flash_script.lst"],
             ],
         },
+    },
+    "rkdeveloptool": {
+        "split": True,
+        "depends": ["rkdeveloptool"],
+        "actions": {
+            "list_devices": [["rkdeveloptool", "list"]],
+            "flash_rootfs": [
+                ["rkdeveloptool", "write-partition", "$PARTITION_ROOTFS",
+                    "$IMAGE_SPLIT_ROOT"]
+            ],
+            "flash_kernel": [
+                ["rkdeveloptool", "write-partition", "$PARTITION_KERNEL",
+                    "$IMAGE_SPLIT_BOOT"]
+            ],
+        },
+    },
+    "mtkclient": {
+        "depends": ["mtkclient"],
+        "actions": {
+            "flash_rootfs": [["mtk", "w", "$PARTITION_ROOTFS",
+                              "$IMAGE"]],
+            "flash_kernel": [["mtk", "w", "$PARTITION_KERNEL",
+                              "$BOOT/boot.img$FLAVOR"]],
+            "flash_vbmeta": [
+                # Generate vbmeta image with "disable verification" flag
+                ["avbtool", "make_vbmeta_image", "--flags", "2",
+                    "--padding_size", "$FLASH_PAGESIZE",
+                    "--output", "/vbmeta.img"],
+                ["mtk", "w", "$PARTITION_VBMETA", "/vbmeta.img"],
+                ["rm", "-f", "/vbmeta.img"]
+            ],
+            "flash_dtbo": [["mtk", "w", "$PARTITION_DTBO",
+                            "$BOOT/dtbo.img"]],
+            "flash_lk2nd": [["mtk", "w", "$PARTITION_KERNEL",
+                             "$BOOT/lk2nd.img"]]
+        }
     }
 }
 
@@ -852,9 +1123,15 @@ newapkbuild_arguments_switches_other = [
 # Patterns of package names to ignore for automatic pmaport upgrading
 # ("pmbootstrap aportupgrade --all")
 upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*",
-                  "*-aarch64", "*-armhf", "*-armv7"]
+                  "*-aarch64", "*-armhf", "*-armv7", "*-riscv64"]
 
 #
 # SIDELOAD
 #
 sideload_sudo_prompt = "[sudo] password for %u@%h: "
+
+#
+# CI
+#
+# Valid options für 'pmbootstrap ci', see https://postmarketos.org/pmb-ci
+ci_valid_options = ["native", "slow"]

pmb/config/init.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -11,9 +11,11 @@ import pmb.config
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.devices
+import pmb.helpers.git
 import pmb.helpers.http
 import pmb.helpers.logging
 import pmb.helpers.other
+import pmb.helpers.pmaports
 import pmb.helpers.run
 import pmb.helpers.ui
 import pmb.chroot.zap
@@ -32,6 +34,23 @@ def require_programs():
                            f" {', '.join(missing)}")
 
 
+def ask_for_username(args):
+    """
+    Ask for a reasonable username for the non-root user.
+
+    :returns: the username
+    """
+    while True:
+        ret = pmb.helpers.cli.ask("Username", None, args.user, False,
+                                  "[a-z_][a-z0-9_-]*")
+        if ret == "root":
+            logging.fatal("ERROR: don't put \"root\" here. This is about"
+                          " creating an additional non-root user. Don't worry,"
+                          " the root user will also be created ;)")
+            continue
+        return ret
+
+
 def ask_for_work_path(args):
     """
     Ask for the work path, until we can create it (when it does not exist) and
@@ -47,7 +66,7 @@ def ask_for_work_path(args):
     while True:
         try:
             work = os.path.expanduser(pmb.helpers.cli.ask(
-                args, "Work path", None, args.work, False))
+                "Work path", None, args.work, False))
             work = os.path.realpath(work)
             exists = os.path.exists(work)
 
@@ -63,10 +82,12 @@ def ask_for_work_path(args):
             if not exists:
                 os.makedirs(work, 0o700, True)
 
-            if not os.listdir(work):
-                # Directory is empty, either because we just created it or
-                # because user created it before running pmbootstrap init
-                with open(f"{work}/version", "w") as handle:
+            # If the version file doesn't exists yet because we either just
+            # created the work directory or the user has deleted it for
+            # whatever reason then we need to write initialize it.
+            work_version_file = f"{work}/version"
+            if not os.path.isfile(work_version_file):
+                with open(work_version_file, "w") as handle:
                     handle.write(f"{pmb.config.work_version}\n")
 
             # Create cache_git dir, so it is owned by the host system's user
@@ -89,7 +110,10 @@ def ask_for_channel(args):
     # List channels
     logging.info("Choose the postmarketOS release channel.")
     logging.info(f"Available ({count}):")
-    for channel, channel_data in channels_cfg["channels"].items():
+    # Only show the first 3 releases. This includes edge, the latest supported
+    # release plus one. Should be a good solution until new needs arrive when
+    # we might want to have a custom channels.cfg attribute.
+    for channel, channel_data in list(channels_cfg["channels"].items())[:3]:
         logging.info(f"* {channel}: {channel_data['description']}")
 
     # Default for first run: "recommended" from channels.cfg
@@ -103,7 +127,7 @@ def ask_for_channel(args):
 
     # Ask until user gives valid channel
     while True:
-        ret = pmb.helpers.cli.ask(args, "Channel", None, default,
+        ret = pmb.helpers.cli.ask("Channel", None, default,
                                   complete=choices)
         if ret in choices:
             return ret
@@ -111,8 +135,7 @@ def ask_for_channel(args):
                       " from the list above.")
 
 
-def ask_for_ui(args, device):
-    info = pmb.parse.deviceinfo(args, device)
+def ask_for_ui(args, info):
     ui_list = pmb.helpers.ui.list(args, info["arch"])
     hidden_ui_count = 0
     device_is_accelerated = info.get("gpu_accelerated") == "true"
@@ -126,19 +149,22 @@ def ask_for_ui(args, device):
                 ui_list.pop(i)
                 hidden_ui_count += 1
 
+    # Get default
+    default = args.ui
+    if default not in dict(ui_list).keys():
+        default = pmb.config.defaults["ui"]
+
     logging.info(f"Available user interfaces ({len(ui_list) - 1}): ")
     ui_completion_list = []
     for ui in ui_list:
         logging.info(f"* {ui[0]}: {ui[1]}")
         ui_completion_list.append(ui[0])
     if hidden_ui_count > 0:
-        logging.info(f"NOTE: {hidden_ui_count} user interfaces are not"
-                     " available. If device supports GPU acceleration,"
-                     " set \"deviceinfo_gpu_accelerated\" to make UIs"
-                     " available. See: <https://wiki.postmarketos.org/wiki/"
-                     "Deviceinfo_reference")
+        logging.info(f"NOTE: {hidden_ui_count} UIs are hidden because"
+                     " \"deviceinfo_gpu_accelerated\" is not set (see"
+                     " https://postmarketos.org/deviceinfo).")
     while True:
-        ret = pmb.helpers.cli.ask(args, "User interface", None, args.ui, True,
+        ret = pmb.helpers.cli.ask("User interface", None, default, True,
                                   complete=ui_completion_list)
         if ret in dict(ui_list).keys():
             return ret
@@ -163,8 +189,7 @@ def ask_for_ui_extras(args, ui):
                                    default=args.ui_extras)
 
 
-def ask_for_keymaps(args, device):
-    info = pmb.parse.deviceinfo(args, device)
+def ask_for_keymaps(args, info):
     if "keymaps" not in info or info["keymaps"].strip() == "":
         return ""
     options = info["keymaps"].split(' ')
@@ -174,7 +199,7 @@ def ask_for_keymaps(args, device):
         args.keymap = options[0]
 
     while True:
-        ret = pmb.helpers.cli.ask(args, "Keymap", None, args.keymap,
+        ret = pmb.helpers.cli.ask("Keymap", None, args.keymap,
                                   True, complete=options)
         if ret in options:
             return ret
@@ -208,6 +233,80 @@ def ask_for_timezone(args):
     return "GMT"
 
 
+def ask_for_provider_select(args, apkbuild, providers_cfg):
+    """
+    Ask for selectable providers that are specified using "_pmb_select"
+    in a APKBUILD.
+
+    :param apkbuild: the APKBUILD with the _pmb_select
+    :param providers_cfg: the configuration section with previously selected
+                          providers. Updated with new providers after selection
+    """
+    for select in apkbuild["_pmb_select"]:
+        providers = pmb.helpers.pmaports.find_providers(args, select)
+        logging.info(f"Available providers for {select} ({len(providers)}):")
+
+        has_default = False
+        providers_short = {}
+        last_selected = providers_cfg.get(select, 'default')
+
+        for pkgname, pkg in providers:
+            # Strip provider prefix if possible
+            short = pkgname
+            if short.startswith(f'{select}-'):
+                short = short[len(f"{select}-"):]
+
+            # Allow selecting the package using both short and long name
+            providers_short[pkgname] = pkgname
+            providers_short[short] = pkgname
+
+            if pkgname == last_selected:
+                last_selected = short
+
+            if not has_default and pkg.get('provider_priority', 0) != 0:
+                # Display as default provider
+                styles = pmb.config.styles
+                logging.info(f"* {short}: {pkg['pkgdesc']} "
+                             f"{styles['BOLD']}(default){styles['END']}")
+                has_default = True
+            else:
+                logging.info(f"* {short}: {pkg['pkgdesc']}")
+
+        while True:
+            ret = pmb.helpers.cli.ask("Provider", None, last_selected, True,
+                                      complete=providers_short.keys())
+
+            if has_default and ret == 'default':
+                # Selecting default means to not select any provider explicitly
+                # In other words, apk chooses it automatically based on
+                # "provider_priority"
+                if select in providers_cfg:
+                    del providers_cfg[select]
+                break
+            if ret in providers_short:
+                providers_cfg[select] = providers_short[ret]
+                break
+            logging.fatal("ERROR: Invalid provider specified, please type in"
+                          " one from the list above.")
+
+
+def ask_for_provider_select_pkg(args, pkgname, providers_cfg):
+    """
+    Look up the APKBUILD for the specified pkgname and ask for selectable
+    providers that are specified using "_pmb_select".
+
+    :param pkgname: name of the package to search APKBUILD for
+    :param providers_cfg: the configuration section with previously selected
+                          providers. Updated with new providers after selection
+    """
+    apkbuild = pmb.helpers.pmaports.get(args, pkgname,
+                                        subpackages=False, must_exist=False)
+    if not apkbuild:
+        return
+
+    ask_for_provider_select(args, apkbuild, providers_cfg)
+
+
 def ask_for_device_kernel(args, device):
     """
     Ask for the kernel that should be used with the device.
@@ -241,7 +340,7 @@ def ask_for_device_kernel(args, device):
     for type in sorted(kernels.keys()):
         logging.info(f"* {type}: {kernels[type]}")
     while True:
-        ret = pmb.helpers.cli.ask(args, "Kernel", None, default, True,
+        ret = pmb.helpers.cli.ask("Kernel", None, default, True,
                                   complete=kernels)
         if ret in kernels.keys():
             return ret
@@ -266,7 +365,7 @@ def ask_for_device_nonfree(args, device):
            "userland": args.nonfree_userland}
     if not apkbuild_path:
         return ret
-    apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
+    apkbuild = pmb.parse.apkbuild(apkbuild_path)
 
     # Only run when there is a "nonfree" subpackage
     nonfree_found = False
@@ -310,7 +409,7 @@ def ask_for_device(args):
         current_codename = args.device.split("-", 1)[1]
 
     while True:
-        vendor = pmb.helpers.cli.ask(args, "Vendor", None, current_vendor,
+        vendor = pmb.helpers.cli.ask("Vendor", None, current_vendor,
                                      False, r"[a-z0-9]+", vendors)
 
         new_vendor = vendor not in vendors
@@ -332,7 +431,7 @@ def ask_for_device(args):
 
         if current_vendor != vendor:
             current_codename = ''
-        codename = pmb.helpers.cli.ask(args, "Device codename", None,
+        codename = pmb.helpers.cli.ask("Device codename", None,
                                        current_codename, False, r"[a-z0-9]+",
                                        codenames)
 
@@ -390,20 +489,20 @@ def ask_for_additional_options(args, cfg):
                  " enough to fit the rootfs (pmbootstrap#1904)."
                  " How much extra free space do you want to add to the image"
                  " (in MB)?")
-    answer = pmb.helpers.cli.ask(args, "Extra space size", None,
+    answer = pmb.helpers.cli.ask("Extra space size", None,
                                  args.extra_space, validation_regex="^[0-9]+$")
     cfg["pmbootstrap"]["extra_space"] = answer
 
     # Boot size
     logging.info("What should be the boot partition size (in MB)?")
-    answer = pmb.helpers.cli.ask(args, "Boot size", None, args.boot_size,
+    answer = pmb.helpers.cli.ask("Boot size", None, args.boot_size,
                                  validation_regex="^[1-9][0-9]*$")
     cfg["pmbootstrap"]["boot_size"] = answer
 
     # Parallel job count
     logging.info("How many jobs should run parallel on this machine, when"
                  " compiling?")
-    answer = pmb.helpers.cli.ask(args, "Jobs", None, args.jobs,
+    answer = pmb.helpers.cli.ask("Jobs", None, args.jobs,
                                  validation_regex="^[1-9][0-9]*$")
     cfg["pmbootstrap"]["jobs"] = answer
 
@@ -414,7 +513,7 @@ def ask_for_additional_options(args, cfg):
                  " current usage with 'pmbootstrap stats'. Answer with 0 for"
                  " infinite.")
     regex = "0|[0-9]+(k|M|G|T|Ki|Mi|Gi|Ti)"
-    answer = pmb.helpers.cli.ask(args, "Ccache size", None, args.ccache_size,
+    answer = pmb.helpers.cli.ask("Ccache size", None, args.ccache_size,
                                  lowercase_answer=False,
                                  validation_regex=regex)
     cfg["pmbootstrap"]["ccache_size"] = answer
@@ -483,7 +582,7 @@ def ask_for_mirror(args):
     mirrors_list = []
     # require one valid mirror index selected by user
     while len(mirrors_list) != 1:
-        answer = pmb.helpers.cli.ask(args, "Select a mirror", None,
+        answer = pmb.helpers.cli.ask("Select a mirror", None,
                                      ",".join(mirror_indexes),
                                      validation_regex=regex)
         mirrors_list = []
@@ -499,8 +598,7 @@ def ask_for_mirror(args):
 
 def ask_for_hostname(args, device):
     while True:
-        ret = pmb.helpers.cli.ask(args,
-                                  "Device hostname (short form, e.g. 'foo')",
+        ret = pmb.helpers.cli.ask("Device hostname (short form, e.g. 'foo')",
                                   None, (args.hostname or device), True)
         if not pmb.helpers.other.validate_hostname(ret):
             continue
@@ -528,15 +626,36 @@ def ask_build_pkgs_on_install(args):
                                    default=args.build_pkgs_on_install)
 
 
+def get_locales():
+    ret = []
+    list_path = f"{pmb.config.pmb_src}/pmb/data/locales"
+    with open(list_path, "r") as handle:
+        for line in handle:
+            ret += [line.rstrip()]
+    return ret
+
+
 def ask_for_locale(args):
-    locales = pmb.config.locales
-    logging.info(f"Available locales ({len(locales)}): {', '.join(locales)}")
-    return pmb.helpers.cli.ask(args, "Choose default locale for installation",
-                               choices=None,
-                               default=args.locale,
-                               lowercase_answer=False,
-                               validation_regex="|".join(locales),
-                               complete=locales)
+    locales = get_locales()
+    logging.info("Choose your preferred locale, like e.g. en_US. Only UTF-8"
+                 " is supported, it gets appended automatically. Use"
+                 " tab-completion if needed.")
+
+    while True:
+        ret = pmb.helpers.cli.ask("Locale",
+                                  choices=None,
+                                  default=args.locale.replace(".UTF-8", ""),
+                                  lowercase_answer=False,
+                                  complete=locales)
+        ret = ret.replace(".UTF-8", "")
+        if ret not in locales:
+            logging.info("WARNING: this locale is not in the list of known"
+                         " valid locales.")
+            if pmb.helpers.cli.ask() != "y":
+                # Ask again
+                continue
+
+        return f"{ret}.UTF-8"
 
 
 def frontend(args):
@@ -562,6 +681,15 @@ def frontend(args):
     pmb.config.pmaports.switch_to_channel_branch(args, channel)
     cfg["pmbootstrap"]["is_default_channel"] = "False"
 
+    # Copy the git hooks if master was checked out. (Don't symlink them and
+    # only do it on master, so the git hooks don't change unexpectedly when
+    # having a random branch checked out.)
+    branch_current = pmb.helpers.git.rev_parse(args, args.aports,
+                                               extra_args=["--abbrev-ref"])
+    if branch_current == "master":
+        logging.info("NOTE: pmaports is on master branch, copying git hooks.")
+        pmb.config.pmaports.install_githooks(args)
+
     # Device
     device, device_exists, kernel, nonfree = ask_for_device(args)
     cfg["pmbootstrap"]["device"] = device
@@ -569,25 +697,32 @@ def frontend(args):
     cfg["pmbootstrap"]["nonfree_firmware"] = str(nonfree["firmware"])
     cfg["pmbootstrap"]["nonfree_userland"] = str(nonfree["userland"])
 
+    info = pmb.parse.deviceinfo(args, device)
+    apkbuild_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
+    if apkbuild_path:
+        apkbuild = pmb.parse.apkbuild(apkbuild_path)
+        ask_for_provider_select(args, apkbuild, cfg["providers"])
+
     # Device keymap
     if device_exists:
-        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, device)
+        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, info)
+
+    cfg["pmbootstrap"]["user"] = ask_for_username(args)
+    ask_for_provider_select_pkg(args, "postmarketos-base", cfg["providers"])
 
-    # Username
-    cfg["pmbootstrap"]["user"] = pmb.helpers.cli.ask(args, "Username", None,
-                                                     args.user, False,
-                                                     "[a-z_][a-z0-9_-]*")
     # UI and various build options
-    ui = ask_for_ui(args, device)
+    ui = ask_for_ui(args, info)
     cfg["pmbootstrap"]["ui"] = ui
     cfg["pmbootstrap"]["ui_extras"] = str(ask_for_ui_extras(args, ui))
+    ask_for_provider_select_pkg(args, f"postmarketos-ui-{ui}",
+                                cfg["providers"])
     ask_for_additional_options(args, cfg)
 
     # Extra packages to be installed to rootfs
     logging.info("Additional packages that will be installed to rootfs."
                  " Specify them in a comma separated list (e.g.: vim,file)"
                  " or \"none\"")
-    extra = pmb.helpers.cli.ask(args, "Extra packages", None,
+    extra = pmb.helpers.cli.ask("Extra packages", None,
                                 args.extra_packages,
                                 validation_regex=r"^([-.+\w]+)(,[-.+\w]+)*$")
     cfg["pmbootstrap"]["extra_packages"] = extra
@@ -620,7 +755,7 @@ def frontend(args):
             pmb.helpers.cli.confirm(
                 args, "Zap existing chroots to apply configuration?",
                 default=True)):
-        setattr(args, "deviceinfo", pmb.parse.deviceinfo(args, device=device))
+        setattr(args, "deviceinfo", info)
 
         # Do not zap any existing packages or cache_http directories
         pmb.chroot.zap(args, confirm=False)

pmb/config/load.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import configparser
@@ -13,6 +13,8 @@ def load(args):
 
     if "pmbootstrap" not in cfg:
         cfg["pmbootstrap"] = {}
+    if "providers" not in cfg:
+        cfg["providers"] = {}
 
     for key in pmb.config.defaults:
         if key in pmb.config.config_keys and key not in cfg["pmbootstrap"]:

pmb/config/merge_with_args.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config
 
@@ -29,6 +29,7 @@ def merge_with_args(args):
                 if isinstance(default, bool):
                     value = (value.lower() == "true")
             setattr(args, key, value)
+    setattr(args, 'selected_providers', cfg['providers'])
 
     # Use defaults from pmb.config.defaults
     for key, value in pmb.config.defaults.items():

pmb/config/pmaports.py

@@ -1,8 +1,9 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 import logging
 import os
+import sys
 
 import pmb.config
 import pmb.helpers.git
@@ -58,7 +59,7 @@ def check_version_pmaports(real):
 
 def check_version_pmbootstrap(min):
     # Compare versions
-    real = pmb.config.version
+    real = pmb.__version__
     if pmb.parse.version.compare(real, min) >= 0:
         return
 
@@ -84,14 +85,14 @@ def read_config(args):
     """ Read and verify pmaports.cfg. """
     # Try cache first
     cache_key = "pmb.config.pmaports.read_config"
-    if args.cache[cache_key]:
-        return args.cache[cache_key]
+    if pmb.helpers.other.cache[cache_key]:
+        return pmb.helpers.other.cache[cache_key]
 
     # Migration message
     if not os.path.exists(args.aports):
-        raise RuntimeError("We have split the aports repository from the"
-                           " pmbootstrap repository (#383). Please run"
-                           " 'pmbootstrap init' again to clone it.")
+        logging.error(f"ERROR: pmaports dir not found: {args.aports}")
+        logging.error("Did you run 'pmbootstrap init'?")
+        sys.exit(1)
 
     # Require the config
     path_cfg = args.aports + "/pmaports.cfg"
@@ -112,7 +113,7 @@ def read_config(args):
     ret["channel"] = pmb.helpers.pmaports.get_channel_new(ret["channel"])
 
     # Cache and return
-    args.cache[cache_key] = ret
+    pmb.helpers.other.cache[cache_key] = ret
     return ret
 
 
@@ -186,8 +187,22 @@ def switch_to_channel_branch(args, channel_new):
                            f"{args.aports}")
 
     # Invalidate all caches
-    pmb.helpers.args.add_cache(args)
+    pmb.helpers.other.init_cache()
 
     # Verify pmaports.cfg on new branch
     read_config(args)
     return True
+
+
+def install_githooks(args):
+    hooks_dir = os.path.join(args.aports, ".githooks")
+    if not os.path.exists(hooks_dir):
+        logging.info("No .githooks dir found")
+        return
+    for h in os.listdir(hooks_dir):
+        src = os.path.join(hooks_dir, h)
+        # Use git default hooks dir so users can ignore our hooks
+        # if they dislike them by setting "core.hooksPath" git config
+        dst = os.path.join(args.aports, ".git", "hooks", h)
+        if pmb.helpers.run.user(args, ["cp", src, dst], check=False):
+            logging.warning(f"WARNING: Copying git hook failed: {dst}")

pmb/config/save.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/config/sudo.py

@@ -0,0 +1,36 @@
+# Copyright 2023 Anjandev Momi
+# SPDX-License-Identifier: GPL-3.0-or-later
+import os
+import shutil
+from functools import lru_cache
+from typing import Optional
+
+
+@lru_cache()
+def which_sudo() -> Optional[str]:
+    """Returns a command required to run commands as root, if any.
+
+    Find whether sudo or doas is installed for commands that require root.
+    Allows user to override preferred sudo with PMB_SUDO env variable.
+    """
+
+    if os.getuid() == 0:
+        return None
+
+    supported_sudos = ['doas', 'sudo']
+
+    user_set_sudo = os.getenv("PMB_SUDO")
+    if user_set_sudo is not None:
+        if shutil.which(user_set_sudo) is None:
+            raise RuntimeError("PMB_SUDO environmental variable is set to"
+                               f" {user_set_sudo} but pmbootstrap cannot find"
+                               " this command on your system.")
+        return user_set_sudo
+
+    for sudo in supported_sudos:
+        if shutil.which(sudo) is not None:
+            return sudo
+
+    raise RuntimeError("Can't find sudo or doas required to run pmbootstrap."
+                       " Please install sudo, doas, or specify your own sudo"
+                       " with the PMB_SUDO environmental variable.")

pmb/config/workdir.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """ Save, read, verify workdir state related information in $WORK/workdir.cfg,
     for example the init dates of the chroots. This is not saved in

pmb/data/keys/alpine-devel@lists.alpinelinux.org-5e69ca50.rsa.pub

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXEJ8uVwJPODshTkf2BH
+pH5fVVDppOa974+IQJsZDmGd3Ny0dcd+WwYUhNFUW3bAfc3/egaMWCaprfaHn+oS
+4ddbOFgbX8JCHdru/QMAAU0aEWSMybfJGA569c38fNUF/puX6XK/y0lD2SS3YQ/a
+oJ5jb5eNrQGR1HHMAd0G9WC4JeZ6WkVTkrcOw55F00aUPGEjejreXBerhTyFdabo
+dSfc1TILWIYD742Lkm82UBOPsOSdSfOdsMOOkSXxhdCJuCQQ70DHkw7Epy9r+X33
+ybI4r1cARcV75OviyhD8CFhAlapLKaYnRFqFxlA515e6h8i8ih/v3MSEW17cCK0b
+QwIDAQAB
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-60ac2099.rsa.pub

@@ -0,0 +1,9 @@
+-----BEGIN PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwR4uJVtJOnOFGchnMW5Y
+j5/waBdG1u5BTMlH+iQMcV5+VgWhmpZHJCBz3ocD+0IGk2I68S5TDOHec/GSC0lv
+6R9o6F7h429GmgPgVKQsc8mPTPtbjJMuLLs4xKc+viCplXc0Nc0ZoHmCH4da6fCV
+tdpHQjVe6F9zjdquZ4RjV6R6JTiN9v924dGMAkbW/xXmamtz51FzondKC52Gh8Mo
+/oA0/T0KsCMCi7tb4QNQUYrf+Xcha9uus4ww1kWNZyfXJB87a2kORLiWMfs2IBBJ
+TmZ2Fnk0JnHDb8Oknxd9PvJPT0mvyT8DA+KIAPqNvOjUXP4bnjEHJcoCP9S5HkGC
+IQIDAQAB
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-6165ee59.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAutQkua2CAig4VFSJ7v54
+ALyu/J1WB3oni7qwCZD3veURw7HxpNAj9hR+S5N/pNeZgubQvJWyaPuQDm7PTs1+
+tFGiYNfAsiibX6Rv0wci3M+z2XEVAeR9Vzg6v4qoofDyoTbovn2LztaNEjTkB+oK
+tlvpNhg1zhou0jDVYFniEXvzjckxswHVb8cT0OMTKHALyLPrPOJzVtM9C1ew2Nnc
+3848xLiApMu3NBk0JqfcS3Bo5Y2b1FRVBvdt+2gFoKZix1MnZdAEZ8xQzL/a0YS5
+Hd0wj5+EEKHfOd3A75uPa/WQmA+o0cBFfrzm69QDcSJSwGpzWrD1ScH3AK8nWvoj
+v7e9gukK/9yl1b4fQQ00vttwJPSgm9EnfPHLAtgXkRloI27H6/PuLoNvSAMQwuCD
+hQRlyGLPBETKkHeodfLoULjhDi1K2gKJTMhtbnUcAA7nEphkMhPWkBpgFdrH+5z4
+Lxy+3ek0cqcI7K68EtrffU8jtUj9LFTUC8dERaIBs7NgQ/LfDbDfGh9g6qVj1hZl
+k9aaIPTm/xsi8v3u+0qaq7KzIBc9s59JOoA8TlpOaYdVgSQhHHLBaahOuAigH+VI
+isbC9vmqsThF2QdDtQt37keuqoda2E6sL7PUvIyVXDRfwX7uMDjlzTxHTymvq2Ck
+htBqojBnThmjJQFgZXocHG8CAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-61666e3f.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlEyxkHggKCXC2Wf5Mzx4
+nZLFZvU2bgcA3exfNPO/g1YunKfQY+Jg4fr6tJUUTZ3XZUrhmLNWvpvSwDS19ZmC
+IXOu0+V94aNgnhMsk9rr59I8qcbsQGIBoHzuAl8NzZCgdbEXkiY90w1skUw8J57z
+qCsMBydAueMXuWqF5nGtYbi5vHwK42PffpiZ7G5Kjwn8nYMW5IZdL6ZnMEVJUWC9
+I4waeKg0yskczYDmZUEAtrn3laX9677ToCpiKrvmZYjlGl0BaGp3cxggP2xaDbUq
+qfFxWNgvUAb3pXD09JM6Mt6HSIJaFc9vQbrKB9KT515y763j5CC2KUsilszKi3mB
+HYe5PoebdjS7D1Oh+tRqfegU2IImzSwW3iwA7PJvefFuc/kNIijfS/gH/cAqAK6z
+bhdOtE/zc7TtqW2Wn5Y03jIZdtm12CxSxwgtCF1NPyEWyIxAQUX9ACb3M0FAZ61n
+fpPrvwTaIIxxZ01L3IzPLpbc44x/DhJIEU+iDt6IMTrHOphD9MCG4631eIdB0H1b
+6zbNX1CXTsafqHRFV9XmYYIeOMggmd90s3xIbEujA6HKNP/gwzO6CDJ+nHFDEqoF
+SkxRdTkEqjTjVKieURW7Swv7zpfu5PrsrrkyGnsRrBJJzXlm2FOOxnbI2iSL1B5F
+rO5kbUxFeZUIDq+7Yv4kLWcCAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616a9724.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnC+bR4bHf/L6QdU4puhQ
+gl1MHePszRC38bzvVFDUJsmCaMCL2suCs2A2yxAgGb9pu9AJYLAmxQC4mM3jNqhg
+/E7yuaBbek3O02zN/ctvflJ250wZCy+z0ZGIp1ak6pu1j14IwHokl9j36zNfGtfv
+ADVOcdpWITFFlPqwq1qt/H3UsKVmtiF3BNWWTeUEQwKvlU8ymxgS99yn0+4OPyNT
+L3EUeS+NQJtDS01unau0t7LnjUXn+XIneWny8bIYOQCuVR6s/gpIGuhBaUqwaJOw
+7jkJZYF2Ij7uPb4b5/R3vX2FfxxqEHqssFSg8FFUNTZz3qNZs0CRVyfA972g9WkJ
+hPfn31pQYil4QGRibCMIeU27YAEjXoqfJKEPh4UWMQsQLrEfdGfb8VgwrPbniGfU
+L3jKJR3VAafL9330iawzVQDlIlwGl6u77gEXMl9K0pfazunYhAp+BMP+9ot5ckK+
+osmrqj11qMESsAj083GeFdfV3pXEIwUytaB0AKEht9DbqUfiE/oeZ/LAXgySMtVC
+sbC4ESmgVeY2xSBIJdDyUap7FR49GGrw0W49NUv9gRgQtGGaNVQQO9oGL2PBC41P
+iWF9GLoX30HIz1P8PF/cZvicSSPkQf2Z6TV+t0ebdGNS5DjapdnCrq8m9Z0pyKsQ
+uxAL2a7zX8l5i1CZh1ycUGsCAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616abc23.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0MfCDrhODRCIxR9Dep1s
+eXafh5CE5BrF4WbCgCsevyPIdvTeyIaW4vmO3bbG4VzhogDZju+R3IQYFuhoXP5v
+Y+zYJGnwrgz3r5wYAvPnLEs1+dtDKYOgJXQj+wLJBW1mzRDL8FoRXOe5iRmn1EFS
+wZ1DoUvyu7/J5r0itKicZp3QKED6YoilXed+1vnS4Sk0mzN4smuMR9eO1mMCqNp9
+9KTfRDHTbakIHwasECCXCp50uXdoW6ig/xUAFanpm9LtK6jctNDbXDhQmgvAaLXZ
+LvFqoaYJ/CvWkyYCgL6qxvMvVmPoRv7OPcyni4xR/WgWa0MSaEWjgPx3+yj9fiMA
+1S02pFWFDOr5OUF/O4YhFJvUCOtVsUPPfA/Lj6faL0h5QI9mQhy5Zb9TTaS9jB6p
+Lw7u0dJlrjFedk8KTJdFCcaGYHP6kNPnOxMylcB/5WcztXZVQD5WpCicGNBxCGMm
+W64SgrV7M07gQfL/32QLsdqPUf0i8hoVD8wfQ3EpbQzv6Fk1Cn90bZqZafg8XWGY
+wddhkXk7egrr23Djv37V2okjzdqoyLBYBxMz63qQzFoAVv5VoY2NDTbXYUYytOvG
+GJ1afYDRVWrExCech1mX5ZVUB1br6WM+psFLJFoBFl6mDmiYt0vMYBddKISsvwLl
+IJQkzDwtXzT2cSjoj3T5QekCAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616ac3bc.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvaaoSLab+IluixwKV5Od
+0gib2YurjPatGIbn5Ov2DLUFYiebj2oJINXJSwUOO+4WcuHFEqiL/1rya+k5hLZt
+hnPL1tn6QD4rESznvGSasRCQNT2vS/oyZbTYJRyAtFkEYLlq0t3S3xBxxHWuvIf0
+qVxVNYpQWyM3N9RIeYBR/euXKJXileSHk/uq1I5wTC0XBIHWcthczGN0m9wBEiWS
+0m3cnPk4q0Ea8mUJ91Rqob19qETz6VbSPYYpZk3qOycjKosuwcuzoMpwU8KRiMFd
+5LHtX0Hx85ghGsWDVtS0c0+aJa4lOMGvJCAOvDfqvODv7gKlCXUpgumGpLdTmaZ8
+1RwqspAe3IqBcdKTqRD4m2mSg23nVx2FAY3cjFvZQtfooT7q1ItRV5RgH6FhQSl7
++6YIMJ1Bf8AAlLdRLpg+doOUGcEn+pkDiHFgI8ylH1LKyFKw+eXaAml/7DaWZk1d
+dqggwhXOhc/UUZFQuQQ8A8zpA13PcbC05XxN2hyP93tCEtyynMLVPtrRwDnHxFKa
+qKzs3rMDXPSXRn3ZZTdKH3069ApkEjQdpcwUh+EmJ1Ve/5cdtzT6kKWCjKBFZP/s
+91MlRrX2BTRdHaU5QJkUheUtakwxuHrdah2F94lRmsnQlpPr2YseJu6sIE+Dnx4M
+CfhdVbQL2w54R645nlnohu8CAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616adfeb.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU
+pNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X
+0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6
+ADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ
+raQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy
+KLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj
+/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H
+Kuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs
+mFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1
+/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq
+XIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4
+VORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616ae350.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0
+XkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ
+piM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46
+xLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP
+ODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM
+EyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr
+DzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+
+f53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF
+HwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk
+Ok9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6
+9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT
+JZDr++FjKrnnijbyNF8b98UCAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616db30d.rsa.pub

@@ -0,0 +1,14 @@
+-----BEGIN PUBLIC KEY-----
+MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0
+lIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm
+a6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw
+tO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C
+US/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP
+hP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv
+LSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0
+x9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF
+wmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG
+LPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV
+GZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C
+8CM1S15HxV78s9dFntEqIokCAwEAAQ==
+-----END PUBLIC KEY-----

pmb/data/locales

@@ -0,0 +1,304 @@
+C
+a_DJ
+aa_ER
+aa_ET
+af_ZA
+agr_PE
+ak_GH
+am_ET
+an_ES
+anp_IN
+ar_AE
+ar_BH
+ar_DZ
+ar_EG
+ar_IN
+ar_IQ
+ar_JO
+ar_KW
+ar_LB
+ar_LY
+ar_MA
+ar_OM
+ar_QA
+ar_SA
+ar_SD
+ar_SS
+ar_SY
+ar_TN
+ar_YE
+as_IN
+ast_ES
+ayc_PE
+az_AZ
+az_IR
+be_BY
+bem_ZM
+ber_DZ
+ber_MA
+bg_BG
+bhb_IN
+bho_IN
+bho_NP
+bi_VU
+bn_BD
+bn_IN
+bo_CN
+bo_IN
+br_FR
+brx_IN
+bs_BA
+byn_ER
+ca_AD
+ca_ES
+ca_FR
+ca_IT
+ce_RU
+ch_DE
+chr_US
+cmn_TW
+crh_UA
+cs_CZ
+csb_PL
+cv_RU
+cy_GB
+da_DK
+de_AT
+de_BE
+de_CH
+de_DE
+de_IT
+de_LI
+de_LU
+doi_IN
+dsb_DE
+dv_MV
+dz_BT
+el_CY
+el_GR
+en_AG
+en_AU
+en_BW
+en_CA
+en_DK
+en_GB
+en_HK
+en_IE
+en_IL
+en_IN
+en_NG
+en_NZ
+en_PH
+en_SC
+en_SG
+en_US
+en_ZA
+en_ZM
+en_ZW
+eo
+es_AR
+es_BO
+es_CL
+es_CO
+es_CR
+es_CU
+es_DO
+es_EC
+es_ES
+es_GT
+es_HN
+es_MX
+es_NI
+es_PA
+es_PE
+es_PR
+es_PY
+es_SV
+es_US
+es_UY
+es_VE
+et_EE
+eu_ES
+fa_IR
+ff_SN
+fi_FI
+fil_PH
+fo_FO
+fr_BE
+fr_CA
+fr_CH
+fr_FR
+fr_LU
+fur_IT
+fy_DE
+fy_NL
+ga_IE
+gd_GB
+gez_ER
+gez_ET
+gl_ES
+gu_IN
+gv_GB
+ha_NG
+hak_TW
+he_IL
+hi_IN
+hif_FJ
+hne_IN
+hr_HR
+hsb_DE
+ht_HT
+hu_HU
+hy_AM
+ia_FR
+id_ID
+ig_NG
+ik_CA
+is_IS
+it_CH
+it_IT
+iu_CA
+ja_JP
+ka_GE
+kab_DZ
+kk_KZ
+kl_GL
+km_KH
+kn_IN
+ko_KR
+kok_IN
+ks_IN
+ku_TR
+kw_GB
+ky_KG
+lb_LU
+lg_UG
+li_BE
+li_NL
+lij_IT
+ln_CD
+lo_LA
+lt_LT
+lv_LV
+lzh_TW
+mag_IN
+mai_IN
+mai_NP
+mfe_MU
+mg_MG
+mhr_RU
+mi_NZ
+miq_NI
+mjw_IN
+mk_MK
+ml_IN
+mn_MN
+mni_IN
+mnw_MM
+mr_IN
+ms_MY
+mt_MT
+my_MM
+nan_TW
+nb_NO
+nds_DE
+nds_NL
+ne_NP
+nhn_MX
+niu_NU
+niu_NZ
+nl_AW
+nl_BE
+nl_NL
+nn_NO
+nr_ZA
+nso_ZA
+oc_FR
+om_ET
+om_KE
+or_IN
+os_RU
+pa_IN
+pa_PK
+pap_AW
+pap_CW
+pl_PL
+ps_AF
+pt_BR
+pt_PT
+quz_PE
+raj_IN
+ro_RO
+ru_RU
+ru_UA
+rw_RW
+sa_IN
+sah_RU
+sat_IN
+sc_IT
+sd_IN
+se_NO
+sgs_LT
+shn_MM
+shs_CA
+si_LK
+sid_ET
+sk_SK
+sl_SI
+sm_WS
+so_DJ
+so_ET
+so_KE
+so_SO
+sq_AL
+sq_MK
+sr_ME
+sr_RS
+ss_ZA
+st_ZA
+sv_FI
+sv_SE
+sw_KE
+sw_TZ
+szl_PL
+ta_IN
+ta_LK
+tcy_IN
+te_IN
+tg_TJ
+th_TH
+the_NP
+ti_ER
+ti_ET
+tig_ER
+tk_TM
+tl_PH
+tn_ZA
+to_TO
+tpi_PG
+tr_CY
+tr_TR
+ts_ZA
+tt_RU
+ug_CN
+uk_UA
+unm_US
+ur_IN
+ur_PK
+uz_UZ
+ve_ZA
+vi_VN
+wa_BE
+wae_CH
+wal_ET
+wo_SN
+xh_ZA
+yi_US
+yo_NG
+yue_HK
+yuw_PG
+zh_CN
+zh_HK
+zh_SG
+zh_TW
+zu_ZA

pmb/export/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.export.frontend import frontend
 from pmb.export.odin import odin

pmb/export/odin.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -19,6 +19,12 @@ def odin(args, flavor, folder):
     pmb.flasher.init(args)
     suffix = "rootfs_" + args.device
 
+    # Backwards compatibility with old mkinitfs (pma#660)
+    suffix_flavor = f"-{flavor}"
+    pmaports_cfg = pmb.config.pmaports.read_config(args)
+    if pmaports_cfg.get("supported_mkinitfs_without_flavors", False):
+        suffix_flavor = ""
+
     # Validate method
     method = args.deviceinfo["flash_method"]
     if not method.startswith("heimdall-"):
@@ -54,16 +60,16 @@ def odin(args, flavor, folder):
         if method == "heimdall-isorec":
             handle.write(
                 # Kernel: copy and append md5
-                f"cp /boot/vmlinuz-{flavor} {odin_kernel_md5}\n"
+                f"cp /boot/vmlinuz{suffix_flavor} {odin_kernel_md5}\n"
                 f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n"
                 # Initramfs: recompress with lzop, append md5
-                f"gunzip -c /boot/initramfs-{flavor}"
+                f"gunzip -c /boot/initramfs{suffix_flavor}"
                 f" | lzop > {odin_initfs_md5}\n"
                 f"md5sum -t {odin_initfs_md5} >> {odin_initfs_md5}\n")
         elif method == "heimdall-bootimg":
             handle.write(
                 # boot.img: copy and append md5
-                f"cp /boot/boot.img-{flavor} {odin_kernel_md5}\n"
+                f"cp /boot/boot.img{suffix_flavor} {odin_kernel_md5}\n"
                 f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n")
         handle.write(
             # Create tar, remove included files and append md5

pmb/export/symlinks.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -37,6 +37,7 @@ def symlinks(args, flavor, folder):
         f"{args.device}-boot.img": "Boot partition image",
         f"{args.device}-root.img": "Root partition image",
         f"pmos-{args.device}.zip": "Android recovery flashable zip",
+        "lk2nd.img": "Secondary Android bootloader",
     }
 
     # Generate a list of patterns
@@ -53,7 +54,8 @@ def symlinks(args, flavor, folder):
                 f"{path_native}/home/pmos/rootfs/{args.device}-boot.img",
                 f"{path_native}/home/pmos/rootfs/{args.device}-root.img",
                 f"{path_buildroot}/var/lib/postmarketos-android-recovery-" +
-                f"installer/pmos-{args.device}.zip"]
+                f"installer/pmos-{args.device}.zip",
+                f"{path_boot}/lk2nd.img"]
 
     # Generate a list of files from the patterns
     files = []

pmb/flasher/__init__.py

@@ -1,6 +1,7 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.flasher.init import init
+from pmb.flasher.init import install_depends
 from pmb.flasher.run import run
 from pmb.flasher.run import check_partition_blacklist
 from pmb.flasher.variables import variables

pmb/flasher/frontend.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -20,7 +20,7 @@ def kernel(args):
         pmb.chroot.initfs.build(args, flavor, "rootfs_" + args.device)
 
     # Check kernel config
-    pmb.parse.kconfig.check(args, flavor)
+    pmb.parse.kconfig.check(args, flavor, must_exist=False)
 
     # Generate the paths and run the flasher
     if args.action_flasher == "boot":
@@ -88,11 +88,8 @@ def list_devices(args):
 
 
 def sideload(args):
-    method = args.flash_method or args.deviceinfo["flash_method"]
-    cfg = pmb.config.flashers[method]
-
     # Install depends
-    pmb.chroot.apk.install(args, cfg["depends"])
+    pmb.flasher.install_depends(args)
 
     # Mount the buildroot
     suffix = "buildroot_" + args.deviceinfo["arch"]
@@ -112,29 +109,63 @@ def sideload(args):
     pmb.flasher.run(args, "sideload")
 
 
+def flash_lk2nd(args):
+    method = args.flash_method or args.deviceinfo["flash_method"]
+    if method == "fastboot":
+        # In the future this could be expanded to use "fastboot flash lk2nd $img"
+        # which reflashes/updates lk2nd from itself. For now let the user handle this
+        # manually since supporting the codepath with heimdall requires more effort.
+        pmb.flasher.init(args)
+        logging.info("(native) checking current fastboot product")
+        output = pmb.chroot.root(args, ["fastboot", "getvar", "product"],
+                                 output="interactive", output_return=True)
+        # Variable "product" is e.g. "LK2ND_MSM8974" or "lk2nd-msm8226" depending
+        # on the lk2nd version.
+        if "lk2nd" in output.lower():
+            raise RuntimeError("You are currently running lk2nd. Please reboot into the regular"
+                               " bootloader mode to re-flash lk2nd.")
+
+    # Get the lk2nd package (which is a dependency of the device package)
+    device_pkg = f"device-{args.device}"
+    apkbuild = pmb.helpers.pmaports.get(args, device_pkg)
+    lk2nd_pkg = None
+    for dep in apkbuild["depends"]:
+        if dep.startswith("lk2nd"):
+            lk2nd_pkg = dep
+            break
+
+    if not lk2nd_pkg:
+        raise RuntimeError(f"{device_pkg} does not depend on any lk2nd package")
+
+    suffix = "rootfs_" + args.device
+    pmb.chroot.apk.install(args, [lk2nd_pkg], suffix)
+
+    logging.info("(native) flash lk2nd image")
+    pmb.flasher.run(args, "flash_lk2nd")
+
+
 def frontend(args):
     action = args.action_flasher
     method = args.flash_method or args.deviceinfo["flash_method"]
 
-    # Legacy alias
-    if action == "flash_system":
-        action = "flash_rootfs"
-
-    if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs"]:
+    if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs",
+                                       "flash_lk2nd"]:
         logging.info("This device doesn't support any flash method.")
         return
 
     if action in ["boot", "flash_kernel"]:
         kernel(args)
-    if action == "flash_rootfs":
+    elif action == "flash_rootfs":
         rootfs(args)
-    if action == "flash_vbmeta":
+    elif action == "flash_vbmeta":
         flash_vbmeta(args)
-    if action == "flash_dtbo":
+    elif action == "flash_dtbo":
         flash_dtbo(args)
-    if action == "list_flavors":
+    elif action == "flash_lk2nd":
+        flash_lk2nd(args)
+    elif action == "list_flavors":
         list_flavors(args)
-    if action == "list_devices":
+    elif action == "list_devices":
         list_devices(args)
-    if action == "sideload":
+    elif action == "sideload":
         sideload(args)

pmb/flasher/init.py

@@ -1,12 +1,12 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import pmb.config
 import pmb.chroot.apk
+import pmb.config
+import pmb.config.pmaports
 import pmb.helpers.mount
 
 
-def init(args):
-    # Validate method
+def install_depends(args):
     if hasattr(args, 'flash_method'):
         method = args.flash_method or args.deviceinfo["flash_method"]
     else:
@@ -20,10 +20,28 @@ def init(args):
                            "Make sure, it is packaged for Alpine Linux, or"
                            " package it yourself, and then add it to"
                            " pmb/config/__init__.py.")
-    cfg = pmb.config.flashers[method]
+    depends = pmb.config.flashers[method]["depends"]
 
-    # Install depends
-    pmb.chroot.apk.install(args, cfg["depends"])
+    # Depends for some flash methods may be different for various pmaports
+    # branches, so read them from pmaports.cfg.
+    if method == "fastboot":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_fastboot_depends",
+                                   "android-tools,avbtool").split(",")
+    elif method == "heimdall-bootimg":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_heimdall_depends",
+                                   "heimdall,avbtool").split(",")
+    elif method == "mtkclient":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_mtkclient_depends",
+                                   "mtkclient,android-tools").split(",")
+
+    pmb.chroot.apk.install(args, depends)
+
+
+def init(args):
+    install_depends(args)
 
     # Mount folders from host system
     for folder in pmb.config.flash_mount_bind:

pmb/flasher/run.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.flasher
 import pmb.chroot.initfs

pmb/flasher/variables.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config.pmaports
 
@@ -10,20 +10,41 @@ def variables(args, flavor, method):
 
     flash_pagesize = args.deviceinfo['flash_pagesize']
 
+    # TODO Remove _partition_system deviceinfo support once pmaports has been
+    # updated and minimum pmbootstrap version bumped.
+    # See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243
+
     if method.startswith("fastboot"):
         _partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\
             or "boot"
-        _partition_system = args.deviceinfo["flash_fastboot_partition_system"]\
-            or "system"
+        _partition_rootfs = args.deviceinfo["flash_fastboot_partition_rootfs"]\
+            or args.deviceinfo["flash_fastboot_partition_system"] or "userdata"
         _partition_vbmeta = args.deviceinfo["flash_fastboot_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_fastboot_partition_dtbo"]\
             or None
+    # Require that the partitions are specified in deviceinfo for now
+    elif method.startswith("rkdeveloptool"):
+        _partition_kernel = args.deviceinfo["flash_rk_partition_kernel"]\
+            or None
+        _partition_rootfs = args.deviceinfo["flash_rk_partition_rootfs"]\
+            or args.deviceinfo["flash_rk_partition_system"] or None
+        _partition_vbmeta = None
+        _partition_dtbo = None
+    elif method.startswith("mtkclient"):
+        _partition_kernel = args.deviceinfo["flash_mtkclient_partition_kernel"]\
+            or "boot"
+        _partition_rootfs = args.deviceinfo["flash_mtkclient_partition_rootfs"]\
+            or "userdata"
+        _partition_vbmeta = args.deviceinfo["flash_mtkclient_partition_vbmeta"]\
+            or None
+        _partition_dtbo = args.deviceinfo["flash_mtkclient_partition_dtbo"]\
+            or None
     else:
         _partition_kernel = args.deviceinfo["flash_heimdall_partition_kernel"]\
             or "KERNEL"
-        _partition_system = args.deviceinfo["flash_heimdall_partition_system"]\
-            or "SYSTEM"
+        _partition_rootfs = args.deviceinfo["flash_heimdall_partition_rootfs"]\
+            or args.deviceinfo["flash_heimdall_partition_system"] or "SYSTEM"
         _partition_vbmeta = args.deviceinfo["flash_heimdall_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_heimdall_partition_dtbo"]\
@@ -33,7 +54,7 @@ def variables(args, flavor, method):
         # Only one operation is done at same time so it doesn't matter
         # sharing the arg
         _partition_kernel = args.partition
-        _partition_system = args.partition
+        _partition_rootfs = args.partition
         _partition_vbmeta = args.partition
         _partition_dtbo = args.partition
 
@@ -51,7 +72,7 @@ def variables(args, flavor, method):
         "$PARTITION_KERNEL": _partition_kernel,
         "$PARTITION_INITFS": args.deviceinfo[
             "flash_heimdall_partition_initfs"] or "RECOVERY",
-        "$PARTITION_SYSTEM": _partition_system,
+        "$PARTITION_ROOTFS": _partition_rootfs,
         "$PARTITION_VBMETA": _partition_vbmeta,
         "$PARTITION_DTBO": _partition_dtbo,
         "$FLASH_PAGESIZE": flash_pagesize,

pmb/helpers/__init__.py

@@ -1,2 +1,2 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later

pmb/helpers/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Johannes Marbach, Oliver Smith
+# Copyright 2023 Johannes Marbach, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 
@@ -6,6 +6,7 @@ import pmb.chroot.root
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.run
+import pmb.helpers.run_core
 import pmb.parse.version
 
 
@@ -62,7 +63,7 @@ def _create_command_with_progress(command, fifo):
     """
     flags = ["--no-progress", "--progress-fd", "3"]
     command_full = [command[0]] + flags + command[1:]
-    command_flat = pmb.helpers.run.flat_cmd(command_full)
+    command_flat = pmb.helpers.run_core.flat_cmd(command_full)
     command_flat = f"exec 3>{fifo}; {command_flat}"
     return ["sh", "-c", command_flat]
 

pmb/helpers/aportupgrade.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Luca Weiss
+# Copyright 2023 Luca Weiss
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import fnmatch
@@ -19,9 +19,10 @@ ANITYA_API_BASE = "https://release-monitoring.org/api/v2"
 GITHUB_API_BASE = "https://api.github.com"
 GITLAB_HOSTS = [
     "https://gitlab.com",
+    "https://gitlab.freedesktop.org",
+    "https://gitlab.gnome.org",
     "https://invent.kde.org",
     "https://source.puri.sm",
-    "https://gitlab.freedesktop.org",
 ]
 
 
@@ -33,7 +34,7 @@ def init_req_headers() -> None:
         return
     # Generic request headers
     req_headers = {
-        'User-Agent': f'pmbootstrap/{pmb.config.version} aportupgrade'}
+        'User-Agent': f'pmbootstrap/{pmb.__version__} aportupgrade'}
 
     # Request headers specific to GitHub
     req_headers_github = dict(req_headers)
@@ -134,7 +135,11 @@ def upgrade_git_package(args, pkgname: str, package) -> bool:
     sha_new = verinfo["sha"]
 
     # Format the new pkgver, keep the value before _git the same
-    pkgver = package["pkgver"]
+    if package["pkgver"] == "9999":
+        pkgver = package["_pkgver"]
+    else:
+        pkgver = package["pkgver"]
+
     pkgver_match = re.match(r"([\d.]+)_git", pkgver)
     date_pkgver = verinfo["date"].strftime("%Y%m%d")
     pkgver_new = f"{pkgver_match.group(1)}_git{date_pkgver}"
@@ -154,7 +159,10 @@ def upgrade_git_package(args, pkgname: str, package) -> bool:
         logging.info(f"  Would change pkgrel from {pkgrel} to {pkgrel_new}")
         return True
 
-    pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
+    if package["pkgver"] == "9999":
+        pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
+    else:
+        pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
     pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
     pmb.helpers.file.replace_apkbuild(args, pkgname, "_commit", sha_new, True)
     return True
@@ -169,38 +177,56 @@ def upgrade_stable_package(args, pkgname: str, package) -> bool:
     :param package: a dict containing package information
     :returns: if something (would have) been changed
     """
-    projects = pmb.helpers.http.retrieve_json(
-        f"{ANITYA_API_BASE}/projects/?name={pkgname}", headers=req_headers)
-    if projects["total_items"] < 1:
-        # There is no Anitya project with the package name.
-        # Looking up if there's a custom mapping from postmarketOS package name
-        # to Anitya project name.
-        mappings = pmb.helpers.http.retrieve_json(
-            f"{ANITYA_API_BASE}/packages/?distribution=postmarketOS"
-            f"&name={pkgname}", headers=req_headers)
-        if mappings["total_items"] < 1:
-            logging.warning("{}: failed to get Anitya project".format(pkgname))
-            return False
-        project_name = mappings["items"][0]["project"]
+
+    # Looking up if there's a custom mapping from postmarketOS package name
+    # to Anitya project name.
+    mappings = pmb.helpers.http.retrieve_json(
+        f"{ANITYA_API_BASE}/packages/?distribution=postmarketOS"
+        f"&name={pkgname}", headers=req_headers)
+    if mappings["total_items"] < 1:
         projects = pmb.helpers.http.retrieve_json(
-            f"{ANITYA_API_BASE}/projects/?name={project_name}",
+            f"{ANITYA_API_BASE}/projects/?name={pkgname}", headers=req_headers)
+        if projects["total_items"] < 1:
+            logging.warning(f"{pkgname}: failed to get Anitya project")
+            return False
+    else:
+        project_name = mappings["items"][0]["project"]
+        ecosystem = mappings["items"][0]["ecosystem"]
+        projects = pmb.helpers.http.retrieve_json(
+            f"{ANITYA_API_BASE}/projects/?name={project_name}&"
+            f"ecosystem={ecosystem}",
             headers=req_headers)
 
+    if projects["total_items"] < 1:
+        logging.warning(f"{pkgname}: didn't find any projects, can't upgrade!")
+        return False
+    if projects["total_items"] > 1:
+        logging.warning(f"{pkgname}: found more than one project, can't "
+                        f"upgrade! Please create an explicit mapping of "
+                        f"\"project\" to the package name.")
+        return False
+
     # Get the first, best-matching item
     project = projects["items"][0]
 
     # Check that we got a version number
-    if project["version"] is None:
+    if len(project["stable_versions"]) < 1:
         logging.warning("{}: got no version number, ignoring".format(pkgname))
         return False
 
+    version = project["stable_versions"][0]
+
     # Compare the pmaports version with the project version
-    if package["pkgver"] == project["version"]:
+    if package["pkgver"] == version:
         logging.info("{}: up-to-date".format(pkgname))
         return False
 
-    pkgver = package["pkgver"]
-    pkgver_new = project["version"]
+    if package["pkgver"] == "9999":
+        pkgver = package["_pkgver"]
+    else:
+        pkgver = package["pkgver"]
+
+    pkgver_new = version
 
     pkgrel = package["pkgrel"]
     pkgrel_new = 0
@@ -216,7 +242,11 @@ def upgrade_stable_package(args, pkgname: str, package) -> bool:
         logging.info(f"  Would change pkgrel from {pkgrel} to {pkgrel_new}")
         return True
 
-    pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
+    if package["pkgver"] == "9999":
+        pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
+    else:
+        pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
+
     pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
     return True
 

pmb/helpers/args.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import copy
 import os
@@ -31,30 +31,7 @@ import pmb.helpers.git
        args.device ("samsung-i9100", "qemu-amd64" etc.)
        args.work ("/home/user/.local/var/pmbootstrap", override with --work)
 
-    3. Shortcuts
-       Long variables or function calls that always return the same information
-       may have a shortcut defined, to make the code more readable (see
-       add_shortcuts() below).
-
-       Example:
-       args.arch_native ("x86_64" etc.)
-
-    4. Cache
-       pmbootstrap uses this dictionary to save the result of expensive
-       results, so they work a lot faster the next time they are needed in the
-       same session. Usually the cache is written to and read from in the same
-       Python file, with code similar to the following:
-
-       def lookup(args, key):
-           if key in args.cache["mycache"]:
-               return args.cache["mycache"][key]
-           ret = expensive_operation(args, key)
-           args.cache["mycache"][key] = ret
-           return ret
-
-       See add_cache() below for details.
-
-    5. Parsed configs
+    3. Parsed configs
        Similar to the cache above, specific config files get parsed and added
        to args, so they can get accessed quickly (without parsing the configs
        over and over). These configs are not only used in one specific
@@ -116,33 +93,11 @@ def replace_placeholders(args):
             setattr(args, key, os.path.expanduser(getattr(args, key)))
 
 
-def add_shortcuts(args):
-    """ Add convenience shortcuts """
-    setattr(args, "arch_native", pmb.parse.arch.alpine_native())
-
-
-def add_cache(args):
-    """ Add a caching dict (caches parsing of files etc. for the current
-        session) """
-    repo_update = {"404": [], "offline_msg_shown": False}
-    setattr(args, "cache", {"apkindex": {},
-                            "apkbuild": {},
-                            "apk_min_version_checked": [],
-                            "apk_repository_list_updated": [],
-                            "built": {},
-                            "find_aport": {},
-                            "pmb.helpers.package.depends_recurse": {},
-                            "pmb.helpers.package.get": {},
-                            "pmb.helpers.repo.update": repo_update,
-                            "pmb.helpers.git.parse_channels_cfg": {},
-                            "pmb.config.pmaports.read_config": None})
-
-
 def add_deviceinfo(args):
     """ Add and verify the deviceinfo (only after initialization) """
     setattr(args, "deviceinfo", pmb.parse.deviceinfo(args))
     arch = args.deviceinfo["arch"]
-    if (arch != args.arch_native and
+    if (arch != pmb.config.arch_native and
             arch not in pmb.config.build_device_architectures):
         raise ValueError("Arch '" + arch + "' is not available in"
                          " postmarketOS. If you would like to add it, see:"
@@ -154,8 +109,7 @@ def init(args):
     fix_mirrors_postmarketos(args)
     pmb.config.merge_with_args(args)
     replace_placeholders(args)
-    add_shortcuts(args)
-    add_cache(args)
+    pmb.helpers.other.init_cache()
 
     # Initialize logs (we could raise errors below)
     pmb.helpers.logging.init(args)
@@ -177,9 +131,7 @@ def update_work(args, work):
     args_new = copy.deepcopy(args.from_argparse)
 
     # Keep from the modified args:
-    # * the old log file descriptor (so we can close it)
     # * the unmodified args from argparse (to check if --aports was specified)
-    args_new.logfd = args.logfd
     args_new.from_argparse = args.from_argparse
 
     # Generate modified args again, replacing $WORK with the new work folder

pmb/helpers/cli.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import logging
@@ -38,7 +38,7 @@ class ReadlineTabCompleter:
         return None
 
 
-def ask(args, question="Continue?", choices=["y", "n"], default="n",
+def ask(question="Continue?", choices=["y", "n"], default="n",
         lowercase_answer=True, validation_regex=None, complete=None):
     """
     Ask a question on the terminal.
@@ -83,8 +83,8 @@ def ask(args, question="Continue?", choices=["y", "n"], default="n",
         if ret == "":
             ret = str(default)
 
-        args.logfd.write(f"{line}: {ret}\n")
-        args.logfd.flush()
+        pmb.helpers.logging.logfd.write(f"{line}: {ret}\n")
+        pmb.helpers.logging.logfd.flush()
 
         # Validate with regex
         if not validation_regex:
@@ -110,7 +110,7 @@ def confirm(args, question="Continue?", default=False, no_assumptions=False):
     if args.assume_yes and not no_assumptions:
         logging.info(question + " (y/n) [" + default_str + "]: y")
         return True
-    answer = ask(args, question, ["y", "n"], default_str, True, "(y|n)")
+    answer = ask(question, ["y", "n"], default_str, True, "(y|n)")
     return answer == "y"
 
 

pmb/helpers/devices.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob
@@ -59,7 +59,7 @@ def list_apkbuilds(args):
     ret = {}
     for device in list_codenames(args):
         apkbuild_path = f"{args.aports}/device/*/device-{device}/APKBUILD"
-        ret[device] = pmb.parse.apkbuild(args, apkbuild_path)
+        ret[device] = pmb.parse.apkbuild(apkbuild_path)
     return ret
 
 

pmb/helpers/file.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os
@@ -26,7 +26,7 @@ def replace_apkbuild(args, pkgname, key, new, in_quotes=False):
         :param in_quotes: expect the value to be in quotation marks ("") """
     # Read old value
     path = pmb.helpers.pmaports.find(args, pkgname) + "/APKBUILD"
-    apkbuild = pmb.parse.apkbuild(args, path)
+    apkbuild = pmb.parse.apkbuild(path)
     old = apkbuild[key]
 
     # Prepare old/new strings
@@ -41,8 +41,8 @@ def replace_apkbuild(args, pkgname, key, new, in_quotes=False):
     replace(path, "\n" + line_old + "\n", "\n" + line_new + "\n")
 
     # Verify
-    del (args.cache["apkbuild"][path])
-    apkbuild = pmb.parse.apkbuild(args, path)
+    del (pmb.helpers.other.cache["apkbuild"][path])
+    apkbuild = pmb.parse.apkbuild(path)
     if apkbuild[key] != str(new):
         raise RuntimeError("Failed to set '{}' for pmaport '{}'. Make sure"
                            " that there's a line with exactly the string '{}'"

pmb/helpers/frontend.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import json
@@ -9,13 +9,14 @@ import sys
 import pmb.aportgen
 import pmb.build
 import pmb.build.autodetect
-import pmb.sideload
 import pmb.chroot
 import pmb.chroot.initfs
 import pmb.chroot.other
+import pmb.ci
 import pmb.config
 import pmb.export
 import pmb.flasher
+import pmb.helpers.aportupgrade
 import pmb.helpers.devices
 import pmb.helpers.git
 import pmb.helpers.lint
@@ -25,12 +26,13 @@ import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.helpers.repo_missing
 import pmb.helpers.run
-import pmb.helpers.aportupgrade
 import pmb.helpers.status
 import pmb.install
 import pmb.install.blockdevice
+import pmb.netboot
 import pmb.parse
 import pmb.qemu
+import pmb.sideload
 
 
 def _parse_flavor(args, autoinstall=True):
@@ -140,6 +142,11 @@ def sideload(args):
                           args.packages)
 
 
+def netboot(args):
+    if args.action_netboot == "serve":
+        pmb.netboot.start_nbd_server(args)
+
+
 def chroot(args):
     # Suffix
     suffix = _parse_suffix(args)
@@ -254,6 +261,10 @@ def install(args):
         if args.filesystem:
             raise ValueError("--on-device-installer cannot be combined with"
                              " --filesystem")
+
+        if args.deviceinfo["cgpt_kpart"]:
+            raise ValueError("--on-device-installer cannot be used with"
+                             " ChromeOS devices")
     else:
         if args.ondev_cp:
             raise ValueError("--cp can only be combined with --ondev")
@@ -366,14 +377,17 @@ def newapkbuild(args):
 
 def kconfig(args):
     if args.action_kconfig == "check":
+        details = args.kconfig_check_details
+        # Build the components list from cli arguments (--waydroid etc.)
+        components_list = []
+        for name in pmb.parse.kconfig.get_all_component_names():
+            if getattr(args, f"kconfig_check_{name}"):
+                components_list += [name]
+
         # Handle passing a file directly
         if args.file:
-            if pmb.parse.kconfig.check_file(args, args.package,
-                                            anbox=args.anbox,
-                                            nftables=args.nftables,
-                                            containers=args.containers,
-                                            zram=args.zram,
-                                            details=True):
+            if pmb.parse.kconfig.check_file(args.package, components_list,
+                                            details=details):
                 logging.info("kconfig check succeeded!")
                 return
             raise RuntimeError("kconfig check failed!")
@@ -396,17 +410,12 @@ def kconfig(args):
                 pkgname = package if package.startswith("linux-") \
                     else "linux-" + package
                 aport = pmb.helpers.pmaports.find(args, pkgname)
-                apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
+                apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
                 if "!pmb:kconfigcheck" in apkbuild["options"]:
                     skipped += 1
                     continue
-            if not pmb.parse.kconfig.check(
-                    args, package,
-                    force_anbox_check=args.anbox,
-                    force_nftables_check=args.nftables,
-                    force_containers_check=args.containers,
-                    force_zram_check=args.zram,
-                    details=True):
+            if not pmb.parse.kconfig.check(args, package, components_list,
+                                           details=details):
                 error = True
 
         # At least one failure
@@ -417,12 +426,13 @@ def kconfig(args):
                 logging.info("NOTE: " + str(skipped) + " kernel(s) was skipped"
                              " (consider 'pmbootstrap kconfig check -f')")
             logging.info("kconfig check succeeded!")
-    elif args.action_kconfig == "edit":
+    elif args.action_kconfig in ["edit", "migrate"]:
         if args.package:
             pkgname = args.package
         else:
             pkgname = args.deviceinfo["codename"]
-        pmb.build.menuconfig(args, pkgname)
+        use_oldconfig = args.action_kconfig == "migrate"
+        pmb.build.menuconfig(args, pkgname, use_oldconfig)
 
 
 def deviceinfo_parse(args):
@@ -450,12 +460,12 @@ def apkbuild_parse(args):
         print(package + ":")
         aport = pmb.helpers.pmaports.find(args, package)
         path = aport + "/APKBUILD"
-        print(json.dumps(pmb.parse.apkbuild(args, path), indent=4,
+        print(json.dumps(pmb.parse.apkbuild(path), indent=4,
                          sort_keys=True))
 
 
 def apkindex_parse(args):
-    result = pmb.parse.apkindex.parse(args, args.apkindex_path)
+    result = pmb.parse.apkindex.parse(args.apkindex_path)
     if args.package:
         if args.package not in result:
             raise RuntimeError("Package not found in the APKINDEX: " +
@@ -506,7 +516,7 @@ def shutdown(args):
 def stats(args):
     # Chroot suffix
     suffix = "native"
-    if args.arch != args.arch_native:
+    if args.arch != pmb.config.arch_native:
         suffix = "buildroot_" + args.arch
 
     # Install ccache and display stats
@@ -521,18 +531,25 @@ def work_migrate(args):
 
 
 def log(args):
+    log_testsuite = f"{args.work}/log_testsuite.txt"
+
     if args.clear_log:
         pmb.helpers.run.user(args, ["truncate", "-s", "0", args.log])
-    pmb.helpers.run.user(args, ["tail", "-n", args.lines, "-F", args.log],
-                         output="tui")
+        pmb.helpers.run.user(args, ["truncate", "-s", "0", log_testsuite])
 
+    cmd = ["tail", "-n", args.lines, "-F"]
 
-def log_distccd(args):
-    logpath = "/home/pmos/distccd.log"
-    if args.clear_log:
-        pmb.chroot.user(args, ["truncate", "-s", "0", logpath])
-    pmb.chroot.user(args, ["tail", "-n", args.lines, "-f", logpath],
-                    output="tui")
+    # Follow the testsuite's log file too if it exists. It will be created when
+    # starting a test case that writes to it (git -C test grep log_testsuite).
+    if os.path.exists(log_testsuite):
+        cmd += [log_testsuite]
+
+    # tail writes the last lines of the files to the terminal. Put the regular
+    # log at the end, so that output is visible at the bottom (where the user
+    # looks for an error / what's currently going on).
+    cmd += [args.log]
+
+    pmb.helpers.run.user(args, cmd, output="tui")
 
 
 def zap(args):
@@ -540,7 +557,7 @@ def zap(args):
                    distfiles=args.distfiles, pkgs_local=args.pkgs_local,
                    pkgs_local_mismatch=args.pkgs_local_mismatch,
                    pkgs_online_mismatch=args.pkgs_online_mismatch,
-                   rust=args.rust)
+                   rust=args.rust, netboot=args.netboot)
 
     # Don't write the "Done" message
     pmb.helpers.logging.disable()
@@ -550,7 +567,7 @@ def bootimg_analyze(args):
     bootimg = pmb.parse.bootimg(args, args.path)
     tmp_output = "Put these variables in the deviceinfo file of your device:\n"
     for line in pmb.aportgen.device.\
-            generate_deviceinfo_fastboot_content(args, bootimg).split("\n"):
+            generate_deviceinfo_fastboot_content(bootimg).split("\n"):
         tmp_output += "\n" + line.lstrip()
     logging.info(tmp_output)
 
@@ -592,3 +609,47 @@ def lint(args):
 def status(args):
     if not pmb.helpers.status.print_status(args, args.details):
         sys.exit(1)
+
+
+def ci(args):
+    topdir = pmb.helpers.git.get_topdir(args, os.getcwd())
+    if not os.path.exists(topdir):
+        logging.error("ERROR: change your current directory to a git"
+                      " repository (e.g. pmbootstrap, pmaports) before running"
+                      " 'pmbootstrap ci'.")
+        exit(1)
+
+    scripts_available = pmb.ci.get_ci_scripts(topdir)
+    scripts_available = pmb.ci.sort_scripts_by_speed(scripts_available)
+    if not scripts_available:
+        logging.error("ERROR: no supported CI scripts found in current git"
+                      " repository, see https://postmarketos.org/pmb-ci")
+        exit(1)
+
+    scripts_selected = {}
+    if args.scripts:
+        if args.all:
+            raise RuntimeError("Combining --all with script names doesn't"
+                               " make sense")
+        for script in args.scripts:
+            if script not in scripts_available:
+                logging.error(f"ERROR: script '{script}' not found in git"
+                              " repository, found these:"
+                              f" {', '.join(scripts_available.keys())}")
+                exit(1)
+            scripts_selected[script] = scripts_available[script]
+    elif args.all:
+        scripts_selected = scripts_available
+
+    if args.fast:
+        for script, script_data in scripts_available.items():
+            if "slow" not in script_data["options"]:
+                scripts_selected[script] = script_data
+
+    if not pmb.helpers.git.clean_worktree(args, topdir):
+        logging.warning("WARNING: this git repository has uncommitted changes")
+
+    if not scripts_selected:
+        scripts_selected = pmb.ci.ask_which_scripts_to_run(scripts_available)
+
+    pmb.ci.run_scripts(args, topdir, scripts_selected)

pmb/helpers/git.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 import logging
@@ -109,8 +109,8 @@ def parse_channels_cfg(args):
                                            ...}} """
     # Cache during one pmbootstrap run
     cache_key = "pmb.helpers.git.parse_channels_cfg"
-    if args.cache[cache_key]:
-        return args.cache[cache_key]
+    if pmb.helpers.other.cache[cache_key]:
+        return pmb.helpers.other.cache[cache_key]
 
     # Read with configparser
     cfg = configparser.ConfigParser()
@@ -147,7 +147,7 @@ def parse_channels_cfg(args):
             value = cfg.get(channel, key)
             ret["channels"][channel_new][key] = value
 
-    args.cache[cache_key] = ret
+    pmb.helpers.other.cache[cache_key] = ret
     return ret
 
 
@@ -229,7 +229,7 @@ def pull(args, name_repo):
     return 0
 
 
-def is_outdated(args, path):
+def is_outdated(path):
     # FETCH_HEAD always exists in repositories cloned by pmbootstrap.
     # Usually it does not (before first git fetch/pull), but there is no good
     # fallback. For exampe, getting the _creation_ date of .git/HEAD is non-
@@ -246,3 +246,29 @@ def is_outdated(args, path):
 
     date_outdated = time.time() - pmb.config.git_repo_outdated
     return date_head <= date_outdated
+
+
+def get_topdir(args, path):
+    """ :returns: a string with the top dir of the git repository, or an
+                  empty string if it's not a git repository. """
+    return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"],
+                                path, output_return=True, check=False).rstrip()
+
+
+def get_files(args, path):
+    """ Get all files inside a git repository, that are either already in the
+        git tree or are not in gitignore. Do not list deleted files. To be used
+        for creating a tarball of the git repository.
+        :param path: top dir of the git repository
+        :returns: all files in a git repository as list, relative to path """
+    ret = []
+    files = pmb.helpers.run.user(args, ["git", "ls-files"], path,
+                                 output_return=True).split("\n")
+    files += pmb.helpers.run.user(args, ["git", "ls-files",
+                                  "--exclude-standard", "--other"], path,
+                                  output_return=True).split("\n")
+    for file in files:
+        if os.path.exists(f"{path}/{file}"):
+            ret += [file]
+
+    return ret

pmb/helpers/http.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import hashlib
 import json

pmb/helpers/lint.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Danct12 <danct12@disroot.org>
+# Copyright 2023 Danct12 <danct12@disroot.org>
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -17,7 +17,6 @@ def check(args, pkgnames):
     :param pkgnames: Names of the packages to lint
     """
     pmb.chroot.apk.install(args, ["atools"])
-    pmb.build.init(args)
 
     # Mount pmaports.git inside the chroot so that we don't have to copy the
     # package folders

pmb/helpers/logging.py

@@ -1,10 +1,12 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
 import sys
 import pmb.config
 
+logfd = None
+
 
 class log_handler(logging.StreamHandler):
     """
@@ -53,8 +55,8 @@ class log_handler(logging.StreamHandler):
 
             # Everything: Write to logfd
             msg = "(" + str(os.getpid()).zfill(6) + ") " + msg
-            self._args.logfd.write(msg + "\n")
-            self._args.logfd.flush()
+            logfd.write(msg + "\n")
+            logfd.flush()
 
         except (KeyboardInterrupt, SystemExit):
             raise
@@ -83,23 +85,24 @@ def add_verbose_log_level():
 
 def init(args):
     """
-    Set log format and add the log file descriptor to args.logfd, add the
+    Set log format and add the log file descriptor to logfd, add the
     verbose log level.
     """
+    global logfd
     # Set log file descriptor (logfd)
     if args.details_to_stdout:
-        setattr(args, "logfd", sys.stdout)
+        logfd = sys.stdout
     else:
         # Require containing directory to exist (so we don't create the work
         # folder and break the folder migration logic, which needs to set the
         # version upon creation)
         dir = os.path.dirname(args.log)
         if os.path.exists(dir):
-            setattr(args, "logfd", open(args.log, "a+"))
+            logfd = open(args.log, "a+")
         else:
-            setattr(args, "logfd", open(os.devnull, "a+"))
+            logfd = open(os.devnull, "a+")
             if args.action != "init":
-                print("WARNING: Can't create log file in '" + dir + "', path"
+                print(f"WARNING: Can't create log file in '{dir}', path"
                       " does not exist!")
 
     # Set log format

pmb/helpers/mount.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import pmb.helpers.run

pmb/helpers/other.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -29,7 +29,7 @@ def folder_size(args, path):
     return ret
 
 
-def check_grsec(args):
+def check_grsec():
     """
     Check if the current kernel is based on the grsec patchset, and if
     the chroot_deny_chmod option is enabled. Raise an exception in that
@@ -54,9 +54,12 @@ def check_binfmt_misc(args):
     if os.path.exists(path):
         return
 
+    # check=False: this might be built-in instead of being a module
     pmb.helpers.run.root(args, ["modprobe", "binfmt_misc"], check=False)
+
+    # check=False: we check it below and print a more helpful message on error
     pmb.helpers.run.root(args, ["mount", "-t", "binfmt_misc", "none",
-                                "/proc/sys/fs/binfmt_misc"])
+                                "/proc/sys/fs/binfmt_misc"], check=False)
 
     if not os.path.exists(path):
         link = "https://postmarketos.org/binfmt_misc"
@@ -264,14 +267,49 @@ def validate_hostname(hostname):
         return False
 
     # Check that it only contains valid chars
-    if not re.match("^[0-9a-z-]*$", hostname):
+    if not re.match(r"^[0-9a-z-\.]*$", hostname):
         logging.fatal("ERROR: Hostname must only contain letters (a-z),"
-                      " digits (0-9) or minus signs (-)")
+                      " digits (0-9), minus signs (-), or periods (.)")
         return False
 
-    # Check that doesn't begin or end with a minus sign
-    if hostname[:1] == "-" or hostname[-1:] == "-":
+    # Check that doesn't begin or end with a minus sign or period
+    if re.search(r"^-|^\.|-$|\.$", hostname):
         logging.fatal("ERROR: Hostname must not begin or end with a minus"
-                      " sign")
+                      " sign or period")
         return False
+
     return True
+
+
+"""
+pmbootstrap uses this dictionary to save the result of expensive
+results, so they work a lot faster the next time they are needed in the
+same session. Usually the cache is written to and read from in the same
+Python file, with code similar to the following:
+
+def lookup(key):
+    if key in pmb.helpers.other.cache["mycache"]:
+        return pmb.helpers.other.cache["mycache"][key]
+    ret = expensive_operation(args, key)
+    pmb.helpers.other.cache["mycache"][key] = ret
+    return ret
+"""
+cache = None
+
+
+def init_cache():
+    global cache
+    """ Add a caching dict (caches parsing of files etc. for the current
+        session) """
+    repo_update = {"404": [], "offline_msg_shown": False}
+    cache = {"apkindex": {},
+             "apkbuild": {},
+             "apk_min_version_checked": [],
+             "apk_repository_list_updated": [],
+             "built": {},
+             "find_aport": {},
+             "pmb.helpers.package.depends_recurse": {},
+             "pmb.helpers.package.get": {},
+             "pmb.helpers.repo.update": repo_update,
+             "pmb.helpers.git.parse_channels_cfg": {},
+             "pmb.config.pmaports.read_config": None}

pmb/helpers/package.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with both pmaports and binary package repos. See also:
@@ -33,10 +33,16 @@ def get(args, pkgname, arch, replace_subpkgnames=False, must_exist=True):
                   * None if the package was not found """
     # Cached result
     cache_key = "pmb.helpers.package.get"
-    if (arch in args.cache[cache_key] and
-            pkgname in args.cache[cache_key][arch] and
-            replace_subpkgnames in args.cache[cache_key][arch][pkgname]):
-        return args.cache[cache_key][arch][pkgname][replace_subpkgnames]
+    if (
+        arch in pmb.helpers.other.cache[cache_key] and
+        pkgname in pmb.helpers.other.cache[cache_key][arch] and
+        replace_subpkgnames in pmb.helpers.other.cache[cache_key][arch][
+            pkgname
+        ]
+    ):
+        return pmb.helpers.other.cache[cache_key][arch][pkgname][
+            replace_subpkgnames
+        ]
 
     # Find in pmaports
     ret = None
@@ -96,11 +102,13 @@ def get(args, pkgname, arch, replace_subpkgnames=False, must_exist=True):
 
     # Save to cache and return
     if ret:
-        if arch not in args.cache[cache_key]:
-            args.cache[cache_key][arch] = {}
-        if pkgname not in args.cache[cache_key][arch]:
-            args.cache[cache_key][arch][pkgname] = {}
-        args.cache[cache_key][arch][pkgname][replace_subpkgnames] = ret
+        if arch not in pmb.helpers.other.cache[cache_key]:
+            pmb.helpers.other.cache[cache_key][arch] = {}
+        if pkgname not in pmb.helpers.other.cache[cache_key][arch]:
+            pmb.helpers.other.cache[cache_key][arch][pkgname] = {}
+        pmb.helpers.other.cache[cache_key][arch][pkgname][
+            replace_subpkgnames
+        ] = ret
         return ret
 
     # Could not find the package
@@ -119,9 +127,9 @@ def depends_recurse(args, pkgname, arch):
                    "linux-samsung-i9100", ...] """
     # Cached result
     cache_key = "pmb.helpers.package.depends_recurse"
-    if (arch in args.cache[cache_key] and
-            pkgname in args.cache[cache_key][arch]):
-        return args.cache[cache_key][arch][pkgname]
+    if (arch in pmb.helpers.other.cache[cache_key] and
+            pkgname in pmb.helpers.other.cache[cache_key][arch]):
+        return pmb.helpers.other.cache[cache_key][arch][pkgname]
 
     # Build ret (by iterating over the queue)
     queue = [pkgname]
@@ -141,9 +149,9 @@ def depends_recurse(args, pkgname, arch):
     ret.sort()
 
     # Save to cache and return
-    if arch not in args.cache[cache_key]:
-        args.cache[cache_key][arch] = {}
-    args.cache[cache_key][arch][pkgname] = ret
+    if arch not in pmb.helpers.other.cache[cache_key]:
+        pmb.helpers.other.cache[cache_key][arch] = {}
+    pmb.helpers.other.cache[cache_key][arch][pkgname] = ret
     return ret
 
 

pmb/helpers/pkgrel_bump.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 
@@ -18,7 +18,7 @@ def package(args, pkgname, reason="", dry=False):
     """
     # Current and new pkgrel
     path = pmb.helpers.pmaports.find(args, pkgname) + "/APKBUILD"
-    apkbuild = pmb.parse.apkbuild(args, path)
+    apkbuild = pmb.parse.apkbuild(path)
     pkgrel = int(apkbuild["pkgrel"])
     pkgrel_new = pkgrel + 1
 
@@ -34,8 +34,8 @@ def package(args, pkgname, reason="", dry=False):
     pmb.helpers.file.replace(path, old, new)
 
     # Verify
-    del(args.cache["apkbuild"][path])
-    apkbuild = pmb.parse.apkbuild(args, path)
+    del pmb.helpers.other.cache["apkbuild"][path]
+    apkbuild = pmb.parse.apkbuild(path)
     if int(apkbuild["pkgrel"]) != pkgrel_new:
         raise RuntimeError("Failed to bump pkgrel for package '" + pkgname +
                            "'. Make sure that there's a line with exactly the"
@@ -81,6 +81,10 @@ def auto_apkindex_package(args, arch, aport, apk, dry=False):
                                                       ", ".join(depends)))
     missing = []
     for depend in depends:
+        if depend.startswith("!"):
+            # Ignore conflict-dependencies
+            continue
+
         providers = pmb.parse.apkindex.providers(args, depend, arch,
                                                  must_exist=False)
         if providers == {}:
@@ -107,7 +111,7 @@ def auto(args, dry=False):
         paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False)
         for path in paths:
             logging.info("scan " + path)
-            index = pmb.parse.apkindex.parse(args, path, False)
+            index = pmb.parse.apkindex.parse(path, False)
             for pkgname, apk in index.items():
                 origin = apk["origin"]
                 # Only increase once!
@@ -120,7 +124,7 @@ def auto(args, dry=False):
                     logging.warning("{}: origin '{}' aport not found".format(
                                     pkgname, origin))
                     continue
-                aport = pmb.parse.apkbuild(args, aport_path + "/APKBUILD")
+                aport = pmb.parse.apkbuild(f"{aport_path}/APKBUILD")
                 if auto_apkindex_package(args, arch, aport, apk, dry):
                     ret.append(pkgname)
     return ret

pmb/helpers/pmaports.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with pmaports. See also:
@@ -15,7 +15,7 @@ import pmb.parse
 def _find_apkbuilds(args):
     # Try to get a cached result first (we assume that the aports don't change
     # in one pmbootstrap call)
-    apkbuilds = args.cache.get("pmb.helpers.pmaports.apkbuilds")
+    apkbuilds = pmb.helpers.other.cache.get("pmb.helpers.pmaports.apkbuilds")
     if apkbuilds is not None:
         return apkbuilds
 
@@ -32,7 +32,7 @@ def _find_apkbuilds(args):
     apkbuilds = dict(sorted(apkbuilds.items()))
 
     # Save result in cache
-    args.cache["pmb.helpers.pmaports.apkbuilds"] = apkbuilds
+    pmb.helpers.other.cache["pmb.helpers.pmaports.apkbuilds"] = apkbuilds
     return apkbuilds
 
 
@@ -100,6 +100,41 @@ def guess_main(args, subpkgname):
             return os.path.dirname(path)
 
 
+def _find_package_in_apkbuild(package, path):
+    """
+    Look through subpackages and all provides to see if the APKBUILD at the
+    specified path contains (or provides) the specified package.
+
+    :param package: The package to search for
+    :param path: The path to the apkbuild
+    :return: True if the APKBUILD contains or provides the package
+    """
+    apkbuild = pmb.parse.apkbuild(path)
+
+    # Subpackages
+    if package in apkbuild["subpackages"]:
+        return True
+
+    # Search for provides in both package and subpackages
+    apkbuild_pkgs = [apkbuild, *apkbuild["subpackages"].values()]
+    for apkbuild_pkg in apkbuild_pkgs:
+        if not apkbuild_pkg:
+            continue
+
+        # Provides (cut off before equals sign for entries like
+        # "mkbootimg=0.0.1")
+        for provides_i in apkbuild_pkg["provides"]:
+            # Ignore provides without version, they shall never be
+            # automatically selected
+            if "=" not in provides_i:
+                continue
+
+            if package == provides_i.split("=", 1)[0]:
+                return True
+
+    return False
+
+
 def find(args, package, must_exist=True):
     """
     Find the aport path that provides a certain subpackage.
@@ -111,8 +146,8 @@ def find(args, package, must_exist=True):
     # Try to get a cached result first (we assume that the aports don't change
     # in one pmbootstrap call)
     ret = None
-    if package in args.cache["find_aport"]:
-        ret = args.cache["find_aport"][package]
+    if package in pmb.helpers.other.cache["find_aport"]:
+        ret = pmb.helpers.other.cache["find_aport"][package]
     else:
         # Sanity check
         if "*" in package:
@@ -123,36 +158,23 @@ def find(args, package, must_exist=True):
         if path:
             ret = os.path.dirname(path)
 
+        # Try to guess based on the subpackage name
+        guess = guess_main(args, package)
+        if guess:
+            # ... but see if we were right
+            if _find_package_in_apkbuild(package, f'{guess}/APKBUILD'):
+                ret = guess
+
         # Search in subpackages and provides
         if not ret:
             for path_current in _find_apkbuilds(args).values():
-                apkbuild = pmb.parse.apkbuild(args, path_current)
-                found = False
-
-                # Subpackages
-                if package in apkbuild["subpackages"]:
-                    found = True
-
-                # Provides (cut off before equals sign for entries like
-                # "mkbootimg=0.0.1")
-                if not found:
-                    for provides_i in apkbuild["provides"]:
-                        # Ignore provides without version, they shall never be
-                        # automatically selected
-                        if "=" not in provides_i:
-                            continue
-
-                        if package == provides_i.split("=", 1)[0]:
-                            found = True
-                            break
-
-                if found:
+                if _find_package_in_apkbuild(package, path_current):
                     ret = os.path.dirname(path_current)
                     break
 
-        # Guess a main package
+        # Use the guess otherwise
         if not ret:
-            ret = guess_main(args, package)
+            ret = guess
 
     # Crash when necessary
     if ret is None and must_exist:
@@ -160,7 +182,7 @@ def find(args, package, must_exist=True):
                            package)
 
     # Save result in cache
-    args.cache["find_aport"][package] = ret
+    pmb.helpers.other.cache["find_aport"][package] = ret
     return ret
 
 
@@ -185,11 +207,11 @@ def get(args, pkgname, must_exist=True, subpackages=True):
     if subpackages:
         aport = find(args, pkgname, must_exist)
         if aport:
-            return pmb.parse.apkbuild(args, aport + "/APKBUILD")
+            return pmb.parse.apkbuild(f"{aport}/APKBUILD")
     else:
         path = _find_apkbuilds(args).get(pkgname)
         if path:
-            return pmb.parse.apkbuild(args, path)
+            return pmb.parse.apkbuild(path)
         if must_exist:
             raise RuntimeError("Could not find APKBUILD for package:"
                                f" {pkgname}")
@@ -197,6 +219,30 @@ def get(args, pkgname, must_exist=True, subpackages=True):
     return None
 
 
+def find_providers(args, provide):
+    """
+    Search for providers of the specified (virtual) package in pmaports.
+    Note: Currently only providers from a single APKBUILD are returned.
+
+    :param provide: the (virtual) package to search providers for
+    :returns: tuple list (pkgname, apkbuild_pkg) with providers, sorted by
+              provider_priority. The provider with the highest priority
+              (which would be selected by default) comes first.
+    """
+
+    providers = {}
+
+    apkbuild = get(args, provide)
+    for subpkgname, subpkg in apkbuild["subpackages"].items():
+        for provides in subpkg["provides"]:
+            # Strip provides version (=$pkgver-r$pkgrel)
+            if provides.split("=", 1)[0] == provide:
+                providers[subpkgname] = subpkg
+
+    return sorted(providers.items(), reverse=True,
+                  key=lambda p: p[1].get('provider_priority', 0))
+
+
 def get_repo(args, pkgname, must_exist=True):
     """ Get the repository folder of an aport.
 

pmb/helpers/repo.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with binary package repos. See also:
@@ -43,10 +43,10 @@ def hash(url, length=8):
 def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
     """
     Get a list of repository URLs, as they are in /etc/apk/repositories.
-    :param user_repository: add /mnt/pmbootstrap-packages
+    :param user_repository: add /mnt/pmbootstrap/packages
     :param postmarketos_mirror: add postmarketos mirror URLs
     :param alpine: add alpine mirror URLs
-    :returns: list of mirror strings, like ["/mnt/pmbootstrap-packages",
+    :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
                                             "http://...", ...]
     """
     ret = []
@@ -59,7 +59,7 @@ def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
 
     # Local user repository (for packages compiled with pmbootstrap)
     if user_repository:
-        ret.append("/mnt/pmbootstrap-packages")
+        ret.append("/mnt/pmbootstrap/packages")
 
     # Upstream postmarketOS binary repository
     if postmarketos_mirror:
@@ -96,7 +96,7 @@ def apkindex_files(args, arch=None, user_repository=True, pmos=True,
     :returns: list of absolute APKINDEX.tar.gz file paths
     """
     if not arch:
-        arch = args.arch_native
+        arch = pmb.config.arch_native
 
     ret = []
     # Local user repository (for packages compiled with pmbootstrap)
@@ -127,9 +127,9 @@ def update(args, arch=None, force=False, existing_only=False):
     # Skip in offline mode, only show once
     cache_key = "pmb.helpers.repo.update"
     if args.offline:
-        if not args.cache[cache_key]["offline_msg_shown"]:
+        if not pmb.helpers.other.cache[cache_key]["offline_msg_shown"]:
             logging.info("NOTE: skipping package index update (offline mode)")
-            args.cache[cache_key]["offline_msg_shown"] = True
+            pmb.helpers.other.cache[cache_key]["offline_msg_shown"] = True
         return False
 
     # Architectures and retention time
@@ -151,7 +151,7 @@ def update(args, arch=None, force=False, existing_only=False):
 
             # Find update reason, possibly skip non-existing or known 404 files
             reason = None
-            if url_full in args.cache[cache_key]["404"]:
+            if url_full in pmb.helpers.other.cache[cache_key]["404"]:
                 # We already attempted to download this file once in this
                 # session
                 continue
@@ -184,7 +184,7 @@ def update(args, arch=None, force=False, existing_only=False):
         temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
                                          logging.DEBUG, True)
         if not temp:
-            args.cache[cache_key]["404"].append(url)
+            pmb.helpers.other.cache[cache_key]["404"].append(url)
             continue
         target_folder = os.path.dirname(target)
         if not os.path.exists(target_folder):
@@ -209,7 +209,7 @@ def alpine_apkindex_path(args, repo="main", arch=None):
         raise RuntimeError("Invalid Alpine repository: " + repo)
 
     # Download the file
-    arch = arch or args.arch_native
+    arch = arch or pmb.config.arch_native
     update(args, arch)
 
     # Find it on disk

pmb/helpers/repo_missing.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 

pmb/helpers/run.py

@@ -1,38 +1,8 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import shlex
 import pmb.helpers.run_core
 
 
-def flat_cmd(cmd, working_dir=None, env={}):
-    """
-    Convert a shell command passed as list into a flat shell string with
-    proper escaping.
-
-    :param cmd: command as list, e.g. ["echo", "string with spaces"]
-    :param working_dir: when set, prepend "cd ...;" to execute the command
-                        in the given working directory
-    :param env: dict of environment variables to be passed to the command, e.g.
-                {"JOBS": "5"}
-    :returns: the flat string, e.g.
-              echo 'string with spaces'
-              cd /home/pmos;echo 'string with spaces'
-    """
-    # Merge env and cmd into escaped list
-    escaped = []
-    for key, value in env.items():
-        escaped.append(key + "=" + shlex.quote(value))
-    for i in range(len(cmd)):
-        escaped.append(shlex.quote(cmd[i]))
-
-    # Prepend working dir
-    ret = " ".join(escaped)
-    if working_dir:
-        ret = "cd " + shlex.quote(working_dir) + ";" + ret
-
-    return ret
-
-
 def user(args, cmd, working_dir=None, output="log", output_return=False,
          check=None, env={}, sudo=False):
     """
@@ -54,7 +24,7 @@ def user(args, cmd, working_dir=None, output="log", output_return=False,
 
     # Add environment variables and run
     if env:
-        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
+        cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
     return pmb.helpers.run_core.core(args, msg, cmd, working_dir, output,
                                      output_return, check, sudo)
 
@@ -62,7 +32,7 @@ def user(args, cmd, working_dir=None, output="log", output_return=False,
 def root(args, cmd, working_dir=None, output="log", output_return=False,
          check=None, env={}):
     """
-    Run a command on the host system as root, with sudo.
+    Run a command on the host system as root, with sudo or doas.
 
     :param env: dict of environment variables to be passed to the command, e.g.
                 {"JOBS": "5"}
@@ -71,8 +41,8 @@ def root(args, cmd, working_dir=None, output="log", output_return=False,
     arguments and the return value.
     """
     if env:
-        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
-    cmd = ["sudo"] + cmd
+        cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
+    cmd = pmb.config.sudo(cmd)
 
     return user(args, cmd, working_dir, output, output_return, check, env,
                 True)

pmb/helpers/run_core.py

@@ -1,13 +1,14 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fcntl
 import logging
+import os
 import selectors
+import shlex
 import subprocess
 import sys
 import threading
 import time
-import os
 import pmb.helpers.run
 
 """ For a detailed description of all output modes, read the description of
@@ -15,6 +16,35 @@ import pmb.helpers.run
     called by core(). """
 
 
+def flat_cmd(cmd, working_dir=None, env={}):
+    """
+    Convert a shell command passed as list into a flat shell string with
+    proper escaping.
+
+    :param cmd: command as list, e.g. ["echo", "string with spaces"]
+    :param working_dir: when set, prepend "cd ...;" to execute the command
+                        in the given working directory
+    :param env: dict of environment variables to be passed to the command, e.g.
+                {"JOBS": "5"}
+    :returns: the flat string, e.g.
+              echo 'string with spaces'
+              cd /home/pmos;echo 'string with spaces'
+    """
+    # Merge env and cmd into escaped list
+    escaped = []
+    for key, value in env.items():
+        escaped.append(key + "=" + shlex.quote(value))
+    for i in range(len(cmd)):
+        escaped.append(shlex.quote(cmd[i]))
+
+    # Prepend working dir
+    ret = " ".join(escaped)
+    if working_dir:
+        ret = "cd " + shlex.quote(working_dir) + ";" + ret
+
+    return ret
+
+
 def sanity_checks(output="log", output_return=False, check=None):
     """
     Raise an exception if the parameters passed to core() don't make sense
@@ -35,23 +65,24 @@ def sanity_checks(output="log", output_return=False, check=None):
         raise RuntimeError("Can't use output_return with output: " + output)
 
 
-def background(args, cmd, working_dir=None):
+def background(cmd, working_dir=None):
     """ Run a subprocess in background and redirect its output to the log. """
-    ret = subprocess.Popen(cmd, stdout=args.logfd, stderr=args.logfd,
-                           cwd=working_dir)
+    ret = subprocess.Popen(cmd, stdout=pmb.helpers.logging.logfd,
+                           stderr=pmb.helpers.logging.logfd, cwd=working_dir)
     logging.debug(f"New background process: pid={ret.pid}, output=background")
     return ret
 
 
-def pipe(args, cmd, working_dir=None):
+def pipe(cmd, working_dir=None):
     """ Run a subprocess in background and redirect its output to a pipe. """
-    ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=args.logfd,
-                           cwd=working_dir)
+    ret = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                           stdin=subprocess.DEVNULL,
+                           stderr=pmb.helpers.logging.logfd, cwd=working_dir)
     logging.verbose(f"New background process: pid={ret.pid}, output=pipe")
     return ret
 
 
-def pipe_read(args, process, output_to_stdout=False, output_return=False,
+def pipe_read(process, output_to_stdout=False, output_return=False,
               output_return_buffer=False):
     """
     Read all available output from a subprocess and copy it to the log and
@@ -69,7 +100,7 @@ def pipe_read(args, process, output_to_stdout=False, output_return=False,
         # Copy available output
         out = process.stdout.readline()
         if len(out):
-            args.logfd.buffer.write(out)
+            pmb.helpers.logging.logfd.buffer.write(out)
             if output_to_stdout:
                 sys.stdout.buffer.write(out)
             if output_return:
@@ -77,7 +108,7 @@ def pipe_read(args, process, output_to_stdout=False, output_return=False,
             continue
 
         # No more output (flush buffers)
-        args.logfd.flush()
+        pmb.helpers.logging.logfd.flush()
         if output_to_stdout:
             sys.stdout.flush()
         return
@@ -125,7 +156,7 @@ def kill_command(args, pid, sudo):
 
 def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False,
                     output_return=False, output_timeout=True,
-                    sudo=False):
+                    sudo=False, stdin=None):
     """
     Run a subprocess in foreground with redirected output and optionally kill
     it after being silent for too long.
@@ -145,7 +176,8 @@ def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False,
     """
     # Start process in background (stdout and stderr combined)
     process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                               stderr=subprocess.STDOUT, cwd=working_dir)
+                               stderr=subprocess.STDOUT, cwd=working_dir,
+                               stdin=stdin)
 
     # Make process.stdout non-blocking
     handle = process.stdout.fileno()
@@ -175,11 +207,11 @@ def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False,
                 continue
 
         # Read all currently available output
-        pipe_read(args, process, output_to_stdout, output_return,
+        pipe_read(process, output_to_stdout, output_return,
                   output_buffer)
 
     # There may still be output after the process quit
-    pipe_read(args, process, output_to_stdout, output_return, output_buffer)
+    pipe_read(process, output_to_stdout, output_return, output_buffer)
 
     # Return the return code and output (the output gets built as list of
     # output chunks and combined at the end, this is faster than extending the
@@ -216,7 +248,8 @@ def check_return_code(args, code, log_message):
         logging.debug("^" * 70)
         logging.info("NOTE: The failed command's output is above the ^^^ line"
                      " in the log file: " + args.log)
-        raise RuntimeError("Command failed: " + log_message)
+        raise RuntimeError(f"Command failed (exit code {str(code)}): " +
+                           log_message)
 
 
 def sudo_timer_iterate():
@@ -224,22 +257,25 @@ def sudo_timer_iterate():
     Run sudo -v and schedule a new timer to repeat the same.
     """
 
-    subprocess.Popen(["sudo", "-v"]).wait()
+    if pmb.config.which_sudo() == "sudo":
+        subprocess.Popen(["sudo", "-v"]).wait()
+    else:
+        subprocess.Popen(pmb.config.sudo(["true"])).wait()
 
     timer = threading.Timer(interval=60, function=sudo_timer_iterate)
     timer.daemon = True
     timer.start()
 
 
-def sudo_timer_start(args):
+def sudo_timer_start():
     """
     Start a timer to call sudo -v periodically, so that the password is only
     needed once.
     """
 
-    if "sudo_timer_active" in args.cache:
+    if "sudo_timer_active" in pmb.helpers.other.cache:
         return
-    args.cache["sudo_timer_active"] = True
+    pmb.helpers.other.cache["sudo_timer_active"] = True
 
     sudo_timer_iterate()
 
@@ -279,14 +315,14 @@ def core(args, log_message, cmd, working_dir=None, output="log",
                    their properties. "wait" indicates that we wait for the
                    process to complete.
 
-                   output value  | timeout | out to log | out to stdout | wait
-                   -----------------------------------------------------------
-                   "log"         | x       | x          |               | x
-                   "stdout"      | x       | x          | x             | x
-                   "interactive" |         | x          | x             | x
-                   "tui"         |         |            | x             | x
-                   "background"  |         | x          |               |
-                   "pipe"        |         |            |               |
+                   output value  | timeout | out to log | out to stdout | wait | pass stdin
+                   ------------------------------------------------------------------------
+                   "log"         | x       | x          |               | x    |
+                   "stdout"      | x       | x          | x             | x    |
+                   "interactive" |         | x          | x             | x    | x
+                   "tui"         |         |            | x             | x    | x
+                   "background"  |         | x          |               |      |
+                   "pipe"        |         |            |               |      |
 
     :param output_return: in addition to writing the program's output to the
                           destinations above in real time, write to a buffer
@@ -304,8 +340,17 @@ def core(args, log_message, cmd, working_dir=None, output="log",
     """
     sanity_checks(output, output_return, check)
 
+    # Preserve proxy environment variables
+    env = {}
+    for var in ["FTP_PROXY", "ftp_proxy", "HTTP_PROXY", "http_proxy",
+                "HTTPS_PROXY", "https_proxy", "HTTP_PROXY_AUTH"]:
+        if var in os.environ:
+            env[var] = os.environ[var]
+    if env:
+        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
+
     if args.sudo_timer and sudo:
-        sudo_timer_start(args)
+        sudo_timer_start()
 
     # Log simplified and full command (pmbootstrap -v)
     logging.debug(log_message)
@@ -313,11 +358,11 @@ def core(args, log_message, cmd, working_dir=None, output="log",
 
     # Background
     if output == "background":
-        return background(args, cmd, working_dir)
+        return background(cmd, working_dir)
 
     # Pipe
     if output == "pipe":
-        return pipe(args, cmd, working_dir)
+        return pipe(cmd, working_dir)
 
     # Foreground
     output_after_run = ""
@@ -332,11 +377,13 @@ def core(args, log_message, cmd, working_dir=None, output="log",
 
         output_timeout = output in ["log", "stdout"] and not disable_timeout
 
+        stdin = subprocess.DEVNULL if output in ["log", "stdout"] else None
+
         (code, output_after_run) = foreground_pipe(args, cmd, working_dir,
                                                    output_to_stdout,
                                                    output_return,
                                                    output_timeout,
-                                                   sudo)
+                                                   sudo, stdin)
 
     # Check the return code
     if check is not False:

pmb/helpers/status.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -92,7 +92,7 @@ def print_checks_git_repo(args, repo, details=True):
     log_ok("up to date with remote branch")
 
     # Outdated remote information
-    if pmb.helpers.git.is_outdated(args, path):
+    if pmb.helpers.git.is_outdated(path):
         return log_nok_ret(-5, "outdated remote information",
                            "update with 'pmbootstrap pull'")
     log_ok("remote information updated recently (via git fetch/pull)")

pmb/helpers/ui.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Clayton Craft
+# Copyright 2023 Clayton Craft
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob
@@ -12,9 +12,11 @@ def list(args, arch):
     :param arch: device architecture, for which the UIs must be available
     :returns: [("none", "No graphical..."), ("weston", "Wayland reference...")]
     """
-    ret = [("none", "No graphical environment")]
+    ret = [("none", "Bare minimum OS image for testing and manual"
+                    " customization. The \"console\" UI should be selected if"
+                    " a graphical UI is not desired.")]
     for path in sorted(glob.glob(args.aports + "/main/postmarketos-ui-*")):
-        apkbuild = pmb.parse.apkbuild(args, path + "/APKBUILD")
+        apkbuild = pmb.parse.apkbuild(f"{path}/APKBUILD")
         ui = os.path.basename(path).split("-", 2)[2]
         if pmb.helpers.package.check_arch(args, apkbuild["pkgname"], arch):
             ret.append((ui, apkbuild["pkgdesc"]))

pmb/install/__init__.py

@@ -1,8 +1,9 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.install._install import install
 from pmb.install._install import get_kernel_package
 from pmb.install.partition import partition
+from pmb.install.partition import partition_cgpt
 from pmb.install.format import format
 from pmb.install.format import get_root_filesystem
 from pmb.install.partition import partitions_mount

pmb/install/_install.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -64,8 +64,7 @@ def get_nonfree_packages(args, device):
               ["device-nokia-n900-nonfree-firmware"]
     """
     # Read subpackages
-    apkbuild = pmb.parse.apkbuild(args,
-                                  pmb.helpers.devices.find_path(args, device,
+    apkbuild = pmb.parse.apkbuild(pmb.helpers.devices.find_path(args, device,
                                                                 'APKBUILD'))
     subpackages = apkbuild["subpackages"]
 
@@ -165,8 +164,8 @@ def create_home_from_skel(args):
 def configure_apk(args):
     """
     Copy over all official keys, and the keys used to compile local packages
-    (unless --no-local-pkgs is set). Then disable the /mnt/pmbootstrap-packages
-    repository.
+    (unless --no-local-pkgs is set). Then copy the corresponding APKINDEX files
+    and remove the /mnt/pmbootstrap/packages repository.
     """
     # Official keys
     pattern = f"{pmb.config.apk_keys_path}/*.pub"
@@ -180,8 +179,15 @@ def configure_apk(args):
     for key in glob.glob(pattern):
         pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"])
 
+    # Copy over the corresponding APKINDEX files from cache
+    index_files = pmb.helpers.repo.apkindex_files(args,
+                                                  arch=args.deviceinfo["arch"],
+                                                  user_repository=False)
+    for f in index_files:
+        pmb.helpers.run.root(args, ["cp", f, rootfs + "/var/cache/apk/"])
+
     # Disable pmbootstrap repository
-    pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap-packages/d",
+    pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap\/packages/d",
                                 rootfs + "/etc/apk/repositories"])
     pmb.helpers.run.user(args, ["cat", rootfs + "/etc/apk/repositories"])
 
@@ -190,20 +196,64 @@ def set_user(args):
     """
     Create user with UID 10000 if it doesn't exist.
     Usually the ID for the first user created is 1000, but higher ID is
-    chosen here to avoid conflict with Android UIDs/GIDs.
-
+    chosen here to not cause issues with existing installations. Historically,
+    this was done to avoid conflict with Android UIDs/GIDs, but pmOS has since
+    dropped support for hybris/Halium.
     """
     suffix = "rootfs_" + args.device
     if not pmb.chroot.user_exists(args, args.user, suffix):
         pmb.chroot.root(args, ["adduser", "-D", "-u", "10000", args.user],
                         suffix)
-    groups = pmb.install.ui.get_groups(args) + pmb.config.install_user_groups
+
+    pmaports_cfg = pmb.config.pmaports.read_config(args)
+    groups = []
+    groups += pmaports_cfg.get("install_user_groups",
+                               "audio,input,netdev,plugdev,video,wheel").split(",")
+    groups += pmb.install.ui.get_groups(args)
+
     for group in groups:
         pmb.chroot.root(args, ["addgroup", "-S", group], suffix,
                         check=False)
         pmb.chroot.root(args, ["addgroup", args.user, group], suffix)
 
 
+def setup_login_chpasswd_user_from_arg(args, suffix):
+    """
+    Set the user's password from what the user passed as --password. Make an
+    effort to not have the password end up in the log file by writing it to
+    a temp file, instead of "echo user:$pass | chpasswd". The user should of
+    course only use this with a test password anyway, but let's be nice and try
+    to have the user protected from accidentally posting their password in
+    any case.
+
+    :param suffix: of the chroot, where passwd will be execute (either the
+                   f"rootfs_{args.device}", or f"installer_{args.device}")
+    """
+    path = "/tmp/pmbootstrap_chpasswd_in"
+    path_outside = f"{args.work}/chroot_{suffix}{path}"
+
+    with open(path_outside, "w", encoding="utf-8") as handle:
+        handle.write(f"{args.user}:{args.password}")
+
+    pmb.chroot.root(args, ["sh", "-c", f"cat {shlex.quote(path)} | chpasswd"],
+                    suffix)
+
+    os.unlink(path_outside)
+
+
+def is_root_locked(args, suffix):
+    """
+    Figure out from /etc/shadow if root is already locked. The output of this
+    is stored in the log, so use grep to only log the line for root, not the
+    line for the user which contains a hash of the user's password.
+
+    :param suffix: either rootfs_{args.device} or installer_{args.device}
+    """
+    shadow_root = pmb.chroot.root(args, ["grep", "^root:!:", "/etc/shadow"],
+                                  suffix, output_return=True, check=False)
+    return shadow_root.startswith("root:!:")
+
+
 def setup_login(args, suffix):
     """
     Loop until the password for user has been set successfully, and disable
@@ -214,19 +264,25 @@ def setup_login(args, suffix):
     """
     if not args.on_device_installer:
         # User password
-        logging.info(" *** SET LOGIN PASSWORD FOR: '" + args.user + "' ***")
-        while True:
-            try:
-                pmb.chroot.root(args, ["passwd", args.user], suffix,
-                                output="interactive")
-                break
-            except RuntimeError:
-                logging.info("WARNING: Failed to set the password. Try it"
-                             " one more time.")
-                pass
+        logging.info(f" *** SET LOGIN PASSWORD FOR: '{args.user}' ***")
+        if args.password:
+            setup_login_chpasswd_user_from_arg(args, suffix)
+        else:
+            while True:
+                try:
+                    pmb.chroot.root(args, ["passwd", args.user], suffix,
+                                    output="interactive")
+                    break
+                except RuntimeError:
+                    logging.info("WARNING: Failed to set the password. Try it"
+                                 " one more time.")
 
     # Disable root login
-    pmb.chroot.root(args, ["passwd", "-l", "root"], suffix)
+    if is_root_locked(args, suffix):
+        logging.debug(f"({suffix}) root is already locked")
+    else:
+        logging.debug(f"({suffix}) locking root")
+        pmb.chroot.root(args, ["passwd", "-l", "root"], suffix)
 
 
 def copy_ssh_keys(args):
@@ -236,7 +292,7 @@ def copy_ssh_keys(args):
     if not args.ssh_keys:
         return
     keys = []
-    for key in glob.glob(os.path.expanduser("~/.ssh/id_*.pub")):
+    for key in glob.glob(os.path.expanduser(args.ssh_key_glob)):
         with open(key, "r") as infile:
             keys += infile.readlines()
 
@@ -304,6 +360,26 @@ def setup_keymap(args):
         logging.info("NOTE: No valid keymap specified for device")
 
 
+def setup_timezone(args):
+    suffix = f"rootfs_{args.device}"
+
+    arch = args.deviceinfo["arch"]
+    alpine_conf = pmb.helpers.package.get(args, "alpine-conf", arch)
+    version = alpine_conf["version"].split("-r")[0]
+
+    setup_tz_cmd = ["setup-timezone"]
+    # setup-timezone will, by default, copy the timezone to /etc/zoneinfo
+    # and disregard tzdata, to save space. If we actually have tzdata
+    # installed, make sure that setup-timezone makes use of it, since
+    # there's no space to be saved.
+    if "tzdata" in pmb.chroot.apk.installed(args, suffix):
+        setup_tz_cmd += ["-i"]
+    if not pmb.parse.version.check_string(version, ">=3.14.0"):
+        setup_tz_cmd += ["-z"]
+    setup_tz_cmd += [args.timezone]
+    pmb.chroot.root(args, setup_tz_cmd, suffix)
+
+
 def setup_hostname(args):
     """
     Set the hostname and update localhost address in /etc/hosts
@@ -327,6 +403,25 @@ def setup_hostname(args):
     pmb.chroot.root(args, ["sed", "-i", "-e", regex, "/etc/hosts"], suffix)
 
 
+def setup_appstream(args):
+    """
+    If alpine-appstream-downloader has been downloaded, execute it to have
+    update AppStream data on new installs
+    """
+    suffix = "rootfs_" + args.device
+    installed_pkgs = pmb.chroot.apk.installed(args, suffix)
+
+    if "alpine-appstream-downloader" not in installed_pkgs or args.offline:
+        return
+
+    pmb.chroot.root(args, ["alpine-appstream-downloader",
+                           "/mnt/appstream-data"], suffix)
+    pmb.chroot.root(args, ["mkdir", "-p", "/var/lib/swcatalog"], suffix)
+    pmb.chroot.root(args, ["cp", "-r", "/mnt/appstream-data/icons",
+                           "/mnt/appstream-data/xml",
+                           "-t", "/var/lib/swcatalog"], suffix)
+
+
 def disable_sshd(args):
     if not args.no_sshd:
         return
@@ -510,6 +605,22 @@ def embed_firmware(args, suffix):
                                "bs=" + str(step), "seek=" + str(offset)])
 
 
+def write_cgpt_kpart(args, layout, suffix):
+    """
+    Write the kernel to the ChromeOS kernel partition.
+
+    :param layout: partition layout from get_partition_layout()
+    :param suffix: of the chroot, which holds the image file to be flashed
+    """
+    if not args.deviceinfo["cgpt_kpart"] or not args.install_cgpt:
+        return
+
+    device_rootfs = mount_device_rootfs(args, suffix)
+    filename = f"{device_rootfs}{args.deviceinfo['cgpt_kpart']}"
+    pmb.chroot.root(
+        args, ["dd", f"if={filename}", f"of=/dev/installp{layout['kernel']}"])
+
+
 def sanity_check_sdcard(args):
     device = args.sdcard
     device_name = os.path.basename(device)
@@ -564,6 +675,99 @@ def sanity_check_ondev_version(args):
                            f" / in the binary packages has version {ver_pkg}.")
 
 
+def get_partition_layout(reserve, kernel):
+    """
+    :param reserve: create an empty partition between root and boot (pma#463)
+    :param kernel: create a separate kernel partition before all other
+                   partitions, e.g. for the ChromeOS devices with cgpt
+    :returns: the partition layout, e.g. without reserve and kernel:
+              {"kernel": None, "boot": 1, "reserve": None, "root": 2}
+    """
+    ret = {}
+    ret["kernel"] = None
+    ret["boot"] = 1
+    ret["reserve"] = None
+    ret["root"] = 2
+
+    if kernel:
+        ret["kernel"] = 1
+        ret["boot"] += 1
+        ret["root"] += 1
+
+    if reserve:
+        ret["reserve"] = ret["root"]
+        ret["root"] += 1
+    return ret
+
+
+def get_uuid(args, partition):
+    """
+    Get UUID of a partition
+
+    :param partition: block device for getting UUID from
+    """
+    return pmb.chroot.root(
+        args,
+        [
+            "blkid",
+            "-s", "UUID",
+            "-o", "value",
+            partition,
+        ],
+        output_return=True
+    ).rstrip()
+
+
+def create_crypttab(args, layout, suffix):
+    """
+    Create /etc/crypttab config
+
+    :param layout: partition layout from get_partition_layout()
+    :param suffix: of the chroot, which crypttab will be created to
+    """
+
+    luks_uuid = get_uuid(args, f"/dev/installp{layout['root']}")
+
+    crypttab = f"root UUID={luks_uuid} none luks\n"
+
+    open(f"{args.work}/chroot_{suffix}/tmp/crypttab", "w").write(crypttab)
+    pmb.chroot.root(args, ["mv", "/tmp/crypttab", "/etc/crypttab"], suffix)
+
+
+def create_fstab(args, layout, suffix):
+    """
+    Create /etc/fstab config
+
+    :param layout: partition layout from get_partition_layout()
+    :param suffix: of the chroot, which fstab will be created to
+    """
+
+    # Do not install fstab into target rootfs when using on-device
+    # installer. Provide fstab only to installer suffix
+    if args.on_device_installer and "rootfs_" in suffix:
+        return
+
+    boot_dev = f"/dev/installp{layout['boot']}"
+    root_dev = f"/dev/installp{layout['root']}"
+
+    boot_mount_point = f"UUID={get_uuid(args, boot_dev)}"
+    root_mount_point = "/dev/mapper/root" if args.full_disk_encryption \
+        else f"UUID={get_uuid(args, root_dev)}"
+
+    boot_filesystem = args.deviceinfo["boot_filesystem"] or "ext2"
+    root_filesystem = pmb.install.get_root_filesystem(args)
+
+    fstab = f"""
+# <file system> <mount point> <type> <options> <dump> <pass>
+{root_mount_point} / {root_filesystem} defaults 0 0
+{boot_mount_point} /boot {boot_filesystem} defaults 0 0
+""".lstrip()
+
+    with open(f"{args.work}/chroot_{suffix}/tmp/fstab", "w") as f:
+        f.write(fstab)
+    pmb.chroot.root(args, ["mv", "/tmp/fstab", "/etc/fstab"], suffix)
+
+
 def install_system_image(args, size_reserve, suffix, step, steps,
                          boot_label="pmOS_boot", root_label="pmOS_root",
                          split=False, sdcard=None):
@@ -582,16 +786,37 @@ def install_system_image(args, size_reserve, suffix, step, steps,
     logging.info(f"*** ({step}/{steps}) PREPARE INSTALL BLOCKDEVICE ***")
     pmb.chroot.shutdown(args, True)
     (size_boot, size_root) = get_subpartitions_size(args, suffix)
+    layout = get_partition_layout(size_reserve, args.deviceinfo["cgpt_kpart"] \
+             and args.install_cgpt)
     if not args.rsync:
         pmb.install.blockdevice.create(args, size_boot, size_root,
                                        size_reserve, split, sdcard)
         if not split:
-            pmb.install.partition(args, size_boot, size_reserve)
+            if args.deviceinfo["cgpt_kpart"] and args.install_cgpt:
+                pmb.install.partition_cgpt(
+                    args, layout, size_boot, size_reserve)
+            else:
+                pmb.install.partition(args, layout, size_boot, size_reserve)
     if not split:
-        root_id = 3 if size_reserve else 2
-        pmb.install.partitions_mount(args, root_id, sdcard)
+        pmb.install.partitions_mount(args, layout, sdcard)
 
-    pmb.install.format(args, size_reserve, boot_label, root_label, sdcard)
+    pmb.install.format(args, layout, boot_label, root_label, sdcard)
+
+    # Create /etc/fstab and /etc/crypttab
+    logging.info("(native) create /etc/fstab")
+    create_fstab(args, layout, suffix)
+    if args.full_disk_encryption:
+        logging.info("(native) create /etc/crypttab")
+        create_crypttab(args, layout, suffix)
+
+    # Run mkinitfs to pass UUIDs to cmdline
+    logging.info(f"({suffix}) mkinitfs")
+    pmb.chroot.root(args, ["mkinitfs"], suffix)
+
+    # Clean up after running mkinitfs in chroot
+    pmb.helpers.mount.umount_all(args, f"{args.work}/chroot_{suffix}")
+    pmb.helpers.run.root(args, ["rm", f"{args.work}/chroot_{suffix}/in-pmbootstrap"])
+    pmb.chroot.remove_mnt_pmbootstrap(args, suffix)
 
     # Just copy all the files
     logging.info(f"*** ({step + 1}/{steps}) FILL INSTALL BLOCKDEVICE ***")
@@ -600,10 +825,11 @@ def install_system_image(args, size_reserve, suffix, step, steps,
     configure_apk(args)
     copy_ssh_keys(args)
 
-    # Don't try to embed firmware on split images since there's no
+    # Don't try to embed firmware and cgpt on split images since there's no
     # place to put it and it will end up in /dev of the chroot instead
     if not split:
         embed_firmware(args, suffix)
+        write_cgpt_kpart(args, layout, suffix)
 
     if sdcard:
         logging.info("Unmounting SD card (this may take a while "
@@ -625,6 +851,19 @@ def install_system_image(args, size_reserve, suffix, step, steps,
         pmb.chroot.user(args, ["mv", "-f", sys_image_sparse, sys_image],
                         working_dir="/home/pmos/rootfs/")
 
+        # patch sparse image for Samsung devices if specified
+        samsungify_strategy = args.deviceinfo["flash_sparse_samsung_format"]
+        if samsungify_strategy:
+            logging.info("(native) convert sparse image into Samsung's sparse image format")
+            pmb.chroot.apk.install(args, ["sm-sparse-image-tool"])
+            sys_image = f"{args.device}.img"
+            sys_image_patched = f"{args.device}-patched.img"
+            pmb.chroot.user(args, ["sm_sparse_image_tool", "samsungify", "--strategy",
+                                   samsungify_strategy, sys_image, sys_image_patched],
+                            working_dir="/home/pmos/rootfs/")
+            pmb.chroot.user(args, ["mv", "-f", sys_image_patched, sys_image],
+                            working_dir="/home/pmos/rootfs/")
+
 
 def print_flash_info(args):
     """ Print flashing information, based on the deviceinfo data and the
@@ -698,6 +937,14 @@ def print_flash_info(args):
                      " the kernel/initramfs directly without flashing."
                      " Use 'pmbootstrap flasher boot' to do that.)")
 
+    if "flash_lk2nd" in flasher_actions and \
+            os.path.exists(args.work + "/chroot_rootfs_" + args.device +
+                           "/boot/lk2nd.img"):
+        logging.info("* Your device supports and may even require"
+                     " flashing lk2nd. You should flash it before"
+                     " flashing anything else. Use 'pmbootstrap flasher"
+                     " flash_lk2nd' to do that.")
+
     # Export information
     logging.info("* If the above steps do not work, you can also create"
                  " symlinks to the generated files with 'pmbootstrap export'"
@@ -757,7 +1004,7 @@ def install_on_device_installer(args, step, steps):
            "ONDEV_CHANNEL_DESCRIPTION": channel_cfg["description"],
            "ONDEV_CHANNEL_MIRRORDIR_ALPINE": channel_cfg["mirrordir_alpine"],
            "ONDEV_CIPHER": args.cipher,
-           "ONDEV_PMBOOTSTRAP_VERSION": pmb.config.version,
+           "ONDEV_PMBOOTSTRAP_VERSION": pmb.__version__,
            "ONDEV_UI": args.ui}
     pmb.chroot.root(args, ["ondev-prepare"], suffix_installer, env=env)
 
@@ -789,6 +1036,24 @@ def install_on_device_installer(args, step, steps):
                          boot_label, "pmOS_install", args.split, args.sdcard)
 
 
+def get_selected_providers(args, packages):
+    """
+    Look through the specified packages and see which providers were selected
+    in "pmbootstrap init". Install those as extra packages to select them
+    instead of the default provider.
+
+    :param packages: the packages that have selectable providers (_pmb_select)
+    :return: additional provider packages to install
+    """
+    providers = []
+    for p in packages:
+        apkbuild = pmb.helpers.pmaports.get(args, p, subpackages=False)
+        for select in apkbuild['_pmb_select']:
+            if select in args.selected_providers:
+                providers.append(args.selected_providers[select])
+    return providers
+
+
 def create_device_rootfs(args, step, steps):
     # List all packages to be installed (including the ones specified by --add)
     # and upgrade the installed packages/apkindexes
@@ -802,17 +1067,23 @@ def create_device_rootfs(args, step, steps):
 
     # Fill install_packages
     install_packages = (pmb.config.install_device_packages +
-                        ["device-" + args.device] +
-                        get_kernel_package(args, args.device) +
-                        get_nonfree_packages(args, args.device) +
-                        pmb.install.ui.get_recommends(args))
+                        ["device-" + args.device])
     if not args.install_base:
         install_packages = [p for p in install_packages
                             if p != "postmarketos-base"]
     if args.ui.lower() != "none":
         install_packages += ["postmarketos-ui-" + args.ui]
+
+    # Add additional providers of base/device/UI package
+    install_packages += get_selected_providers(args, install_packages)
+
+    install_packages += get_kernel_package(args, args.device)
+    install_packages += get_nonfree_packages(args, args.device)
+    if args.ui.lower() != "none":
         if args.ui_extras:
             install_packages += ["postmarketos-ui-" + args.ui + "-extras"]
+        if args.install_recommends:
+            install_packages += pmb.install.ui.get_recommends(args)
     if args.extra_packages.lower() != "none":
         install_packages += args.extra_packages.split(",")
     if args.add:
@@ -830,7 +1101,12 @@ def create_device_rootfs(args, step, steps):
         # install it when the ondev installer is running.
         # Always install it when --fde is specified.
         if args.full_disk_encryption or args.on_device_installer:
-            install_packages += ["osk-sdl"]
+            # Pick the most suitable unlocker depending on the packages
+            # selected for installation
+            unlocker = pmb.parse.depends.package_provider(
+                args, "postmarketos-fde-unlocker", install_packages, suffix)
+            if unlocker["pkgname"] not in install_packages:
+                install_packages += [unlocker["pkgname"]]
         else:
             install_packages += ["postmarketos-base-nofde"]
 
@@ -856,17 +1132,22 @@ def create_device_rootfs(args, step, steps):
     setup_keymap(args)
 
     # Set timezone
-    pmb.chroot.root(args, ["setup-timezone", "-z", args.timezone], suffix)
+    setup_timezone(args)
 
     # Set locale
     if locale_is_set:
-        pmb.chroot.root(args, ["sed", "-i",
-                               f"s/LANG=C.UTF-8/LANG={args.locale}/",
-                               "/etc/profile.d/locale.sh"], suffix)
+        # 10locale-pmos.sh gets sourced before 20locale.sh from
+        # alpine-baselayout by /etc/profile. Since they don't override the
+        # locale if it exists, it warranties we have preference
+        line = f"export LANG=${{LANG:-{shlex.quote(args.locale)}}}"
+        pmb.chroot.root(args, ["sh", "-c", f"echo {shlex.quote(line)}"
+                               " > /etc/profile.d/10locale-pmos.sh"], suffix)
 
     # Set the hostname as the device name
     setup_hostname(args)
 
+    setup_appstream(args)
+
     disable_sshd(args)
     disable_firewall(args)
 

pmb/install/blockdevice.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -24,8 +24,15 @@ def previous_install(args, path):
         pmb.helpers.mount.bind_file(args, blockdevice_outside,
                                     args.work + '/chroot_native' +
                                     blockdevice_inside)
-        label = pmb.chroot.root(args, ["blkid", "-s", "LABEL", "-o", "value",
-                                       blockdevice_inside], output_return=True)
+        try:
+            label = pmb.chroot.root(args, ["blkid", "-s", "LABEL",
+                                           "-o", "value",
+                                           blockdevice_inside],
+                                    output_return=True)
+        except RuntimeError:
+            logging.info("WARNING: Could not get block device label,"
+                         " assume no previous installation on that partition")
+
         pmb.helpers.run.root(args, ["umount", args.work + "/chroot_native" +
                                     blockdevice_inside])
     return "pmOS_boot" in label

pmb/install/format.py

@@ -1,4 +1,4 @@
-# Copyright 2021 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -13,15 +13,15 @@ def install_fsprogs(args, filesystem):
     pmb.chroot.apk.install(args, [fsprogs])
 
 
-def format_and_mount_boot(args, boot_label):
+def format_and_mount_boot(args, device, boot_label):
     """
+    :param device: boot partition on install block device (e.g. /dev/installp1)
     :param boot_label: label of the root partition (e.g. "pmOS_boot")
 
     When adjusting this function, make sure to also adjust
     ondev-prepare-internal-storage.sh in postmarketos-ondev.git!
     """
     mountpoint = "/mnt/install/boot"
-    device = "/dev/installp1"
     filesystem = args.deviceinfo["boot_filesystem"] or "ext2"
     install_fsprogs(args, filesystem)
     logging.info(f"(native) format {device} (boot, {filesystem}), mount to"
@@ -35,6 +35,9 @@ def format_and_mount_boot(args, boot_label):
     elif filesystem == "ext2":
         pmb.chroot.root(args, ["mkfs.ext2", "-F", "-q", "-L", boot_label,
                                device])
+    elif filesystem == "btrfs":
+        pmb.chroot.root(args, ["mkfs.btrfs", "-f", "-q", "-L", boot_label,
+                               device])
     else:
         raise RuntimeError("Filesystem " + filesystem + " is not supported!")
     pmb.chroot.root(args, ["mkdir", "-p", mountpoint])
@@ -100,12 +103,14 @@ def format_and_mount_root(args, device, root_label, sdcard):
             mkfs_root_args = ["mkfs.ext4", "-O", "^metadata_csum", "-F",
                               "-q", "-L", root_label]
             # When we don't know the file system size before hand like
-            # with non-block devices, we need to explicitely set a number of
+            # with non-block devices, we need to explicitly set a number of
             # inodes. See #1717 and #1845 for details
             if not sdcard:
                 mkfs_root_args = mkfs_root_args + ["-N", "100000"]
         elif filesystem == "f2fs":
             mkfs_root_args = ["mkfs.f2fs", "-f", "-l", root_label]
+        elif filesystem == "btrfs":
+            mkfs_root_args = ["mkfs.btrfs", "-f", "-L", root_label]
         else:
             raise RuntimeError(f"Don't know how to format {filesystem}!")
 
@@ -120,18 +125,19 @@ def format_and_mount_root(args, device, root_label, sdcard):
     pmb.chroot.root(args, ["mount", device, mountpoint])
 
 
-def format(args, size_reserve, boot_label, root_label, sdcard):
+def format(args, layout, boot_label, root_label, sdcard):
     """
-    :param size_reserve: empty partition between root and boot in MiB (pma#463)
+    :param layout: partition layout from get_partition_layout()
     :param boot_label: label of the boot partition (e.g. "pmOS_boot")
     :param root_label: label of the root partition (e.g. "pmOS_root")
     :param sdcard: path to sdcard device (e.g. /dev/mmcblk0) or None
     """
-    root_dev = "/dev/installp3" if size_reserve else "/dev/installp2"
+    root_dev = f"/dev/installp{layout['root']}"
+    boot_dev = f"/dev/installp{layout['boot']}"
 
     if args.full_disk_encryption:
         format_luks_root(args, root_dev)
         root_dev = "/dev/mapper/pm_crypt"
 
     format_and_mount_root(args, root_dev, root_label, sdcard)
-    format_and_mount_boot(args, boot_label)
+    format_and_mount_boot(args, boot_dev, boot_label)