parse.apkindex: fix typo

CI apparently enforces spellcheck for comments now, so this also fixes
CI :D

.b4-config

@@ -0,0 +1,9 @@
+# Allow this repository to be used with the 'b4' tool. See
+# https://postmarketos.org/patch-review for details.
+
+[b4]
+  midmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%s
+  linkmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%%3C%s%%3E
+  send-series-to = ~postmarketos/pmbootstrap-devel@lists.sr.ht
+  send-endpoint-web = NONE
+  backend = sourcehut

.build.yml

@@ -0,0 +1,26 @@
+image: alpine/edge
+packages:
+  - sudo
+sources:
+  - https://git.sr.ht/~postmarketos/pmbootstrap
+tasks:
+  - note: |
+      pmbootstrap/.ci/note.sh
+  - shellcheck: |
+      cd pmbootstrap
+      sudo .ci/shellcheck.sh
+  - ruff: |
+      cd pmbootstrap
+      sudo .ci/ruff.sh
+  - vermin: |
+      cd pmbootstrap
+      sudo .ci/vermin.sh
+  - codespell: |
+      cd pmbootstrap
+      sudo .ci/codespell.sh
+  - pytest: |
+      cd pmbootstrap
+      sudo .ci/pytest.sh
+artifacts:
+  - ".local/var/pmbootstrap/log.txt"
+  - ".local/var/pmbootstrap/log_testsuite.txt"

.ci/codespell.sh

@@ -0,0 +1,20 @@
+#!/bin/sh -ex
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Copyright 2023 Oliver Smith
+# Description: find typos
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add \
+		py3-codespell
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+set -x
+
+# -L: words to ignore
+codespell \
+	-L crate \
+	-L hda \
+	.

.ci/note.sh

@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+printf "\n"
+printf "PROTIP: use"
+printf " \e[1;32mpmbootstrap ci\e[0m"
+printf " to run these scripts locally.\n"

.ci/prepare.sh

@@ -1,36 +0,0 @@
-#!/bin/sh -e
-# Install pmbootstrap depends, set up pmos user with sudo
-if [ "$(id -u)" != 0 ]; then
-	echo "ERROR: this script is meant to be executed in the gitlab-ci"
-	echo "environment only."
-	exit 1
-fi
-
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
-
-ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
-
-apk add -q \
-	git \
-	openssl \
-	py3-pytest \
-	py3-pytest-cov \
-	sudo
-
-adduser -D pmos
-chown -R pmos:pmos .
-echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-su pmos -c "git config --global user.email postmarketos-ci@localhost"
-su pmos -c "git config --global user.name postmarketOS_CI"
-
-echo "Initializing pmbootstrap"
-if ! su pmos -c "yes '' | pmbootstrap \
-		--details-to-stdout \
-		init \
-		>/tmp/pmb_init 2>&1"; then
-	cat /tmp/pmb_init
-	exit 1
-fi
-echo ""

.ci/pytest.sh

@@ -1,6 +1,39 @@
 #!/bin/sh -e
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
+# Description: run pmbootstrap python testsuite
+# Options: native slow
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add \
+		git \
+		openssl \
+		py3-pytest \
+		py3-pytest-cov \
+		sudo
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+# Require pytest to be installed on the host system
+if [ -z "$(command -v pytest)" ]; then
+	echo "ERROR: pytest command not found, make sure it is in your PATH."
+	exit 1
+fi
+
+# Use pytest-cov if it is installed to display code coverage
+cov_arg=""
+if python -c "import pytest_cov" >/dev/null 2>&1; then
+	cov_arg="--cov=pmb"
+fi
+
+echo "Initializing pmbootstrap..."
+if ! yes '' | ./pmbootstrap.py \
+		--details-to-stdout \
+		init \
+		>/tmp/pmb_init 2>&1; then
+	cat /tmp/pmb_init
+	exit 1
+fi
 
 # Make sure that the work folder format is up to date, and that there are no
 # mounts from aborted test cases (#1595)
@@ -21,4 +54,19 @@ if ! [ -e "$deviceinfo" ]; then
 	exit 1
 fi
 
-pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"
+# Make sure pmaports is clean, some of the tests will fail otherwise
+if [ -n "$(git -C "$pmaports" status --porcelain)" ]; then
+	echo "ERROR: pmaports dir is not clean"
+	exit 1
+fi
+
+echo "Running pytest..."
+echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
+pytest \
+	--color=yes \
+	-vv \
+	-x \
+	$cov_arg \
+	test \
+		-m "not skip_ci" \
+		"$@"

.ci/ruff.sh

@@ -0,0 +1,19 @@
+#!/bin/sh -e
+# Description: lint all python scripts
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add ruff
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+set -x
+
+# __init__.py with additional ignore:
+# F401: imported, but not used
+# shellcheck disable=SC2046
+ruff --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py')
+
+# Check all other files
+ruff  --exclude=__init__.py .

.ci/shellcheck.sh

@@ -0,0 +1,15 @@
+#!/bin/sh -e
+# Description: lint all shell scripts
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add shellcheck
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+find . -name '*.sh' |
+while read -r file; do
+	echo "shellcheck: $file"
+	shellcheck "$file"
+done

.ci/vermin.sh

@@ -1,17 +1,23 @@
 #!/bin/sh -e
-_vermin() {
-	if ! vermin -q "$@" >/dev/null 2>&1; then
-		vermin -vv "$@"
-	fi
-}
+# Description: verify that we don't use too new python features
+# https://postmarketos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add vermin
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
 
 # shellcheck disable=SC2046
-_vermin \
-	-t=3.6- \
+vermin \
+	-t=3.7- \
 	--backport argparse \
 	--backport configparser \
 	--backport enum \
 	--backport typing \
+	--lint \
+	--no-parse-comments \
+	--eval-annotations \
 	$(find . -name '*.py' \
 		-a -not -path "./.venv/*" \
 		-a -not -path "./venv/*")

.gitlab-ci.yml

@@ -1,66 +0,0 @@
----
-# Author: Clayton Craft <clayton@craftyguy.net>
-
-image: alpine:latest
-
-# defaults for "only"
-# We need to run the CI jobs in a "merge request specific context", if CI is
-# running in a merge request. Otherwise the environment variable that holds the
-# merge request ID is not available. This means, we must set the "only"
-# variable accordingly - and if we only do it for one job, all other jobs will
-# not get executed. So have the defaults here, and use them in all jobs that
-# should run on both the master branch, and in merge requests.
-# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
-.only-default: &only-default
-  only:
-    - master
-    - merge_requests
-    - tags
-
-static-code-analysis:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c "test/static_code_analysis.sh"
-
-vermin:
-  image: alpine:latest
-  <<: *only-default
-  before_script:
-  - "apk -q add py3-pip"
-  - "pip3 -q --disable-pip-version-check install vermin"
-  script:
-  - ".ci/vermin.sh"
-
-# MR settings
-# (Checks for "Allow commits from members who can merge to the target branch")
-mr-settings:
-  only:
-    - merge_requests
-  before_script:
-    - apk -q add python3
-  script:
-    - wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
-    - python3 ./check_mr_settings.py
-
-pytest:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c .ci/pytest.sh
-  after_script:
-     # Move logs so it can be saved as artifacts
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
-    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
-    - "dmesg > $CI_PROJECT_DIR/dmesg.txt"
-  artifacts:
-    when: always
-    paths:
-      - "log.txt"
-      - "log_testsuite.txt"
-      - "dmesg.txt"
-      - "pmbootstrap.cfg"
-    expire_in: 1 week

README.md

@@ -1,22 +1,62 @@
 # pmbootstrap
-[**Introduction**](https://postmarketos.org/blog/2017/05/26/intro/) | [**Security Warning**](https://ollieparanoid.github.io/post/security-warning/) | [**Devices**](https://wiki.postmarketos.org/wiki/Devices)
 
-Sophisticated chroot/build/flash tool to develop and install [postmarketOS](https://postmarketos.org).
+Sophisticated chroot/build/flash tool to develop and install
+[postmarketOS](https://postmarketos.org).
 
-Package build scripts live in the [`pmaports`](https://gitlab.com/postmarketOS/pmaports) repository now.
+## Development
+
+pmbootstrap is being developed on SourceHut
+([what](https://postmarketos.org/blog/2022/07/25/considering-sourcehut/)):
+
+https://git.sr.ht/~postmarketos/pmbootstrap
+
+Send patches via mail or web UI to
+[pmbootstrap-devel](https://lists.sr.ht/~postmarketos/pmbootstrap-devel)
+([subscribe](mailto:~postmarketos/pmbootstrap-devel+subscribe@lists.sr.ht)):
+```
+~postmarketos/pmbootstrap-devel@lists.sr.ht
+```
+
+You can set the default values for sending email in the git checkout
+```
+$ git config sendemail.to "~postmarketos/pmbootstrap-devel@lists.sr.ht"
+$ git config format.subjectPrefix "PATCH pmbootstrap"
+```
+
+Run CI scripts locally with:
+```
+$ pmbootstrap ci
+```
+
+Run a single test file:
+```
+$ pytest -vv ./test/test_keys.py
+```
+
+## Issues
+
+Issues are being tracked
+[here](https://gitlab.com/postmarketOS/pmbootstrap/-/issues).
 
 ## Requirements
-* 2 GB of RAM recommended for compiling
-* Linux distribution on the host system (`x86`, `x86_64`, or `aarch64`)
-  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
-  * Kernels based on the grsec patchset [do **not** work](https://github.com/postmarketOS/pmbootstrap/issues/107)
+* Linux distribution on the host system (`x86`, `x86_64`, `aarch64` or `armv7`)
+  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+    does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
   * [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
-* Python 3.6+
+* Python 3.7+
 * OpenSSL
 * git
+* ps
+* tar
 
 ## Usage Examples
-Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for in-depth coverage of topics such as [porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The help output (`pmbootstrap -h`) has detailed usage instructions for every command. Read on for some generic examples of what can be done with `pmbootstrap`.
+Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for
+in-depth coverage of topics such as
+[porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device)
+or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The
+help output (`pmbootstrap -h`) has detailed usage instructions for every
+command. Read on for some generic examples of what can be done with
+`pmbootstrap`.
 
 ### Installing pmbootstrap
 <https://wiki.postmarketos.org/wiki/Installing_pmbootstrap>
@@ -63,6 +103,27 @@ Generate a template for a new package:
 $ pmbootstrap newapkbuild "https://gitlab.com/postmarketOS/osk-sdl/-/archive/0.52/osk-sdl-0.52.tar.bz2"
 ```
 
+#### Default architecture
+
+Packages will be compiled for the architecture of the device running
+pmbootstrap by default. For example, if your `x86_64` PC runs pmbootstrap, it
+would build a package for `x86_64` with this command:
+```
+$ pmbootstrap build hello-world
+```
+
+If you would rather build for the target device selected in `pmbootstrap init`
+by default, then use the `build_default_device_arch` option:
+```
+$ pmbootstrap config build_default_device_arch True
+```
+
+If your target device is `pine64-pinephone` for example, pmbootstrap will now
+build this package for `aarch64`:
+```
+$ pmbootstrap build hello-world
+```
+
 ### Chroots
 Enter the `armhf` building chroot:
 ```
@@ -80,7 +141,9 @@ $ pmbootstrap zap
 ```
 
 ### Device Porting Assistance
-Analyze Android [`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also works with recovery OS images like TWRP):
+Analyze Android
+[`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also
+works with recovery OS images like TWRP):
 ```
 $ pmbootstrap bootimg_analyze ~/Downloads/twrp-3.2.1-0-fp2.img
 ```
@@ -144,12 +207,14 @@ List pmaports that don't have a binary package:
 $ pmbootstrap repo_missing --arch=armhf --overview
 ```
 
-Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps):
+Increase the `pkgrel` for each aport where the binary package has outdated
+dependencies (e.g. after soname bumps):
 ```
 $ pmbootstrap pkgrel_bump --auto
 ```
 
-Generate cross-compiler aports based on the latest version from Alpine's aports:
+Generate cross-compiler aports based on the latest version from Alpine's
+aports:
 ```
 $ pmbootstrap aportgen binutils-armhf gcc-armhf
 ```
@@ -190,27 +255,24 @@ $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hel
 $ pmbootstrap stats --arch=armhf
 ```
 
-`distccd` log:
-```
-$ pmbootstrap log_distccd
-```
-
 ### Use alternative sudo
 
 pmbootstrap supports `doas` and `sudo`.
 If multiple sudo implementations are installed, pmbootstrap will use `doas`.
-You can set the `PMB_SUDO` environmental variable to define the sudo implementation you want to use.
+You can set the `PMB_SUDO` environmental variable to define the sudo
+implementation you want to use.
 
 ### Select SSH keys to include and make authorized in new images
 
-If the config file option `ssh_keys` is set to `True` (it defaults to `False`), then all files
-matching the glob `~/.ssh/id_*.pub` will be placed in `~/.ssh/authorized_keys` in the user's
-home directory in newly-built images.
+If the config file option `ssh_keys` is set to `True` (it defaults to `False`),
+then all files matching the glob `~/.ssh/id_*.pub` will be placed in
+`~/.ssh/authorized_keys` in the user's home directory in newly-built images.
 
-Sometimes, for example if you have a large number of SSH keys, you may wish to select a
-different set of public keys to include in an image. To do this, set the `ssh_key_glob`
-configuration parameter in the pmbootstrap config file to a string containing a glob that is to
-match the file or files you wish to include.
+Sometimes, for example if you have a large number of SSH keys, you may wish to
+select a different set of public keys to include in an image. To do this, set
+the `ssh_key_glob` configuration parameter in the pmbootstrap config file to a
+string containing a glob that is to match the file or files you wish to
+include.
 
 For example, a `~/.config/pmbootstrap.cfg` may contain:
 
@@ -220,39 +282,5 @@ For example, a `~/.config/pmbootstrap.cfg` may contain:
     ssh_key_glob = ~/.ssh/postmarketos-dev.pub
     # ...
 
-## Development
-### Requirements for running tests
-* [Shellcheck](https://shellcheck.net/)
-
-You also need to install the following python packages (pip can be useful if you distribution hasn't got them packaged):
-* `pytest`
-* `pytest-cov`
-* `flake8`
-
-On Alpine Linux it can be done with:
-```shell
-$ sudo apk add grep shellcheck py3-pytest py3-pytest-cov py3-flake8
-```
-
-### Running linters
-The easiest way is to run the same script CI runs:
-```shell
-$ ./test/static_code_analysis.sh
-```
-
-### Running tests
-You can now run `pytest -vv` inside the pmbootstrap folder to run all available tests.
-
-CI runs slightly reduces set of tests (it skips tests that require running qemu) by this:
-```shell
-$ .ci/pytest.sh
-```
-This is the easiest way to do the same as CI.
-
-Alternatively you can run a single test file if you wish:
-```shell
-$ pytest -vv ./test/test_keys.py
-```
-
 ## License
 [GPLv3](LICENSE)

helpers/envkernel.sh

@@ -49,7 +49,7 @@ export_pmbootstrap_dir() {
 	# See also: <https://stackoverflow.com/a/29835459>
 	# shellcheck disable=SC3054
 	if [ -n "${BASH_SOURCE[0]}" ]; then
-		script_dir="$(dirname "${BASH_SOURCE[0]}")"
+		script_dir="$(dirname "$(realpath "$BASH_SOURCE")")"
 	else
 		script_dir="$(dirname "$1")"
 	fi
@@ -121,7 +121,8 @@ initialize_chroot() {
 	arch_substr="${host_arch:0:3}"
 	if [ "$arch" = "$host_arch" ] || \
 		{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; } || \
-		{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; }; then
+		{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; } || \
+		{ [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; }; then
 		need_cross_compiler=0
 	fi
 
@@ -168,7 +169,9 @@ initialize_chroot() {
 		musl-dev \
 		ncurses-dev \
 		perl \
+		py3-dt-schema \
 		sed \
+		yamllint \
 		yaml-dev \
 		xz || return 1
 
@@ -216,6 +219,10 @@ set_alias_make() {
 		cross_compiler="/usr/bin/$prefix-"
 	fi
 
+	if [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; then
+		cc=$hostcc
+	fi
+
 	# Build make command
 	cmd="echo '*** pmbootstrap envkernel.sh active for $PWD! ***';"
 	cmd="$cmd pmbootstrap -q chroot --user --"
@@ -308,7 +315,7 @@ set_reactivate() {
 
 check_and_deactivate() {
 	if [ "$POSTMARKETOS_ENVKERNEL_ENABLED" = 1 ]; then
-		# we already are runnning in envkernel
+		# we already are running in envkernel
 		deactivate
 	fi
 }
@@ -317,9 +324,7 @@ check_and_deactivate() {
 print_usage() {
 	# shellcheck disable=SC3054
 	if [ -n "${BASH_SOURCE[0]}" ]; then
-		echo "usage: source $(basename "${BASH_SOURCE[0]}")"
-	else
-		echo "usage: source $(basename "$1")"
+		echo "usage: source $(basename "$(realpath "$BASH_SOURCE")")"
 	fi
 	echo "optional arguments:"
 	echo "    --fish        Print fish alias syntax (internally used)"
@@ -405,7 +410,7 @@ main() {
 
 # Print fish alias syntax (when called from envkernel.fish)
 fish_compat() {
-	[ "$1" = "--fish" ] || return
+	[ "$1" = "--fish" ] || return 0
 	for name in make kernelroot pmbootstrap pmbroot; do
 		echo "alias $(alias $name | sed 's/=/ /')"
 	done

pmb/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 # PYTHON_ARGCOMPLETE_OK
 import sys
@@ -14,6 +14,17 @@ from .helpers import logging as pmb_logging
 from .helpers import mount
 from .helpers import other
 
+# pmbootstrap version
+__version__ = "2.0.0"
+
+# Python version check
+version = sys.version_info
+if version < (3, 7):
+    print("You need at least Python 3.7 to run pmbootstrap")
+    print("(You are running it with Python " + str(version.major) +
+          "." + str(version.minor) + ")")
+    sys.exit()
+
 
 def main():
     # Wrap everything to display nice error messages
@@ -23,6 +34,9 @@ def main():
         args = parse.arguments()
         os.umask(0o22)
 
+        # Store script invocation command
+        os.environ["PMBOOTSTRAP_CMD"] = sys.argv[0]
+
         # Sanity checks
         other.check_grsec()
         if not args.as_root and os.geteuid() == 0:
@@ -56,6 +70,10 @@ def main():
                          " shutdown' as necessary)")
         logging.info("DONE!")
 
+    except KeyboardInterrupt:
+        print("\nCaught KeyboardInterrupt, exiting …")
+        sys.exit(130)  # SIGINT(2) + 128
+
     except Exception as e:
         # Dump log to stdout when args (and therefore logging) init failed
         if not args:
@@ -71,7 +89,14 @@ def main():
             log_hint += (" Alternatively you can use '--details-to-stdout' to"
                          " get more output, e.g. 'pmbootstrap"
                          " --details-to-stdout init'.")
+        print()
         print(log_hint)
+        print()
+        print("Before you report this error, ensure that pmbootstrap is "
+              "up to date.")
+        print("Find the latest version here:"
+              " https://git.sr.ht/~postmarketos/pmbootstrap/refs")
+        print(f"Your version: {__version__}")
         return 1
 
 

pmb/aportgen/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/aportgen/binutils.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.helpers.git
@@ -13,9 +13,14 @@ def generate(args, pkgname):
 
     # Rewrite APKBUILD
     fields = {
-        "pkgname": pkgname,
-        "pkgdesc": f"Tools necessary to build programs for {arch} targets",
         "arch": pmb.config.arch_native,
+        "makedepends_host": "zlib-dev jansson-dev zstd-dev",
+        "pkgdesc": f"Tools necessary to build programs for {arch} targets",
+        "pkgname": pkgname,
+    }
+
+    replace_simple = {
+        "*--with-bugurl=*": "\t\t--with-bugurl=\"https://postmarketos.org/issues\" \\"
     }
 
     below_header = """
@@ -24,4 +29,5 @@ def generate(args, pkgname):
     """
 
     pmb.aportgen.core.rewrite(args, pkgname, "main/binutils", fields,
-                              "binutils", below_header=below_header)
+                              "binutils", replace_simple=replace_simple,
+                              below_header=below_header)

pmb/aportgen/busybox_static.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -67,7 +67,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/core.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fnmatch
 import logging
@@ -205,16 +205,18 @@ def get_upstream_aport(args, pkgname, arch=None):
     if compare == 0:
         return aport_path
 
-    # Different version message
-    logging.error("ERROR: Package '" + pkgname + "' has a different version in"
+    # APKBUILD > binary: this is fine
+    if compare == 1:
+        logging.info(f"NOTE: {pkgname} {arch} binary package has a lower"
+                     f" version {package['version']} than the APKBUILD"
+                     f" {apkbuild_version}")
+        return aport_path
+
+    # APKBUILD < binary: aports.git is outdated
+    logging.error("ERROR: Package '" + pkgname + "' has a lower version in"
                   " local checkout of Alpine's aports (" + apkbuild_version +
                   ") compared to Alpine's binary package (" +
                   package["version"] + ")!")
 
-    # APKBUILD < binary
-    if compare == -1:
-        raise RuntimeError("You can update your local checkout with: "
-                           "'pmbootstrap pull'")
-    # APKBUILD > binary
-    raise RuntimeError("You can force an update of your binary package"
-                       " APKINDEX files with: 'pmbootstrap update'")
+    raise RuntimeError("You can update your local checkout with: "
+                       "'pmbootstrap pull'")

pmb/aportgen/device.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -16,7 +16,7 @@ def ask_for_architecture():
         architectures.remove("armhf")
     while True:
         ret = pmb.helpers.cli.ask("Device architecture", architectures,
-                                  architectures[0], complete=architectures)
+                                  "aarch64", complete=architectures)
         if ret in architectures:
             return ret
         logging.fatal("ERROR: Invalid architecture specified. If you want to"
@@ -73,7 +73,8 @@ def ask_for_flash_method():
         logging.info("Which flash method does the device support?")
         method = pmb.helpers.cli.ask("Flash method",
                                      pmb.config.flash_methods,
-                                     pmb.config.flash_methods[0])
+                                     "none",
+                                     complete=pmb.config.flash_methods)
 
         if method in pmb.config.flash_methods:
             if method == "heimdall":
@@ -178,7 +179,6 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
         deviceinfo_codename="{codename}"
         deviceinfo_year="{year}"
         deviceinfo_dtb=""
-        deviceinfo_modules_initfs=""
         deviceinfo_arch="{arch}"
 
         # Device related
@@ -194,13 +194,13 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
 
     content_heimdall_bootimg = """\
         deviceinfo_flash_heimdall_partition_kernel=""
-        deviceinfo_flash_heimdall_partition_system=""
+        deviceinfo_flash_heimdall_partition_rootfs=""
         """
 
     content_heimdall_isorec = """\
         deviceinfo_flash_heimdall_partition_kernel=""
         deviceinfo_flash_heimdall_partition_initfs=""
-        deviceinfo_flash_heimdall_partition_system=""
+        deviceinfo_flash_heimdall_partition_rootfs=""
         """
 
     content_0xffff = """\
@@ -231,6 +231,24 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
             handle.write(line.lstrip() + "\n")
 
 
+def generate_modules_initfs(args):
+    content = """\
+    # Remove this comment after reading, or the file if unnecessary (CHANGEME!)
+    # This file can contain a list of modules to be included in the initramfs,
+    # so that they are available in early boot stages. It should have one
+    # module name per line. If there are multiple kernel variants with different
+    # requirements for modules into the initramfs, one modules-initfs.$variant
+    # file should be created for each of them.
+    """
+
+    # Write to file
+    pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"])
+    path = args.work + "/aportgen/modules-initfs"
+    with open(path, "w", encoding="utf-8") as handle:
+        for line in content.rstrip().split("\n"):
+            handle.write(line.lstrip() + "\n")
+
+
 def generate_apkbuild(args, pkgname, name, arch, flash_method):
     # Dependencies
     depends = ["postmarketos-base",
@@ -239,7 +257,6 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
         depends.append("mkbootimg")
     if flash_method == "0xffff":
         depends.append("uboot-tools")
-    depends.append("mesa-dri-gallium")
 
     # Whole APKBUILD
     depends.sort()
@@ -258,7 +275,10 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
             {depends}
         "
         makedepends="devicepkg-dev"
-        source="deviceinfo"
+        source="
+            deviceinfo
+            modules-initfs
+        "
 
         build() {{
             devicepkg_build $startdir $pkgname
@@ -295,4 +315,5 @@ def generate(args, pkgname):
     generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
                         chassis, has_keyboard, has_external_storage,
                         flash_method, bootimg)
+    generate_modules_initfs(args)
     generate_apkbuild(args, pkgname, name, arch, flash_method)

pmb/aportgen/gcc.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.helpers.git

pmb/aportgen/grub_efi.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Nick Reitemeyer, Oliver Smith
+# Copyright 2023 Nick Reitemeyer, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -57,7 +57,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/linux.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.helpers.run
 import pmb.aportgen.core
@@ -10,7 +10,7 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
     device = "-".join(pkgname.split("-")[1:])
     carch = pmb.parse.arch.alpine_to_kernel(deviceinfo["arch"])
 
-    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "flex",
+    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex",
                    "openssl-dev", "perl"]
 
     build = """

pmb/aportgen/musl.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -93,7 +93,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/build/__init__.py

@@ -1,8 +1,8 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-from pmb.build.init import init, init_compiler
+from pmb.build.init import init, init_abuild_minimal, init_compiler
 from pmb.build.envkernel import package_kernel
-from pmb.build.menuconfig import menuconfig
+from pmb.build.kconfig import menuconfig
 from pmb.build.newapkbuild import newapkbuild
 from pmb.build.other import copy_to_buildpath, is_necessary, \
     index_repo

pmb/build/_package.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import logging
@@ -8,7 +8,6 @@ import pmb.build
 import pmb.build.autodetect
 import pmb.chroot
 import pmb.chroot.apk
-import pmb.chroot.distccd
 import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.parse
@@ -190,7 +189,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     just initialized the build environment for nothing) and then setup the
     whole build environment (abuild, gcc, dependencies, cross-compiler).
 
-    :param cross: None, "native", "distcc", or "crossdirect"
+    :param cross: None, "native", or "crossdirect"
     :param skip_init_buildenv: can be set to False to avoid initializing the
                                build environment. Use this when building
                                something during initialization of the build
@@ -214,7 +213,10 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     if not skip_init_buildenv:
         pmb.build.init(args, suffix)
         pmb.build.other.configure_abuild(args, suffix)
-        pmb.build.other.configure_ccache(args, suffix)
+        if args.ccache:
+            pmb.build.other.configure_ccache(args, suffix)
+            if "rust" in depends or "cargo" in depends:
+                pmb.chroot.apk.install(args, ["sccache"], suffix)
     if not strict and "pmb:strict" not in apkbuild["options"] and len(depends):
         pmb.chroot.apk.install(args, depends, suffix)
     if src:
@@ -223,35 +225,12 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     # Cross-compiler init
     if cross:
         pmb.build.init_compiler(args, depends, cross, arch)
-    if cross == "distcc":
-        pmb.chroot.distccd.start(args, arch)
     if cross == "crossdirect":
         pmb.chroot.mount_native_into_foreign(args, suffix)
-        # Workaround: this specific version currently in pmaports.git master
-        # was built with !tracedeps, so it doesn't pull in the isl dependency
-        # and we need to install it manually. Doing this is easier than bumping
-        # the pkgrel and going out of sync with Alpine's gcc package. This
-        # workaround can be removed once a newer gcc is in Alpine and we
-        # rebuild our cross gcc based on the new APKBUILD. See pmaports#1732.
-        if get_gcc_version(args, arch) == "12.2.1_git20220924-r2":
-            pmb.chroot.apk.install(args, ["isl25"], build=False)
 
     return True
 
 
-def get_gcc_version(args, arch):
-    """
-    Get the GCC version for a specific arch from parsing the right APKINDEX.
-    We feed this to ccache, so it knows the right GCC version, when
-    cross-compiling in a foreign arch chroot with distcc. See the "using
-    ccache with other compiler wrappers" section of their man page:
-    <https://linux.die.net/man/1/ccache>
-    :returns: a string like "6.4.0-r5"
-    """
-    return pmb.parse.apkindex.package(args, "gcc-" + arch,
-                                      pmb.config.arch_native)["version"]
-
-
 def get_pkgver(original_pkgver, original_source=False, now=None):
     """
     Get the original pkgver when using the original source. Otherwise, get the
@@ -284,7 +263,7 @@ def override_source(args, apkbuild, pkgver, src, suffix="native"):
         return
 
     # Mount source in chroot
-    mount_path = "/mnt/pmbootstrap-source-override/"
+    mount_path = "/mnt/pmbootstrap/source-override/"
     mount_path_outside = args.work + "/chroot_" + suffix + mount_path
     pmb.helpers.mount.bind(args, src, mount_path_outside, umount=True)
 
@@ -389,7 +368,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
     depending on the cross-compiler method and target architecture), copy
     the aport to the chroot and execute abuild.
 
-    :param cross: None, "native", "distcc", or "crossdirect"
+    :param cross: None, "native", or "crossdirect"
     :param src: override source used to build the package with a local folder
     :returns: (output, cmd, env), output is the destination apk path relative
               to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
@@ -418,24 +397,28 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
         hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
         env["CROSS_COMPILE"] = hostspec + "-"
         env["CC"] = hostspec + "-gcc"
-    if cross == "distcc":
-        env["CCACHE_PREFIX"] = "distcc"
-        env["CCACHE_PATH"] = f"/usr/lib/arch-bin-masquerade/{arch}:/usr/bin"
-        env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
-        env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
-        env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
-                             args.port_distccd)
-        env["DISTCC_BACKOFF_PERIOD"] = "0"
-        if not args.distcc_fallback:
-            env["DISTCC_FALLBACK"] = "0"
-        if args.verbose:
-            env["DISTCC_VERBOSE"] = "1"
     if cross == "crossdirect":
         env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
                                 pmb.config.chroot_path])
     if not args.ccache:
         env["CCACHE_DISABLE"] = "1"
 
+    # Use sccache without crossdirect (crossdirect uses it via rustc.sh)
+    if args.ccache and cross != "crossdirect":
+        env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
+
+    # Cache binary objects from go in this path (like ccache)
+    env["GOCACHE"] = "/home/pmos/.cache/go-build"
+
+    # Cache go modules (git repositories). Usually these should be bundled and
+    # it should not be required to download them at build time, in that case
+    # the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But
+    # e.g. when using --src they are not bundled, in that case it makes sense
+    # to point GOMODCACHE at pmbootstrap's work dir so the modules are only
+    # downloaded once.
+    if args.go_mod_cache:
+        env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
+
     # Build the abuild command
     cmd = ["abuild", "-D", "postmarketOS"]
     if strict or "pmb:strict" in apkbuild["options"]:

pmb/build/autodetect.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -40,7 +40,7 @@ def arch(args, pkgname):
     :returns: arch string like "x86_64" or "armhf". Preferred order, depending
               on what is supported by the APKBUILD:
               * native arch
-              * device arch
+              * device arch (this will be preferred instead if build_default_device_arch is true)
               * first arch in the APKBUILD
     """
     aport = pmb.helpers.pmaports.find(args, pkgname)
@@ -50,14 +50,19 @@ def arch(args, pkgname):
 
     apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
     arches = apkbuild["arch"]
-    if ("noarch" in arches or
-            "all" in arches or
-            pmb.config.arch_native in arches):
-        return pmb.config.arch_native
 
-    arch_device = args.deviceinfo["arch"]
-    if arch_device in arches:
-        return arch_device
+    if args.build_default_device_arch:
+        preferred_arch = args.deviceinfo["arch"]
+        preferred_arch_2nd = pmb.config.arch_native
+    else:
+        preferred_arch = pmb.config.arch_native
+        preferred_arch_2nd = args.deviceinfo["arch"]
+
+    if "noarch" in arches or "all" in arches or preferred_arch in arches:
+        return preferred_arch
+
+    if preferred_arch_2nd in arches:
+        return preferred_arch_2nd
 
     try:
         return apkbuild["arch"][0]
@@ -77,7 +82,7 @@ def suffix(apkbuild, arch):
 
 def crosscompile(args, apkbuild, arch, suffix):
     """
-        :returns: None, "native", "crossdirect" or "distcc"
+        :returns: None, "native", "crossdirect"
     """
     if not args.cross:
         return None
@@ -85,6 +90,6 @@ def crosscompile(args, apkbuild, arch, suffix):
         return None
     if suffix == "native":
         return "native"
-    if args.no_crossdirect or "!pmb:crossdirect" in apkbuild["options"]:
-        return "distcc"
+    if "!pmb:crossdirect" in apkbuild["options"]:
+        return None
     return "crossdirect"

pmb/build/checksum.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 
@@ -10,7 +10,7 @@ import pmb.helpers.pmaports
 
 def update(args, pkgname):
     """ Fetch all sources and update the checksums in the APKBUILD. """
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) generate checksums for " + pkgname)
     pmb.chroot.user(args, ["abuild", "checksum"],
@@ -24,7 +24,7 @@ def update(args, pkgname):
 
 def verify(args, pkgname):
     """ Fetch all sources and verify their checksums. """
-    pmb.build.init(args)
+    pmb.build.init_abuild_minimal(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) verify checksums for " + pkgname)
 

pmb/build/envkernel.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Robert Yang
+# Copyright 2023 Robert Yang
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -104,6 +104,23 @@ def modify_apkbuild(args, pkgname, aport):
     pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields)
 
 
+def host_build_bindmount(args, chroot, flag_file, mount=False):
+    """
+    Check if the bind mount already exists and unmount it.
+    Then bindmount the current directory into the chroot as
+    /mnt/linux so it can be used by the envkernel abuild wrapper
+    """
+    flag_path = f"{chroot}/{flag_file}"
+    if os.path.exists(flag_path):
+        logging.info("Cleaning up kernel sources bind-mount")
+        pmb.helpers.run.root(args, ["umount", chroot + "/mnt/linux"], check=False)
+        pmb.helpers.run.root(args, ["rm", flag_path])
+
+    if mount:
+        pmb.helpers.mount.bind(args, ".", f"{chroot}/mnt/linux")
+        pmb.helpers.run.root(args, ["touch", flag_path])
+
+
 def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     """
     Prepare build environment and run abuild.
@@ -117,10 +134,30 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     build_path = "/home/pmos/build"
     kbuild_out_source = "/mnt/linux/.output"
 
+    # If the kernel was cross-compiled on the host rather than with the envkernel
+    # helper, we can still use the envkernel logic to package the artifacts for
+    # development, making it easy to quickly sideload a new kernel or pmbootstrap
+    # to create a boot image
+    # This handles bind mounting the current directory (assumed to be kernel sources)
+    # into the chroot so we can run abuild against it for the currently selected
+    # devices kernel package.
+    flag_file = "envkernel-bind-mounted"
+    host_build = False
+
+    if not pmb.helpers.mount.ismount(chroot + "/mnt/linux"):
+        logging.info("envkernel.sh hasn't run, assuming the kernel was cross compiled"
+                     "on host and using current dir as source")
+        host_build = True
+
+    host_build_bindmount(args, chroot, flag_file, mount=host_build)
+
     if not os.path.exists(chroot + kbuild_out_source):
-        raise RuntimeError("No '.output' dir found in your kernel source dir."
-                           "Compile the " + args.device + " kernel with "
-                           "envkernel.sh first, then try again.")
+        raise RuntimeError("No '.output' dir found in your kernel source dir. "
+                           "Compile the " + args.device + " kernel first and "
+                           "then try again. See https://postmarketos.org/envkernel"
+                           "for details. If building on your host and only using "
+                           "--envkernel for packaging, make sure you have O=.output "
+                           "as an argument to make.")
 
     # Create working directory for abuild
     pmb.build.copy_to_buildpath(args, pkgname)
@@ -147,6 +184,9 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     cmd = ["abuild", "rootpkg"]
     pmb.chroot.user(args, cmd, working_dir=build_path, env=env)
 
+    # Clean up bindmount if needed
+    host_build_bindmount(args, chroot, flag_file)
+
     # Clean up symlinks
     if build_output != "":
         if os.path.islink(chroot + "/mnt/linux/" + build_output) and \
@@ -183,6 +223,8 @@ def package_kernel(args):
     depends, _ = pmb.build._package.build_depends(
         args, apkbuild, pmb.config.arch_native, strict=False)
     pmb.build.init(args, suffix)
+    if pmb.parse.arch.cpu_emulation_required(arch):
+        depends.append("binutils-" + arch)
     pmb.chroot.apk.install(args, depends, suffix)
 
     output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +

pmb/build/init.py

@@ -1,8 +1,9 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import os
-import logging
 import glob
+import logging
+import os
+import pathlib
 
 import pmb.build
 import pmb.config
@@ -11,15 +12,14 @@ import pmb.chroot.apk
 import pmb.helpers.run
 
 
-def init(args, suffix="native"):
-    # Check if already initialized
-    marker = "/var/local/pmbootstrap_chroot_build_init_done"
-    if os.path.exists(args.work + "/chroot_" + suffix + marker):
+def init_abuild_minimal(args, suffix="native"):
+    """ Initialize a minimal chroot with abuild where one can do
+        'abuild checksum'. """
+    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_abuild_init_done"
+    if os.path.exists(marker):
         return
 
-    # Initialize chroot, install packages
-    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
-                           build=False)
+    pmb.chroot.apk.install(args, ["abuild"], suffix, build=False)
 
     # Fix permissions
     pmb.chroot.root(args, ["chown", "root:abuild",
@@ -27,6 +27,24 @@ def init(args, suffix="native"):
     pmb.chroot.root(args, ["chmod", "g+w",
                            "/var/cache/distfiles"], suffix)
 
+    # Add user to group abuild
+    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
+
+    pathlib.Path(marker).touch()
+
+
+def init(args, suffix="native"):
+    """ Initialize a chroot for building packages with abuild. """
+    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_build_init_done"
+    if os.path.exists(marker):
+        return
+
+    init_abuild_minimal(args, suffix)
+
+    # Initialize chroot, install packages
+    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
+                           build=False)
+
     # Generate package signing keys
     chroot = args.work + "/chroot_" + suffix
     if not os.path.exists(args.work + "/config_abuild/abuild.conf"):
@@ -36,7 +54,7 @@ def init(args, suffix="native"):
 
         # Copy package signing key to /etc/apk/keys
         for key in glob.glob(chroot +
-                             "/mnt/pmbootstrap-abuild-config/*.pub"):
+                             "/mnt/pmbootstrap/abuild-config/*.pub"):
             key = key[len(chroot):]
             pmb.chroot.root(args, ["cp", key, "/etc/apk/keys/"], suffix)
 
@@ -63,9 +81,6 @@ def init(args, suffix="native"):
                                "/usr/local/bin/gzip"], suffix)
         pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix)
 
-    # Add user to group abuild
-    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
-
     # abuild.conf: Don't clean the build folder after building, so we can
     # inspect it afterwards for debugging
     pmb.chroot.root(args, ["sed", "-i", "-e", "s/^CLEANUP=.*/CLEANUP=''/",
@@ -77,8 +92,7 @@ def init(args, suffix="native"):
                            "s/^ERROR_CLEANUP=.*/ERROR_CLEANUP=''/",
                            "/etc/abuild.conf"], suffix)
 
-    # Mark the chroot as initialized
-    pmb.chroot.root(args, ["touch", marker], suffix)
+    pathlib.Path(marker).touch()
 
 
 def init_compiler(args, depends, cross, arch):
@@ -94,6 +108,11 @@ def init_compiler(args, depends, cross, arch):
     if cross == "crossdirect":
         cross_pkgs += ["crossdirect"]
         if "rust" in depends or "cargo" in depends:
-            cross_pkgs += ["rust"]
+            if args.ccache:
+                cross_pkgs += ["sccache"]
+            # crossdirect for rust installs all build dependencies in the
+            # native chroot too, as some of them can be required for building
+            # native macros / build scripts
+            cross_pkgs += depends
 
     pmb.chroot.apk.install(args, cross_pkgs)

pmb/build/kconfig.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -78,6 +78,16 @@ def get_outputdir(args, pkgname, apkbuild):
                        " template with: pmbootstrap aportgen " + pkgname)
 
 
+def extract_and_patch_sources(args, pkgname, arch):
+    pmb.build.copy_to_buildpath(args, pkgname)
+    logging.info("(native) extract kernel source")
+    pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
+    logging.info("(native) apply patches")
+    pmb.chroot.user(args, ["abuild", "prepare"], "native",
+                    "/home/pmos/build", output="interactive",
+                    env={"CARCH": arch})
+
+
 def menuconfig(args, pkgname, use_oldconfig):
     # Pkgname: allow omitting "linux-" prefix
     if not pkgname.startswith("linux-"):
@@ -119,14 +129,10 @@ def menuconfig(args, pkgname, use_oldconfig):
     if copy_xauth:
         pmb.chroot.other.copy_xauthority(args)
 
-    # Patch and extract sources
-    pmb.build.copy_to_buildpath(args, pkgname)
-    logging.info("(native) extract kernel source")
-    pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
-    logging.info("(native) apply patches")
-    pmb.chroot.user(args, ["abuild", "prepare"], "native",
-                    "/home/pmos/build", output="interactive",
-                    env={"CARCH": arch})
+    extract_and_patch_sources(args, pkgname, arch)
+
+    # Check for background color variable
+    color = os.environ.get("MENUCONFIG_COLOR")
 
     # Run make menuconfig
     outputdir = get_outputdir(args, pkgname, apkbuild)
@@ -137,6 +143,8 @@ def menuconfig(args, pkgname, use_oldconfig):
     if cross:
         env["CROSS_COMPILE"] = f"{hostspec}-"
         env["CC"] = f"{hostspec}-gcc"
+    if color:
+        env["MENUCONFIG_COLOR"] = color
     pmb.chroot.user(args, ["make", kopt], "native",
                     outputdir, output="tui", env=env)
 
@@ -153,12 +161,4 @@ def menuconfig(args, pkgname, use_oldconfig):
     pmb.build.checksum.update(args, pkgname)
 
     # Check config
-    pmb.parse.kconfig.check(args, apkbuild["_flavor"],
-                            force_waydroid_check=False,
-                            force_nftables_check=False,
-                            force_containers_check=False,
-                            force_zram_check=False,
-                            force_netboot_check=False,
-                            force_community_check=False,
-                            force_uefi_check=False,
-                            details=True)
+    pmb.parse.kconfig.check(args, apkbuild["_flavor"], details=True)

pmb/build/newapkbuild.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import os

pmb/build/other.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -28,7 +28,16 @@ def copy_to_buildpath(args, package, suffix="native"):
         pmb.chroot.root(args, ["rm", "-rf", "/home/pmos/build"], suffix)
 
     # Copy aport contents with resolved symlinks
-    pmb.helpers.run.root(args, ["cp", "-rL", aport + "/", build])
+    pmb.helpers.run.root(args, ["mkdir", "-p", build])
+    for entry in os.listdir(aport):
+        # Don't copy those dirs, as those have probably been generated by running `abuild`
+        # on the host system directly and not cleaning up after itself.
+        # Those dirs might contain broken symlinks and cp fails resolving them.
+        if entry in ["src", "pkg"]:
+            logging.warn(f"WARNING: Not copying {entry}, looks like a leftover from abuild")
+            continue
+        pmb.helpers.run.root(args, ["cp", "-rL", f"{aport}/{entry}", f"{build}/{entry}"])
+
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos",
                            "/home/pmos/build"], suffix)
 
@@ -143,7 +152,7 @@ def configure_abuild(args, suffix, verify=False):
                                 suffix)
                 configure_abuild(args, suffix, True)
             return
-    raise RuntimeError("Could not find " + prefix + " line in " + path)
+    pmb.chroot.root(args, ["sed", "-i", f"$ a\\{prefix}{args.jobs}", "/etc/abuild.conf"], suffix)
 
 
 def configure_ccache(args, suffix="native", verify=False):

pmb/chroot/__init__.py

@@ -1,7 +1,7 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.chroot.init import init, init_keys
-from pmb.chroot.mount import mount, mount_native_into_foreign
+from pmb.chroot.mount import mount, mount_native_into_foreign, remove_mnt_pmbootstrap
 from pmb.chroot.root import root
 from pmb.chroot.user import user
 from pmb.chroot.user import exists as user_exists

pmb/chroot/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -143,7 +143,7 @@ def packages_get_locally_built_apks(args, packages, arch):
     :param packages: list of pkgnames
     :param arch: architecture that the locally built packages should have
     :returns: list of apk file paths that are valid inside the chroots, e.g.
-              ["/mnt/pmbootstrap-packages/x86_64/hello-world-1-r6.apk", ...]
+              ["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...]
     """
     channel = pmb.config.pmaports.read_config(args)["channel"]
     ret = []
@@ -157,7 +157,7 @@ def packages_get_locally_built_apks(args, packages, arch):
         if not os.path.exists(f"{args.work}/packages/{channel}/{arch}/{apk_file}"):
             continue
 
-        ret.append(f"/mnt/pmbootstrap-packages/{arch}/{apk_file}")
+        ret.append(f"/mnt/pmbootstrap/packages/{arch}/{apk_file}")
 
     return ret
 

pmb/chroot/apk_static.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/chroot/binfmt.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/chroot/distccd.py

@@ -1,250 +0,0 @@
-# Copyright 2022 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-import errno
-import json
-import logging
-import os
-import pmb.chroot
-import pmb.config
-import pmb.chroot.apk
-
-""" Packages for foreign architectures (e.g. armhf) get built in chroots
-    running with QEMU. While this works, it is painfully slow. So we speed it
-    up by using distcc to let cross compilers running in the native chroots do
-    the heavy lifting.
-
-    This file sets up an SSH server in the native chroot, which will then be
-    used by the foreign arch chroot to communicate with the distcc daemon. We
-    make sure that only the foreign arch chroot can connect to the sshd by only
-    listening on localhost, as well as generating dedicated ssh keys.
-
-    Using the SSH server instead of running distccd directly is a security
-    measure. Distccd does not authenticate its clients and would therefore
-    allow any process of the host system (not related to pmbootstrap) to
-    execute compilers in the native chroot. By modifying the compiler's options
-    or sending malicious data to the compiler, it is likely that the process
-    can gain remote code execution [1]. That way, a compromised, but sandboxed
-    process could gain privilege escalation.
-
-    [1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
-"""
-
-
-def init_server(args):
-    """
-    Install dependencies and generate keys for the server.
-    """
-    # Install dependencies
-    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
-                                  "openssh-server"])
-
-    # Config folder (nothing to do if existing)
-    dir = "/home/pmos/.distcc-sshd"
-    dir_outside = args.work + "/chroot_native" + dir
-    if os.path.exists(dir_outside):
-        return
-
-    # Generate keys
-    logging.info("(native) generate distcc-sshd server keys")
-    pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
-    pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
-
-
-def init_client(args, suffix):
-    """
-    Install dependencies and generate keys for the client.
-    """
-    # Install dependencies
-    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
-                                  "openssh-client"], suffix)
-
-    # Public key path (nothing to do if existing)
-    pub = "/home/pmos/id_ed25519.pub"
-    pub_outside = args.work + "/chroot_" + suffix + pub
-    if os.path.exists(pub_outside):
-        return
-
-    # Generate keys
-    logging.info("(" + suffix + ") generate distcc-sshd client keys")
-    pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
-                           "-f", "/home/pmos/.ssh/id_ed25519"], suffix)
-    pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
-                    suffix)
-
-
-def configure_authorized_keys(args, suffix):
-    """
-    Exclusively allow one foreign arch chroot to access the sshd.
-    """
-    auth = "/home/pmos/.distcc-sshd/authorized_keys"
-    auth_outside = args.work + "/chroot_native/" + auth
-    pub = "/home/pmos/id_ed25519.pub"
-    pub_outside = args.work + "/chroot_" + suffix + pub
-    pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
-
-
-def configure_cmdlist(args, arch):
-    """
-    Create a whitelist of all the cross compiler wrappers.
-
-    Distcc 3.3 and above requires such a whitelist, or else it will only run
-    with the --make-me-a-botnet parameter (even in ssh mode).
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
-        for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
-            cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
-            handle.write(cmd_full + "\n")
-    pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
-    pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
-
-
-def configure_distccd_wrapper(args):
-    """
-    Wrap distccd in a shell script, so we can pass the compiler whitelist and
-    set the verbose flag (when pmbootstrap is running with --verbose).
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
-        handle.write("#!/bin/sh\n"
-                     "export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
-                     "distccd --log-file /home/pmos/distccd.log --nice 19")
-        if args.verbose:
-            handle.write(" --verbose")
-        handle.write(" \"$@\"\n")
-    pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
-    pmb.chroot.user(args, ["cat", dir + "/distccd"])
-    pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
-
-
-def configure_sshd(args):
-    """
-    Configure the SSH daemon in the native chroot.
-    """
-    dir = "/home/pmos/.distcc-sshd"
-    config = """AllowAgentForwarding no
-                AllowTcpForwarding no
-                AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
-                HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
-                ListenAddress 127.0.0.1
-                PasswordAuthentication no
-                PidFile /home/pmos/.distcc-sshd/sshd.pid
-                Port """ + args.port_distccd + """
-                X11Forwarding no"""
-
-    with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
-        for line in config.split("\n"):
-            handle.write(line.lstrip() + "\n")
-    pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
-    pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
-
-
-def get_running_pid(args):
-    """
-    :returns: the running distcc-sshd's pid as integer or None
-    """
-    # PID file must exist
-    pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
-    pidfile_outside = args.work + "/chroot_native" + pidfile
-    if not os.path.exists(pidfile_outside):
-        return None
-
-    # Verify, if it still exists by sending a kill signal
-    with open(pidfile_outside, "r") as handle:
-        pid = int(handle.read()[:-1])
-    try:
-        os.kill(pid, 0)
-    except OSError as err:
-        if err.errno == errno.ESRCH:  # no such process
-            pmb.helpers.run.root(args, ["rm", pidfile_outside])
-            return None
-    return pid
-
-
-def get_running_parameters(args):
-    """
-    Get the parameters of the currently running distcc-sshd instance.
-
-    :returns: a dictionary in the form of
-              {"arch": "armhf", "port": 1234, "verbose": False}
-              If the information can not be read, "arch" is set to "unknown"
-    """
-    # Return defaults
-    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-    if not os.path.exists(path):
-        return {"arch": "unknown", "port": 0, "verbose": False}
-
-    # Parse the file as JSON
-    with open(path, "r") as handle:
-        return json.loads(handle.read())
-
-
-def set_running_parameters(args, arch):
-    """
-    Set the parameters of the currently running distcc-sshd instance.
-    """
-    parameters = {"arch": arch,
-                  "port": args.port_distccd,
-                  "verbose": args.verbose}
-
-    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
-    with open(path, "w") as handle:
-        json.dump(parameters, handle)
-
-
-def is_running_with_same_parameters(args, arch):
-    """
-    Check whether we can use the already running distcc-sshd instance with our
-    current set of parameters. In case we can use it directly, we save some
-    time, otherwise we need to stop it, configure it again, and start it once
-    more.
-    """
-    if not get_running_pid(args):
-        return False
-
-    parameters = get_running_parameters(args)
-    return (parameters["arch"] == arch and
-            parameters["port"] == args.port_distccd and
-            parameters["verbose"] == args.verbose)
-
-
-def stop(args):
-    """
-    Kill the sshd process (by using its pid).
-    """
-    pid = get_running_pid(args)
-    if not pid:
-        return
-
-    parameters = get_running_parameters(args)
-    logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
-    pmb.chroot.user(args, ["kill", str(pid)])
-
-
-def start(args, arch):
-    """
-    Set up a new distcc-sshd instance or use an already running one.
-    """
-    if is_running_with_same_parameters(args, arch):
-        return
-    stop(args)
-
-    # Initialize server and client
-    suffix = "buildroot_" + arch
-    init_server(args)
-    init_client(args, suffix)
-
-    logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
-                 args.port_distccd)
-
-    # Configure server parameters (arch, port, verbose)
-    configure_authorized_keys(args, suffix)
-    configure_distccd_wrapper(args)
-    configure_cmdlist(args, arch)
-    configure_sshd(args)
-
-    # Run
-    dir = "/home/pmos/.distcc-sshd"
-    pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
-                           "-E", dir + "/log.txt"])
-    set_running_parameters(args, arch)

pmb/chroot/init.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os

pmb/chroot/initfs.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/chroot/initfs_hooks.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/chroot/mount.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -106,3 +106,18 @@ def mount_native_into_foreign(args, suffix):
     if not os.path.lexists(musl_link):
         pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,
                                     musl_link])
+
+def remove_mnt_pmbootstrap(args, suffix):
+    """ Safely remove /mnt/pmbootstrap directories from the chroot, without
+        running rm -r as root and potentially removing data inside the
+        mountpoint in case it was still mounted (bug in pmbootstrap, or user
+        ran pmbootstrap 2x in parallel). This is similar to running 'rm -r -d',
+        but we don't assume that the host's rm has the -d flag (busybox does
+        not). """
+    mnt_dir = f"{args.work}/chroot_{suffix}/mnt/pmbootstrap"
+
+    if not os.path.exists(mnt_dir):
+        return
+
+    for path in glob.glob(f"{mnt_dir}/*") + [mnt_dir]:
+        pmb.helpers.run.root(args, ["rmdir", path])

pmb/chroot/other.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/chroot/root.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import shutil
@@ -65,21 +65,17 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
     for key, value in env.items():
         env_all[key] = value
 
-    # Preserve proxy environment variables
-    for var in ["FTP_PROXY", "ftp_proxy", "HTTP_PROXY", "http_proxy",
-                "HTTPS_PROXY", "https_proxy", "HTTP_PROXY_AUTH"]:
-        if var in os.environ:
-            env_all[var] = os.environ[var]
-
     # Build the command in steps and run it, e.g.:
     # cmd: ["echo", "test"]
     # cmd_chroot: ["/sbin/chroot", "/..._native", "/bin/sh", "-c", "echo test"]
     # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
     executables = executables_absolute_path()
     cmd_chroot = [executables["chroot"], chroot, "/bin/sh", "-c",
-                  pmb.helpers.run.flat_cmd(cmd, working_dir)]
-    cmd_sudo = [pmb.config.sudo, "env", "-i", executables["sh"], "-c",
-                pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)]
+                  pmb.helpers.run_core.flat_cmd(cmd, working_dir)]
+    cmd_sudo = pmb.config.sudo([
+        "env", "-i", executables["sh"], "-c",
+        pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)]
+    )
     return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output,
                                      output_return, check, True,
                                      disable_timeout)

pmb/chroot/shutdown.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -7,7 +7,6 @@ import socket
 from contextlib import closing
 
 import pmb.chroot
-import pmb.chroot.distccd
 import pmb.helpers.mount
 import pmb.install.losetup
 import pmb.parse.arch
@@ -23,6 +22,17 @@ def kill_adb(args):
             pmb.chroot.root(args, ["adb", "-P", str(port), "kill-server"])
 
 
+def kill_sccache(args):
+    """
+    Kill sccache daemon if it's running. Unlike ccache it automatically spawns
+    a daemon when you call it and exits after some time of inactivity.
+    """
+    port = 4226
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+        if sock.connect_ex(("127.0.0.1", port)) == 0:
+            pmb.chroot.root(args, ["sccache", "--stop-server"])
+
+
 def shutdown_cryptsetup_device(args, name):
     """
     :param name: cryptsetup device name, usually "pm_crypt" in pmbootstrap
@@ -49,10 +59,9 @@ def shutdown_cryptsetup_device(args, name):
 
 
 def shutdown(args, only_install_related=False):
-    pmb.chroot.distccd.stop(args)
-
-    # Stop adb server
+    # Stop daemons
     kill_adb(args)
+    kill_sccache(args)
 
     # Umount installation-related paths (order is important!)
     pmb.helpers.mount.umount_all(args, args.work +

pmb/chroot/user.py

@@ -1,7 +1,8 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.chroot.root
 import pmb.helpers.run
+import pmb.helpers.run_core
 
 
 def user(args, cmd, suffix="native", working_dir="/", output="log",
@@ -21,7 +22,7 @@ def user(args, cmd, suffix="native", working_dir="/", output="log",
     if "HOME" not in env:
         env["HOME"] = "/home/pmos"
 
-    flat_cmd = pmb.helpers.run.flat_cmd(cmd, env=env)
+    flat_cmd = pmb.helpers.run_core.flat_cmd(cmd, env=env)
     cmd = ["busybox", "su", "pmos", "-c", flat_cmd]
     return pmb.chroot.root(args, cmd, suffix, working_dir, output,
                            output_return, check, {}, auto_init)

pmb/chroot/zap.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -17,7 +17,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False,
         rust=False, netboot=False):
     """
-    Shutdown everything inside the chroots (e.g. distccd, adb), umount
+    Shutdown everything inside the chroots (e.g. adb), umount
     everything and then safely remove folders from the work-directory.
 
     :param dry: Only show what would be deleted, do not delete for real

pmb/ci/__init__.py

@@ -0,0 +1,169 @@
+# Copyright 2023 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+import collections
+import glob
+import logging
+import os
+import shlex
+import pmb.chroot
+import pmb.helpers.cli
+
+
+def get_ci_scripts(topdir):
+    """ Find 'pmbootstrap ci'-compatible scripts inside a git repository, and
+        parse their metadata (description, options). The reference is at:
+        https://postmarketos.org/pmb-ci
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir()
+        :returns: a dict of CI scripts found in the git repository, e.g.
+                  {"ruff": {"description": "lint all python scripts",
+                              "options": []},
+                   ...} """
+    ret = {}
+    for script in glob.glob(f"{topdir}/.ci/*.sh"):
+        is_pmb_ci_script = False
+        description = ""
+        options = []
+
+        with open(script) as handle:
+            for line in handle:
+                if line.startswith("# https://postmarketos.org/pmb-ci"):
+                    is_pmb_ci_script = True
+                elif line.startswith("# Description: "):
+                    description = line.split(": ", 1)[1].rstrip()
+                elif line.startswith("# Options: "):
+                    options = line.split(": ", 1)[1].rstrip().split(" ")
+                elif not line.startswith("#"):
+                    # Stop parsing after the block of comments on top
+                    break
+
+        if not is_pmb_ci_script:
+            continue
+
+        if not description:
+            logging.error(f"ERROR: {script}: missing '# Description: …' line")
+            exit(1)
+
+        for option in options:
+            if option not in pmb.config.ci_valid_options:
+                raise RuntimeError(f"{script}: unsupported option '{option}'."
+                                   " Typo in script or pmbootstrap too old?")
+
+        short_name = os.path.basename(script).split(".", -1)[0]
+        ret[short_name] = {"description": description,
+                           "options": options}
+    return ret
+
+
+def sort_scripts_by_speed(scripts):
+    """ Order the scripts, so fast scripts run before slow scripts. Whether a
+        script is fast or not is determined by the '# Options: slow' comment in
+        the file.
+        :param scripts: return of get_ci_scripts()
+        :returns: same format as get_ci_scripts(), but as ordered dict with
+                  fast scripts before slow scripts """
+    ret = collections.OrderedDict()
+
+    # Fast scripts first
+    for script_name, script in scripts.items():
+        if "slow" in script["options"]:
+            continue
+        ret[script_name] = script
+
+    # Then slow scripts
+    for script_name, script in scripts.items():
+        if "slow" not in script["options"]:
+            continue
+        ret[script_name] = script
+    return ret
+
+
+def ask_which_scripts_to_run(scripts_available):
+    """ Display an interactive prompt about which of the scripts the user
+        wishes to run, or all of them.
+        :param scripts_available: same format as get_ci_scripts()
+        :returns: either full scripts_available (all selected), or a subset """
+    count = len(scripts_available.items())
+    choices = ["all"]
+
+    logging.info(f"Available CI scripts ({count}):")
+    for script_name, script in scripts_available.items():
+        extra = ""
+        if "slow" in script["options"]:
+            extra += " (slow)"
+        logging.info(f"* {script_name}: {script['description']}{extra}")
+        choices += [script_name]
+
+    selection = pmb.helpers.cli.ask("Which script?", None, "all",
+                                    complete=choices)
+    if selection == "all":
+        return scripts_available
+
+    ret = {}
+    ret[selection] = scripts_available[selection]
+    return ret
+
+
+def copy_git_repo_to_chroot(args, topdir):
+    """ Create a tarball of the git repo (including unstaged changes and new
+        files) and extract it in chroot_native.
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir() """
+    pmb.chroot.init(args)
+    tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz"
+    files = pmb.helpers.git.get_files(args, topdir)
+
+    with open(f"{tarball_path}.files", "w") as handle:
+        for file in files:
+            handle.write(file)
+            handle.write("\n")
+
+    pmb.helpers.run.user(args, ["tar", "-cf", tarball_path, "-T",
+                                f"{tarball_path}.files"], topdir)
+
+    ci_dir = "/home/pmos/ci"
+    pmb.chroot.user(args, ["rm", "-rf", ci_dir])
+    pmb.chroot.user(args, ["mkdir", ci_dir])
+    pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"],
+                    working_dir=ci_dir)
+
+
+def run_scripts(args, topdir, scripts):
+    """ Run one of the given scripts after another, either natively or in a
+        chroot. Display a progress message and stop on error (without printing
+        a python stack trace).
+        :param topdir: top directory of the git repository, get it with:
+                       pmb.helpers.git.get_topdir()
+        :param scripts: return of get_ci_scripts() """
+    steps = len(scripts)
+    step = 0
+    repo_copied = False
+
+    for script_name, script in scripts.items():
+        step += 1
+
+        where = "pmbootstrap chroot"
+        if "native" in script["options"]:
+            where = "native"
+
+        script_path = f".ci/{script_name}.sh"
+        logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}"
+                     f" [{where}] ***")
+
+        if "native" in script["options"]:
+            rc = pmb.helpers.run.user(args, [script_path], topdir,
+                                      output="tui")
+            continue
+        else:
+            # Run inside pmbootstrap chroot
+            if not repo_copied:
+                copy_git_repo_to_chroot(args, topdir)
+                repo_copied = True
+
+            env = {"TESTUSER": "pmos"}
+            rc = pmb.chroot.root(args, [script_path], check=False, env=env,
+                                 working_dir="/home/pmos/ci",
+                                 output="tui")
+        if rc:
+            logging.error(f"ERROR: CI script failed: {script_name}")
+            exit(1)

pmb/config/__init__.py

@@ -1,9 +1,10 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import multiprocessing
 import os
 import pmb.parse.arch
 import sys
+from typing import List
 
 #
 # Exported functions
@@ -17,7 +18,6 @@ from pmb.config.sudo import which_sudo
 #
 # Exported variables (internal configuration)
 #
-version = "1.47.1"
 pmb_src = os.path.normpath(os.path.realpath(__file__) + "/../../..")
 apk_keys_path = pmb_src + "/pmb/data/keys"
 arch_native = pmb.parse.arch.alpine_native()
@@ -27,7 +27,9 @@ arch_native = pmb.parse.arch.alpine_native()
 # Update this frequently to prevent a MITM attack with an outdated version
 # (which may contain a vulnerable apk/openssl, and allows an attacker to
 # exploit the system!)
-apk_tools_min_version = {"edge": "2.12.9-r7",
+apk_tools_min_version = {"edge": "2.14.0-r5",
+                         "v3.18": "2.14.0-r2",
+                         "v3.17": "2.12.10-r1",
                          "v3.16": "2.12.9-r3",
                          "v3.15": "2.12.7-r3",
                          "v3.14": "2.12.7-r0",
@@ -51,36 +53,53 @@ ondev_min_version = "0.2.0"
 # Programs that pmbootstrap expects to be available from the host system. Keep
 # in sync with README.md, and try to keep the list as small as possible. The
 # idea is to run almost everything in Alpine chroots.
-required_programs = ["git", "openssl", "ps"]
-sudo = which_sudo()
+required_programs = [
+    "git",
+    "openssl",
+    "ps",
+    "tar",
+]
+
+
+def sudo(cmd: List[str]) -> List[str]:
+    """Adapt a command to run as root."""
+    sudo = which_sudo()
+    if sudo:
+        return [sudo, *cmd]
+    else:
+        return cmd
+
 
 # Keys saved in the config file (mostly what we ask in 'pmbootstrap init')
-config_keys = ["aports",
-               "ccache_size",
-               "device",
-               "extra_packages",
-               "hostname",
-               "build_pkgs_on_install",
-               "is_default_channel",
-               "jobs",
-               "kernel",
-               "keymap",
-               "locale",
-               "mirror_alpine",
-               "mirrors_postmarketos",
-               "nonfree_firmware",
-               "nonfree_userland",
-               "ssh_keys",
-               "ssh_key_glob",
-               "timezone",
-               "ui",
-               "ui_extras",
-               "user",
-               "work",
-               "boot_size",
-               "extra_space",
-               "sudo_timer",
-               "qemu_redir_stdio"]
+config_keys = [
+    "aports",
+    "boot_size",
+    "build_default_device_arch",
+    "build_pkgs_on_install",
+    "ccache_size",
+    "device",
+    "extra_packages",
+    "extra_space",
+    "hostname",
+    "is_default_channel",
+    "jobs",
+    "kernel",
+    "keymap",
+    "locale",
+    "mirror_alpine",
+    "mirrors_postmarketos",
+    "nonfree_firmware",
+    "nonfree_userland",
+    "qemu_redir_stdio",
+    "ssh_key_glob",
+    "ssh_keys",
+    "sudo_timer",
+    "timezone",
+    "ui",
+    "ui_extras",
+    "user",
+    "work",
+]
 
 # Config file/commandline default values
 # $WORK gets replaced with the actual value for args.work (which may be
@@ -104,7 +123,7 @@ defaults = {
     "jobs": str(multiprocessing.cpu_count() + 1),
     "kernel": "stable",
     "keymap": "",
-    "locale": "C.UTF-8",
+    "locale": "en_US.UTF-8",
     "log": "$WORK/log.txt",
     "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
     # NOTE: mirrors_postmarketos variable type is supposed to be
@@ -112,18 +131,18 @@ defaults = {
     "mirrors_postmarketos": "http://mirror.postmarketos.org/postmarketos/",
     "nonfree_firmware": True,
     "nonfree_userland": False,
-    "port_distccd": "33632",
     "ssh_keys": False,
     "ssh_key_glob": "~/.ssh/id_*.pub",
     "timezone": "GMT",
-    "ui": "weston",
+    "ui": "console",
     "ui_extras": False,
     "user": "user",
     "work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
     "boot_size": "256",
     "extra_space": "0",
     "sudo_timer": False,
-    "qemu_redir_stdio": False
+    "qemu_redir_stdio": False,
+    "build_default_device_arch": False,
 }
 
 
@@ -148,26 +167,6 @@ if "NO_COLOR" in os.environ:
     for style in styles.keys():
         styles[style] = ""
 
-
-# List of available locales taken from musl-locales package; see
-# https://pkgs.alpinelinux.org/contents?name=musl-locales
-locales = [
-    "C.UTF-8",
-    "ch_DE.UTF-8",
-    "de_CH.UTF-8",
-    "de_DE.UTF-8",
-    "en_GB.UTF-8",
-    "en_US.UTF-8",
-    "es_ES.UTF-8",
-    "fr_FR.UTF-8",
-    "it_IT.UTF-8",
-    "nb_NO.UTF-8",
-    "nl_NL.UTF-8",
-    "pt_BR.UTF-8",
-    "ru_RU.UTF-8",
-    "sv_SE.UTF-8"
-]
-
 # Supported filesystems and their fstools packages
 filesystems = {"btrfs": "btrfs-progs",
                "ext2": "e2fsprogs",
@@ -207,17 +206,21 @@ chroot_host_path = os.environ["PATH"] + ":/usr/sbin/"
 # $WORK gets replaced with args.work
 # $ARCH gets replaced with the chroot architecture (eg. x86_64, armhf)
 # $CHANNEL gets replaced with the release channel (e.g. edge, v21.03)
+# Use no more than one dir after /mnt/pmbootstrap, see remove_mnt_pmbootstrap.
 chroot_mount_bind = {
     "/proc": "/proc",
     "$WORK/cache_apk_$ARCH": "/var/cache/apk",
-    "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap-ccache",
+    "$WORK/cache_appstream/$ARCH/$CHANNEL": "/mnt/appstream-data",
+    "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap/ccache",
     "$WORK/cache_distfiles": "/var/cache/distfiles",
-    "$WORK/cache_git": "/mnt/pmbootstrap-git",
-    "$WORK/cache_rust": "/mnt/pmbootstrap-rust",
-    "$WORK/config_abuild": "/mnt/pmbootstrap-abuild-config",
+    "$WORK/cache_git": "/mnt/pmbootstrap/git",
+    "$WORK/cache_go": "/mnt/pmbootstrap/go",
+    "$WORK/cache_rust": "/mnt/pmbootstrap/rust",
+    "$WORK/config_abuild": "/mnt/pmbootstrap/abuild-config",
     "$WORK/config_apk_keys": "/etc/apk/keys",
-    "$WORK/images_netboot": "/mnt/pmbootstrap-netboot",
-    "$WORK/packages/$CHANNEL": "/mnt/pmbootstrap-packages",
+    "$WORK/cache_sccache": "/mnt/pmbootstrap/sccache",
+    "$WORK/images_netboot": "/mnt/pmbootstrap/netboot",
+    "$WORK/packages/$CHANNEL": "/mnt/pmbootstrap/packages",
 }
 
 # Building chroots (all chroots, except for the rootfs_ chroot) get symlinks in
@@ -228,13 +231,20 @@ chroot_mount_bind = {
 # a no-go, but at least until this is resolved properly, let's cache the
 # dependencies and downloads as suggested in "Caching the Cargo home in CI":
 # https://doc.rust-lang.org/cargo/guide/cargo-home.html
+# Go: cache the directories "go env GOMODCACHE" and "go env GOCACHE" point to,
+# to avoid downloading dependencies over and over (GOMODCACHE, similar to the
+# rust depends caching described above) and to cache build artifacts (GOCACHE,
+# similar to ccache).
 chroot_home_symlinks = {
-    "/mnt/pmbootstrap-abuild-config": "/home/pmos/.abuild",
-    "/mnt/pmbootstrap-ccache": "/home/pmos/.ccache",
-    "/mnt/pmbootstrap-packages": "/home/pmos/packages/pmos",
-    "/mnt/pmbootstrap-rust/registry/index": "/home/pmos/.cargo/registry/index",
-    "/mnt/pmbootstrap-rust/registry/cache": "/home/pmos/.cargo/registry/cache",
-    "/mnt/pmbootstrap-rust/git/db": "/home/pmos/.cargo/git/db",
+    "/mnt/pmbootstrap/abuild-config": "/home/pmos/.abuild",
+    "/mnt/pmbootstrap/ccache": "/home/pmos/.ccache",
+    "/mnt/pmbootstrap/go/gocache": "/home/pmos/.cache/go-build",
+    "/mnt/pmbootstrap/go/gomodcache": "/home/pmos/go/pkg/mod",
+    "/mnt/pmbootstrap/packages": "/home/pmos/packages/pmos",
+    "/mnt/pmbootstrap/rust/git/db": "/home/pmos/.cargo/git/db",
+    "/mnt/pmbootstrap/rust/registry/cache": "/home/pmos/.cargo/registry/cache",
+    "/mnt/pmbootstrap/rust/registry/index": "/home/pmos/.cargo/registry/index",
+    "/mnt/pmbootstrap/sccache": "/home/pmos/.cache/sccache",
 }
 
 # Device nodes to be created in each chroot. Syntax for each entry:
@@ -262,7 +272,7 @@ chroot_outdated = 3600 * 24 * 2
 # specify architectures supported by Alpine here. For cross-compiling,
 # we need to generate the "musl-$ARCH", "binutils-$ARCH" and "gcc-$ARCH"
 # packages (use "pmbootstrap aportgen musl-armhf" etc.).
-build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86"]
+build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"]
 
 # Packages that will be installed in a chroot before it builds packages
 # for the first time
@@ -284,7 +294,7 @@ build_packages = ["abuild", "build-base", "ccache", "git"]
 #     this e.g. if the order of the elements is important.
 
 # Necessary kernel config options
-necessary_kconfig_options = {
+kconfig_options = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
             "ANDROID_PARANOID_NETWORK": False,
@@ -305,6 +315,16 @@ necessary_kconfig_options = {
             "VT": True,
         }
     },
+    ">=2.6.0": {
+        "all": {
+            "BINFMT_ELF": True,
+        },
+    },
+    ">=3.10.0": {
+        "all": {
+            "BINFMT_SCRIPT": True,
+        },
+    },
     ">=4.0.0": {
         "all": {
             "UEVENT_HELPER": True,
@@ -330,26 +350,31 @@ necessary_kconfig_options = {
 }
 
 # Necessary waydroid kernel config options (android app support)
-necessary_kconfig_options_waydroid = {
+kconfig_options_waydroid = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
-            "SQUASHFS": True,
-            "SQUASHFS_XZ": True,
-            "SQUASHFS_XATTR": True,
-            "TMPFS_XATTR": True,
-            "ANDROID_BINDER_IPC": True,
             "ANDROID_BINDERFS": False,
             "ANDROID_BINDER_DEVICES": ["binder", "hwbinder", "vndbinder"],
+            "ANDROID_BINDER_IPC": True,
+            "ANDROID_BINDER_IPC_SELFTEST": False,
+            "BLK_DEV_LOOP": True,
+            "BPF_SYSCALL": True,
+            "BRIDGE": True,
+            "BRIDGE_VLAN_FILTERING": True,
+            "CGROUP_BPF": True,
+            "FUSE_FS": True,
+            "IP_NF_MANGLE": True,
             "NETFILTER_XTABLES": True,
             "NETFILTER_XT_MATCH_COMMENT": True,
-            "IP_NF_MANGLE": True,
-            "FUSE_FS": True,
-            "BLK_DEV_LOOP": True,
+            "PSI": True,
+            "PSI_DEFAULT_DISABLED": False,
+            "SQUASHFS": True,
+            "SQUASHFS_XATTR": True,
+            "SQUASHFS_XZ": True,
+            "TMPFS_XATTR": True,
             "TUN": True,
             "VETH": True,
             "VLAN_8021Q": True,  # prerequisite for bridge
-            "BRIDGE": True,
-            "BRIDGE_VLAN_FILTERING": True,
         }
     },
     ">=3.5": {
@@ -363,7 +388,7 @@ necessary_kconfig_options_waydroid = {
             "PSI_DEFAULT_DISABLED": False,
         }
     },
-    "<5.18_rc1": {  # option has been dropped
+    "<5.18": {  # option has been dropped
         "all": {
             "ASHMEM": True,
         }
@@ -372,7 +397,7 @@ necessary_kconfig_options_waydroid = {
 
 # Necessary iwd kernel config options (inet wireless daemon)
 # Obtained from 'grep ADD_MISSING src/main.c' in iwd.git
-necessary_kconfig_options_iwd = {
+kconfig_options_iwd = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
             "ASYMMETRIC_KEY_TYPE": True,
@@ -399,7 +424,7 @@ necessary_kconfig_options_iwd = {
 }
 
 # Necessary nftables kernel config options (firewall)
-necessary_kconfig_options_nftables = {
+kconfig_options_nftables = {
     ">=3.13.0": {  # nftables support introduced here
         "all": {  # all arches
             "NETFILTER": True,
@@ -426,7 +451,7 @@ necessary_kconfig_options_nftables = {
             "IP6_NF_NAT": True,
         }
     },
-    ">=3.13.0 <5.17_rc1": {  # option has been dropped
+    ">=3.13.0 <5.17": {  # option has been dropped
         "all": {  # all arches
             "NFT_COUNTER": True,
         },
@@ -434,7 +459,7 @@ necessary_kconfig_options_nftables = {
 }
 
 # Necessary kernel config options for containers (lxc, Docker)
-necessary_kconfig_options_containers = {
+kconfig_options_containers = {
     ">=0.0.0": {  # all versions, more specifically - since >=2.5~2.6
         "all": {  # all arches
             "NAMESPACES": True,
@@ -495,14 +520,19 @@ necessary_kconfig_options_containers = {
     ">=3.6": {
         "all": {  # all arches
             "MEMCG": True,
-            "MEMCG_SWAP": True,
             "DM_THIN_PROVISIONING": True,  # Storage Drivers
+            "SWAP": True,
         },
         "x86 x86_64": {  # only for x86, x86_64 (and sparc64, ia64)
             "HUGETLB_PAGE": True,
             "CGROUP_HUGETLB": True,  # Optional section
         }
     },
+    ">=3.6 <6.1_rc1": {  # option has been dropped
+        "all": {
+            "MEMCG_SWAP": True,
+        }
+    },
     ">=3.7 <5.0": {
         "all": {
             "NF_NAT_IPV4": True,  # Needed for lxc
@@ -545,7 +575,7 @@ necessary_kconfig_options_containers = {
 }
 
 # Necessary zram kernel config options (RAM disk with on-the-fly compression)
-necessary_kconfig_options_zram = {
+kconfig_options_zram = {
     ">=3.14.0": {  # zram support introduced here
         "all": {  # all arches
             "ZRAM": True,
@@ -558,7 +588,7 @@ necessary_kconfig_options_zram = {
 }
 
 # Necessary netboot kernel config options
-necessary_kconfig_options_netboot = {
+kconfig_options_netboot = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
             "BLK_DEV_NBD": True,
@@ -566,8 +596,67 @@ necessary_kconfig_options_netboot = {
     },
 }
 
+# Necessary wireguard & wg-quick kernel config options
+# From https://gitweb.gentoo.org/repo/gentoo.git/tree/net-vpn/wireguard-tools/wireguard-tools-1.0.20210914.ebuild?id=76aaa1eeb6f001baaa68e6946f917ebb091bbd9d # noqa
+kconfig_options_wireguard = {
+    ">=5.6_rc1": {  # all versions
+        "all": {  # all arches
+            "WIREGUARD": True,
+            "IP_ADVANCED_ROUTER": True,
+            "IP_MULTIPLE_TABLES": True,
+            "IPV6_MULTIPLE_TABLES": True,
+            "NF_TABLES": True,
+            "NF_TABLES_IPV4": True,
+            "NF_TABLES_IPV6": True,
+            "NFT_CT": True,
+            "NFT_FIB": True,
+            "NFT_FIB_IPV4": True,
+            "NFT_FIB_IPV6": True,
+            "NF_CONNTRACK_MARK": True,
+        },
+    },
+}
+
+# Necessary file system config options
+kconfig_options_filesystems = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "BTRFS_FS": True,
+            "EXFAT_FS": True,
+            "EXT4_FS": True,
+            "F2FS_FS": True,
+        },
+    },
+}
+
+kconfig_options_usb_gadgets = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            # disable legacy gadgets
+            "USB_ETH": False,
+            "USB_FUNCTIONFS": False,
+            "USB_MASS_STORAGE": False,
+            "USB_G_SERIAL": False,
+            # enable configfs gadgets
+            "USB_CONFIGFS_RNDIS": True,  # USB networking via RNDIS
+        },
+    },
+}
+
+# Various other kernel config options
+kconfig_options_community = {
+    ">=0.0.0": {  # all versions
+        "all": {  # all arches
+            "INPUT_UINPUT": True,  # buffyboard
+            "LEDS_TRIGGER_TIMER": True,  # hfd-service
+            "NETFILTER_XT_MATCH_TCPMSS": True,  # change MTU, e.g. for Wireguard
+            "NETFILTER_XT_TARGET_TCPMSS": True,  # change MTU, e.g. for Wireguard
+        },
+    },
+}
+
 # Necessary UEFI boot config options
-necessary_kconfig_options_uefi = {
+kconfig_options_uefi = {
     ">=0.0.0": {  # all versions
         "all": {  # all arches
             "EFI_STUB": True,
@@ -639,6 +728,9 @@ apkbuild_attributes = {
     "_outdir": {},
     "_config": {},
 
+    # linux-edge
+    "_depends_dev": {"array": True},
+
     # mesa
     "_llvmver": {},
 
@@ -659,9 +751,14 @@ apkbuild_attributes = {
 apkbuild_custom_valid_options = [
     "!pmb:crossdirect",
     "!pmb:kconfig-check",
-    "pmb:kconfigcheck-waydroid",
+    "pmb:kconfigcheck-community",
     "pmb:kconfigcheck-containers",
+    "pmb:kconfigcheck-iwd",
+    "pmb:kconfigcheck-netboot",
     "pmb:kconfigcheck-nftables",
+    "pmb:kconfigcheck-uefi",
+    "pmb:kconfigcheck-waydroid",
+    "pmb:kconfigcheck-zram",
     "pmb:cross-native",
     "pmb:gpu-accel",
     "pmb:strict",
@@ -676,7 +773,6 @@ deviceinfo_attributes = [
     "codename",
     "year",
     "dtb",
-    "modules_initfs",
     "arch",
 
     # device
@@ -696,13 +792,22 @@ deviceinfo_attributes = [
     # flash
     "flash_heimdall_partition_kernel",
     "flash_heimdall_partition_initfs",
-    "flash_heimdall_partition_system",
+    "flash_heimdall_partition_rootfs",
+    "flash_heimdall_partition_system", # deprecated
     "flash_heimdall_partition_vbmeta",
     "flash_heimdall_partition_dtbo",
     "flash_fastboot_partition_kernel",
-    "flash_fastboot_partition_system",
+    "flash_fastboot_partition_rootfs",
+    "flash_fastboot_partition_system", # deprecated
     "flash_fastboot_partition_vbmeta",
     "flash_fastboot_partition_dtbo",
+    "flash_rk_partition_kernel",
+    "flash_rk_partition_rootfs",
+    "flash_rk_partition_system", # deprecated
+    "flash_mtkclient_partition_kernel",
+    "flash_mtkclient_partition_rootfs",
+    "flash_mtkclient_partition_vbmeta",
+    "flash_mtkclient_partition_dtbo",
     "generate_legacy_uboot_initfs",
     "kernel_cmdline",
     "generate_bootimg",
@@ -740,7 +845,7 @@ deviceinfo_attributes = [
     "keymaps",
 ]
 
-# Valid types for the 'chassis' atribute in deviceinfo
+# Valid types for the 'chassis' attribute in deviceinfo
 # See https://www.freedesktop.org/software/systemd/man/machine-info.html
 deviceinfo_chassis_types = [
     "desktop",
@@ -771,9 +876,6 @@ default_ip = "172.16.42.1"
 install_native_packages = ["cryptsetup", "util-linux", "parted"]
 install_device_packages = ["postmarketos-base"]
 
-# Groups for the default user
-install_user_groups = ["wheel", "video", "audio", "input", "plugdev", "netdev"]
-
 #
 # FLASH
 #
@@ -782,6 +884,7 @@ flash_methods = [
     "0xffff",
     "fastboot",
     "heimdall",
+    "mtkclient",
     "none",
     "rkdeveloptool",
     "uuu",
@@ -806,7 +909,7 @@ $IMAGE: Path to the combined boot/rootfs image
 $IMAGE_SPLIT_BOOT: Path to the (split) boot image
 $IMAGE_SPLIT_ROOT: Path to the (split) rootfs image
 $PARTITION_KERNEL: Partition to flash the kernel/boot.img to
-$PARTITION_SYSTEM: Partition to flash the rootfs to
+$PARTITION_ROOTFS: Partition to flash the rootfs to
 
 Fastboot specific: $KERNEL_CMDLINE
 Heimdall specific: $PARTITION_INITFS
@@ -814,10 +917,10 @@ uuu specific: $UUU_SCRIPT
 """
 flashers = {
     "fastboot": {
-        "depends": ["android-tools", "avbtool"],
+        "depends": [],  # pmaports.cfg: supported_fastboot_depends
         "actions": {
             "list_devices": [["fastboot", "devices", "-l"]],
-            "flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
+            "flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
                               "$IMAGE"]],
             "flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
                               "$BOOT/boot.img$FLAVOR"]],
@@ -848,7 +951,7 @@ flashers = {
         "depends": ["android-tools"],
         "actions": {
             "list_devices": [["fastboot", "devices", "-l"]],
-            "flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
+            "flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
                               "$IMAGE_SPLIT_ROOT"]],
             "flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
                               "$IMAGE_SPLIT_BOOT"]],
@@ -866,7 +969,7 @@ flashers = {
             "list_devices": [["heimdall", "detect"]],
             "flash_rootfs": [
                 ["heimdall_wait_for_device.sh"],
-                ["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
+                ["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
             "flash_kernel": [["heimdall_flash_kernel.sh",
                               "$BOOT/initramfs$FLAVOR", "$PARTITION_INITFS",
                               "$BOOT/vmlinuz$FLAVOR$DTB",
@@ -876,12 +979,12 @@ flashers = {
     # Some Samsung devices need a 'boot.img' file, just like the one generated
     # fastboot compatible devices. Example: s7562, n7100
     "heimdall-bootimg": {
-        "depends": ["heimdall", "avbtool"],
+        "depends": [],  # pmaports.cfg: supported_heimdall_depends
         "actions": {
             "list_devices": [["heimdall", "detect"]],
             "flash_rootfs": [
                 ["heimdall_wait_for_device.sh"],
-                ["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
+                ["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
             "flash_kernel": [
                 ["heimdall_wait_for_device.sh"],
                 ["heimdall", "flash", "--$PARTITION_KERNEL",
@@ -924,7 +1027,7 @@ flashers = {
         "actions": {
             "list_devices": [["rkdeveloptool", "list"]],
             "flash_rootfs": [
-                ["rkdeveloptool", "write-partition", "$PARTITION_SYSTEM",
+                ["rkdeveloptool", "write-partition", "$PARTITION_ROOTFS",
                     "$IMAGE_SPLIT_ROOT"]
             ],
             "flash_kernel": [
@@ -932,6 +1035,27 @@ flashers = {
                     "$IMAGE_SPLIT_BOOT"]
             ],
         },
+    },
+    "mtkclient": {
+        "depends": ["mtkclient"],
+        "actions": {
+            "flash_rootfs": [["mtk", "w", "$PARTITION_ROOTFS",
+                              "$IMAGE"]],
+            "flash_kernel": [["mtk", "w", "$PARTITION_KERNEL",
+                              "$BOOT/boot.img$FLAVOR"]],
+            "flash_vbmeta": [
+                # Generate vbmeta image with "disable verification" flag
+                ["avbtool", "make_vbmeta_image", "--flags", "2",
+                    "--padding_size", "$FLASH_PAGESIZE",
+                    "--output", "/vbmeta.img"],
+                ["mtk", "w", "$PARTITION_VBMETA", "/vbmeta.img"],
+                ["rm", "-f", "/vbmeta.img"]
+            ],
+            "flash_dtbo": [["mtk", "w", "$PARTITION_DTBO",
+                            "$BOOT/dtbo.img"]],
+            "flash_lk2nd": [["mtk", "w", "$PARTITION_KERNEL",
+                             "$BOOT/lk2nd.img"]]
+        }
     }
 }
 
@@ -999,9 +1123,15 @@ newapkbuild_arguments_switches_other = [
 # Patterns of package names to ignore for automatic pmaport upgrading
 # ("pmbootstrap aportupgrade --all")
 upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*",
-                  "*-aarch64", "*-armhf", "*-armv7"]
+                  "*-aarch64", "*-armhf", "*-armv7", "*-riscv64"]
 
 #
 # SIDELOAD
 #
 sideload_sudo_prompt = "[sudo] password for %u@%h: "
+
+#
+# CI
+#
+# Valid options für 'pmbootstrap ci', see https://postmarketos.org/pmb-ci
+ci_valid_options = ["native", "slow"]

pmb/config/init.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -11,6 +11,7 @@ import pmb.config
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.devices
+import pmb.helpers.git
 import pmb.helpers.http
 import pmb.helpers.logging
 import pmb.helpers.other
@@ -33,6 +34,23 @@ def require_programs():
                            f" {', '.join(missing)}")
 
 
+def ask_for_username(args):
+    """
+    Ask for a reasonable username for the non-root user.
+
+    :returns: the username
+    """
+    while True:
+        ret = pmb.helpers.cli.ask("Username", None, args.user, False,
+                                  "[a-z_][a-z0-9_-]*")
+        if ret == "root":
+            logging.fatal("ERROR: don't put \"root\" here. This is about"
+                          " creating an additional non-root user. Don't worry,"
+                          " the root user will also be created ;)")
+            continue
+        return ret
+
+
 def ask_for_work_path(args):
     """
     Ask for the work path, until we can create it (when it does not exist) and
@@ -92,7 +110,10 @@ def ask_for_channel(args):
     # List channels
     logging.info("Choose the postmarketOS release channel.")
     logging.info(f"Available ({count}):")
-    for channel, channel_data in channels_cfg["channels"].items():
+    # Only show the first 3 releases. This includes edge, the latest supported
+    # release plus one. Should be a good solution until new needs arrive when
+    # we might want to have a custom channels.cfg attribute.
+    for channel, channel_data in list(channels_cfg["channels"].items())[:3]:
         logging.info(f"* {channel}: {channel_data['description']}")
 
     # Default for first run: "recommended" from channels.cfg
@@ -128,19 +149,22 @@ def ask_for_ui(args, info):
                 ui_list.pop(i)
                 hidden_ui_count += 1
 
+    # Get default
+    default = args.ui
+    if default not in dict(ui_list).keys():
+        default = pmb.config.defaults["ui"]
+
     logging.info(f"Available user interfaces ({len(ui_list) - 1}): ")
     ui_completion_list = []
     for ui in ui_list:
         logging.info(f"* {ui[0]}: {ui[1]}")
         ui_completion_list.append(ui[0])
     if hidden_ui_count > 0:
-        logging.info(f"NOTE: {hidden_ui_count} user interfaces are not"
-                     " available. If device supports GPU acceleration,"
-                     " set \"deviceinfo_gpu_accelerated\" to make UIs"
-                     " available. See: <https://wiki.postmarketos.org/wiki/"
-                     "Deviceinfo_reference")
+        logging.info(f"NOTE: {hidden_ui_count} UIs are hidden because"
+                     " \"deviceinfo_gpu_accelerated\" is not set (see"
+                     " https://postmarketos.org/deviceinfo).")
     while True:
-        ret = pmb.helpers.cli.ask("User interface", None, args.ui, True,
+        ret = pmb.helpers.cli.ask("User interface", None, default, True,
                                   complete=ui_completion_list)
         if ret in dict(ui_list).keys():
             return ret
@@ -602,15 +626,36 @@ def ask_build_pkgs_on_install(args):
                                    default=args.build_pkgs_on_install)
 
 
+def get_locales():
+    ret = []
+    list_path = f"{pmb.config.pmb_src}/pmb/data/locales"
+    with open(list_path, "r") as handle:
+        for line in handle:
+            ret += [line.rstrip()]
+    return ret
+
+
 def ask_for_locale(args):
-    locales = pmb.config.locales
-    logging.info(f"Available locales ({len(locales)}): {', '.join(locales)}")
-    return pmb.helpers.cli.ask("Choose default locale for installation",
-                               choices=None,
-                               default=args.locale,
-                               lowercase_answer=False,
-                               validation_regex="|".join(locales),
-                               complete=locales)
+    locales = get_locales()
+    logging.info("Choose your preferred locale, like e.g. en_US. Only UTF-8"
+                 " is supported, it gets appended automatically. Use"
+                 " tab-completion if needed.")
+
+    while True:
+        ret = pmb.helpers.cli.ask("Locale",
+                                  choices=None,
+                                  default=args.locale.replace(".UTF-8", ""),
+                                  lowercase_answer=False,
+                                  complete=locales)
+        ret = ret.replace(".UTF-8", "")
+        if ret not in locales:
+            logging.info("WARNING: this locale is not in the list of known"
+                         " valid locales.")
+            if pmb.helpers.cli.ask() != "y":
+                # Ask again
+                continue
+
+        return f"{ret}.UTF-8"
 
 
 def frontend(args):
@@ -636,6 +681,15 @@ def frontend(args):
     pmb.config.pmaports.switch_to_channel_branch(args, channel)
     cfg["pmbootstrap"]["is_default_channel"] = "False"
 
+    # Copy the git hooks if master was checked out. (Don't symlink them and
+    # only do it on master, so the git hooks don't change unexpectedly when
+    # having a random branch checked out.)
+    branch_current = pmb.helpers.git.rev_parse(args, args.aports,
+                                               extra_args=["--abbrev-ref"])
+    if branch_current == "master":
+        logging.info("NOTE: pmaports is on master branch, copying git hooks.")
+        pmb.config.pmaports.install_githooks(args)
+
     # Device
     device, device_exists, kernel, nonfree = ask_for_device(args)
     cfg["pmbootstrap"]["device"] = device
@@ -653,11 +707,7 @@ def frontend(args):
     if device_exists:
         cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, info)
 
-    # Username
-    cfg["pmbootstrap"]["user"] = pmb.helpers.cli.ask("Username", None,
-                                                     args.user, False,
-                                                     "[a-z_][a-z0-9_-]*")
-
+    cfg["pmbootstrap"]["user"] = ask_for_username(args)
     ask_for_provider_select_pkg(args, "postmarketos-base", cfg["providers"])
 
     # UI and various build options

pmb/config/load.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import configparser

pmb/config/merge_with_args.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config
 

pmb/config/pmaports.py

@@ -1,8 +1,9 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 import logging
 import os
+import sys
 
 import pmb.config
 import pmb.helpers.git
@@ -58,7 +59,7 @@ def check_version_pmaports(real):
 
 def check_version_pmbootstrap(min):
     # Compare versions
-    real = pmb.config.version
+    real = pmb.__version__
     if pmb.parse.version.compare(real, min) >= 0:
         return
 
@@ -89,9 +90,9 @@ def read_config(args):
 
     # Migration message
     if not os.path.exists(args.aports):
-        raise RuntimeError("We have split the aports repository from the"
-                           " pmbootstrap repository (#383). Please run"
-                           " 'pmbootstrap init' again to clone it.")
+        logging.error(f"ERROR: pmaports dir not found: {args.aports}")
+        logging.error("Did you run 'pmbootstrap init'?")
+        sys.exit(1)
 
     # Require the config
     path_cfg = args.aports + "/pmaports.cfg"
@@ -191,3 +192,17 @@ def switch_to_channel_branch(args, channel_new):
     # Verify pmaports.cfg on new branch
     read_config(args)
     return True
+
+
+def install_githooks(args):
+    hooks_dir = os.path.join(args.aports, ".githooks")
+    if not os.path.exists(hooks_dir):
+        logging.info("No .githooks dir found")
+        return
+    for h in os.listdir(hooks_dir):
+        src = os.path.join(hooks_dir, h)
+        # Use git default hooks dir so users can ignore our hooks
+        # if they dislike them by setting "core.hooksPath" git config
+        dst = os.path.join(args.aports, ".git", "hooks", h)
+        if pmb.helpers.run.user(args, ["cp", src, dst], check=False):
+            logging.warning(f"WARNING: Copying git hook failed: {dst}")

pmb/config/save.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/config/sudo.py

@@ -1,14 +1,22 @@
-# Copyright 2022 Anjandev Momi
+# Copyright 2023 Anjandev Momi
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import shutil
+from functools import lru_cache
+from typing import Optional
 
 
-def which_sudo():
-    """
+@lru_cache()
+def which_sudo() -> Optional[str]:
+    """Returns a command required to run commands as root, if any.
+
     Find whether sudo or doas is installed for commands that require root.
     Allows user to override preferred sudo with PMB_SUDO env variable.
     """
+
+    if os.getuid() == 0:
+        return None
+
     supported_sudos = ['doas', 'sudo']
 
     user_set_sudo = os.getenv("PMB_SUDO")

pmb/config/workdir.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """ Save, read, verify workdir state related information in $WORK/workdir.cfg,
     for example the init dates of the chroots. This is not saved in

pmb/data/locales

@@ -0,0 +1,304 @@
+C
+a_DJ
+aa_ER
+aa_ET
+af_ZA
+agr_PE
+ak_GH
+am_ET
+an_ES
+anp_IN
+ar_AE
+ar_BH
+ar_DZ
+ar_EG
+ar_IN
+ar_IQ
+ar_JO
+ar_KW
+ar_LB
+ar_LY
+ar_MA
+ar_OM
+ar_QA
+ar_SA
+ar_SD
+ar_SS
+ar_SY
+ar_TN
+ar_YE
+as_IN
+ast_ES
+ayc_PE
+az_AZ
+az_IR
+be_BY
+bem_ZM
+ber_DZ
+ber_MA
+bg_BG
+bhb_IN
+bho_IN
+bho_NP
+bi_VU
+bn_BD
+bn_IN
+bo_CN
+bo_IN
+br_FR
+brx_IN
+bs_BA
+byn_ER
+ca_AD
+ca_ES
+ca_FR
+ca_IT
+ce_RU
+ch_DE
+chr_US
+cmn_TW
+crh_UA
+cs_CZ
+csb_PL
+cv_RU
+cy_GB
+da_DK
+de_AT
+de_BE
+de_CH
+de_DE
+de_IT
+de_LI
+de_LU
+doi_IN
+dsb_DE
+dv_MV
+dz_BT
+el_CY
+el_GR
+en_AG
+en_AU
+en_BW
+en_CA
+en_DK
+en_GB
+en_HK
+en_IE
+en_IL
+en_IN
+en_NG
+en_NZ
+en_PH
+en_SC
+en_SG
+en_US
+en_ZA
+en_ZM
+en_ZW
+eo
+es_AR
+es_BO
+es_CL
+es_CO
+es_CR
+es_CU
+es_DO
+es_EC
+es_ES
+es_GT
+es_HN
+es_MX
+es_NI
+es_PA
+es_PE
+es_PR
+es_PY
+es_SV
+es_US
+es_UY
+es_VE
+et_EE
+eu_ES
+fa_IR
+ff_SN
+fi_FI
+fil_PH
+fo_FO
+fr_BE
+fr_CA
+fr_CH
+fr_FR
+fr_LU
+fur_IT
+fy_DE
+fy_NL
+ga_IE
+gd_GB
+gez_ER
+gez_ET
+gl_ES
+gu_IN
+gv_GB
+ha_NG
+hak_TW
+he_IL
+hi_IN
+hif_FJ
+hne_IN
+hr_HR
+hsb_DE
+ht_HT
+hu_HU
+hy_AM
+ia_FR
+id_ID
+ig_NG
+ik_CA
+is_IS
+it_CH
+it_IT
+iu_CA
+ja_JP
+ka_GE
+kab_DZ
+kk_KZ
+kl_GL
+km_KH
+kn_IN
+ko_KR
+kok_IN
+ks_IN
+ku_TR
+kw_GB
+ky_KG
+lb_LU
+lg_UG
+li_BE
+li_NL
+lij_IT
+ln_CD
+lo_LA
+lt_LT
+lv_LV
+lzh_TW
+mag_IN
+mai_IN
+mai_NP
+mfe_MU
+mg_MG
+mhr_RU
+mi_NZ
+miq_NI
+mjw_IN
+mk_MK
+ml_IN
+mn_MN
+mni_IN
+mnw_MM
+mr_IN
+ms_MY
+mt_MT
+my_MM
+nan_TW
+nb_NO
+nds_DE
+nds_NL
+ne_NP
+nhn_MX
+niu_NU
+niu_NZ
+nl_AW
+nl_BE
+nl_NL
+nn_NO
+nr_ZA
+nso_ZA
+oc_FR
+om_ET
+om_KE
+or_IN
+os_RU
+pa_IN
+pa_PK
+pap_AW
+pap_CW
+pl_PL
+ps_AF
+pt_BR
+pt_PT
+quz_PE
+raj_IN
+ro_RO
+ru_RU
+ru_UA
+rw_RW
+sa_IN
+sah_RU
+sat_IN
+sc_IT
+sd_IN
+se_NO
+sgs_LT
+shn_MM
+shs_CA
+si_LK
+sid_ET
+sk_SK
+sl_SI
+sm_WS
+so_DJ
+so_ET
+so_KE
+so_SO
+sq_AL
+sq_MK
+sr_ME
+sr_RS
+ss_ZA
+st_ZA
+sv_FI
+sv_SE
+sw_KE
+sw_TZ
+szl_PL
+ta_IN
+ta_LK
+tcy_IN
+te_IN
+tg_TJ
+th_TH
+the_NP
+ti_ER
+ti_ET
+tig_ER
+tk_TM
+tl_PH
+tn_ZA
+to_TO
+tpi_PG
+tr_CY
+tr_TR
+ts_ZA
+tt_RU
+ug_CN
+uk_UA
+unm_US
+ur_IN
+ur_PK
+uz_UZ
+ve_ZA
+vi_VN
+wa_BE
+wae_CH
+wal_ET
+wo_SN
+xh_ZA
+yi_US
+yo_NG
+yue_HK
+yuw_PG
+zh_CN
+zh_HK
+zh_SG
+zh_TW
+zu_ZA

pmb/export/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.export.frontend import frontend
 from pmb.export.odin import odin

pmb/export/odin.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os

pmb/export/symlinks.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os

pmb/flasher/__init__.py

@@ -1,6 +1,7 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.flasher.init import init
+from pmb.flasher.init import install_depends
 from pmb.flasher.run import run
 from pmb.flasher.run import check_partition_blacklist
 from pmb.flasher.variables import variables

pmb/flasher/frontend.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -88,11 +88,8 @@ def list_devices(args):
 
 
 def sideload(args):
-    method = args.flash_method or args.deviceinfo["flash_method"]
-    cfg = pmb.config.flashers[method]
-
     # Install depends
-    pmb.chroot.apk.install(args, cfg["depends"])
+    pmb.flasher.install_depends(args)
 
     # Mount the buildroot
     suffix = "buildroot_" + args.deviceinfo["arch"]
@@ -113,15 +110,37 @@ def sideload(args):
 
 
 def flash_lk2nd(args):
-    chroot_path = args.work + "/chroot_rootfs_" + args.device
-    lk2nd_path = "/boot/lk2nd.img"
-    if not os.path.exists(chroot_path + lk2nd_path):
-        raise RuntimeError(f"{chroot_path+lk2nd_path} doesn't exist. Your"
-                           " device may not support lk2nd.")
+    method = args.flash_method or args.deviceinfo["flash_method"]
+    if method == "fastboot":
+        # In the future this could be expanded to use "fastboot flash lk2nd $img"
+        # which reflashes/updates lk2nd from itself. For now let the user handle this
+        # manually since supporting the codepath with heimdall requires more effort.
+        pmb.flasher.init(args)
+        logging.info("(native) checking current fastboot product")
+        output = pmb.chroot.root(args, ["fastboot", "getvar", "product"],
+                                 output="interactive", output_return=True)
+        # Variable "product" is e.g. "LK2ND_MSM8974" or "lk2nd-msm8226" depending
+        # on the lk2nd version.
+        if "lk2nd" in output.lower():
+            raise RuntimeError("You are currently running lk2nd. Please reboot into the regular"
+                               " bootloader mode to re-flash lk2nd.")
 
-    logging.info(lk2nd_path)
-    logging.info("It's normal if fastboot warns"
-                 " \"Image not signed or corrupt\" or a similar warning")
+    # Get the lk2nd package (which is a dependency of the device package)
+    device_pkg = f"device-{args.device}"
+    apkbuild = pmb.helpers.pmaports.get(args, device_pkg)
+    lk2nd_pkg = None
+    for dep in apkbuild["depends"]:
+        if dep.startswith("lk2nd"):
+            lk2nd_pkg = dep
+            break
+
+    if not lk2nd_pkg:
+        raise RuntimeError(f"{device_pkg} does not depend on any lk2nd package")
+
+    suffix = "rootfs_" + args.device
+    pmb.chroot.apk.install(args, [lk2nd_pkg], suffix)
+
+    logging.info("(native) flash lk2nd image")
     pmb.flasher.run(args, "flash_lk2nd")
 
 
@@ -129,10 +148,6 @@ def frontend(args):
     action = args.action_flasher
     method = args.flash_method or args.deviceinfo["flash_method"]
 
-    # Legacy alias
-    if action == "flash_system":
-        action = "flash_rootfs"
-
     if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs",
                                        "flash_lk2nd"]:
         logging.info("This device doesn't support any flash method.")
@@ -140,17 +155,17 @@ def frontend(args):
 
     if action in ["boot", "flash_kernel"]:
         kernel(args)
-    if action == "flash_rootfs":
+    elif action == "flash_rootfs":
         rootfs(args)
-    if action == "flash_vbmeta":
+    elif action == "flash_vbmeta":
         flash_vbmeta(args)
-    if action == "flash_dtbo":
+    elif action == "flash_dtbo":
         flash_dtbo(args)
-    if action == "list_flavors":
-        list_flavors(args)
-    if action == "list_devices":
-        list_devices(args)
-    if action == "sideload":
-        sideload(args)
-    if action in ["flash_lk2nd"]:
+    elif action == "flash_lk2nd":
         flash_lk2nd(args)
+    elif action == "list_flavors":
+        list_flavors(args)
+    elif action == "list_devices":
+        list_devices(args)
+    elif action == "sideload":
+        sideload(args)

pmb/flasher/init.py

@@ -1,12 +1,12 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import pmb.config
 import pmb.chroot.apk
+import pmb.config
+import pmb.config.pmaports
 import pmb.helpers.mount
 
 
-def init(args):
-    # Validate method
+def install_depends(args):
     if hasattr(args, 'flash_method'):
         method = args.flash_method or args.deviceinfo["flash_method"]
     else:
@@ -20,10 +20,28 @@ def init(args):
                            "Make sure, it is packaged for Alpine Linux, or"
                            " package it yourself, and then add it to"
                            " pmb/config/__init__.py.")
-    cfg = pmb.config.flashers[method]
+    depends = pmb.config.flashers[method]["depends"]
 
-    # Install depends
-    pmb.chroot.apk.install(args, cfg["depends"])
+    # Depends for some flash methods may be different for various pmaports
+    # branches, so read them from pmaports.cfg.
+    if method == "fastboot":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_fastboot_depends",
+                                   "android-tools,avbtool").split(",")
+    elif method == "heimdall-bootimg":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_heimdall_depends",
+                                   "heimdall,avbtool").split(",")
+    elif method == "mtkclient":
+        pmaports_cfg = pmb.config.pmaports.read_config(args)
+        depends = pmaports_cfg.get("supported_mtkclient_depends",
+                                   "mtkclient,android-tools").split(",")
+
+    pmb.chroot.apk.install(args, depends)
+
+
+def init(args):
+    install_depends(args)
 
     # Mount folders from host system
     for folder in pmb.config.flash_mount_bind:

pmb/flasher/run.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.flasher
 import pmb.chroot.initfs

pmb/flasher/variables.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config.pmaports
 
@@ -10,11 +10,15 @@ def variables(args, flavor, method):
 
     flash_pagesize = args.deviceinfo['flash_pagesize']
 
+    # TODO Remove _partition_system deviceinfo support once pmaports has been
+    # updated and minimum pmbootstrap version bumped.
+    # See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243
+
     if method.startswith("fastboot"):
         _partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\
             or "boot"
-        _partition_system = args.deviceinfo["flash_fastboot_partition_system"]\
-            or "system"
+        _partition_rootfs = args.deviceinfo["flash_fastboot_partition_rootfs"]\
+            or args.deviceinfo["flash_fastboot_partition_system"] or "userdata"
         _partition_vbmeta = args.deviceinfo["flash_fastboot_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_fastboot_partition_dtbo"]\
@@ -23,15 +27,24 @@ def variables(args, flavor, method):
     elif method.startswith("rkdeveloptool"):
         _partition_kernel = args.deviceinfo["flash_rk_partition_kernel"]\
             or None
-        _partition_system = args.deviceinfo["flash_rk_partition_system"]\
-            or None
+        _partition_rootfs = args.deviceinfo["flash_rk_partition_rootfs"]\
+            or args.deviceinfo["flash_rk_partition_system"] or None
         _partition_vbmeta = None
         _partition_dtbo = None
+    elif method.startswith("mtkclient"):
+        _partition_kernel = args.deviceinfo["flash_mtkclient_partition_kernel"]\
+            or "boot"
+        _partition_rootfs = args.deviceinfo["flash_mtkclient_partition_rootfs"]\
+            or "userdata"
+        _partition_vbmeta = args.deviceinfo["flash_mtkclient_partition_vbmeta"]\
+            or None
+        _partition_dtbo = args.deviceinfo["flash_mtkclient_partition_dtbo"]\
+            or None
     else:
         _partition_kernel = args.deviceinfo["flash_heimdall_partition_kernel"]\
             or "KERNEL"
-        _partition_system = args.deviceinfo["flash_heimdall_partition_system"]\
-            or "SYSTEM"
+        _partition_rootfs = args.deviceinfo["flash_heimdall_partition_rootfs"]\
+            or args.deviceinfo["flash_heimdall_partition_system"] or "SYSTEM"
         _partition_vbmeta = args.deviceinfo["flash_heimdall_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_heimdall_partition_dtbo"]\
@@ -41,7 +54,7 @@ def variables(args, flavor, method):
         # Only one operation is done at same time so it doesn't matter
         # sharing the arg
         _partition_kernel = args.partition
-        _partition_system = args.partition
+        _partition_rootfs = args.partition
         _partition_vbmeta = args.partition
         _partition_dtbo = args.partition
 
@@ -59,7 +72,7 @@ def variables(args, flavor, method):
         "$PARTITION_KERNEL": _partition_kernel,
         "$PARTITION_INITFS": args.deviceinfo[
             "flash_heimdall_partition_initfs"] or "RECOVERY",
-        "$PARTITION_SYSTEM": _partition_system,
+        "$PARTITION_ROOTFS": _partition_rootfs,
         "$PARTITION_VBMETA": _partition_vbmeta,
         "$PARTITION_DTBO": _partition_dtbo,
         "$FLASH_PAGESIZE": flash_pagesize,

pmb/helpers/__init__.py

@@ -1,2 +1,2 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later

pmb/helpers/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Johannes Marbach, Oliver Smith
+# Copyright 2023 Johannes Marbach, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 
@@ -6,6 +6,7 @@ import pmb.chroot.root
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.run
+import pmb.helpers.run_core
 import pmb.parse.version
 
 
@@ -62,7 +63,7 @@ def _create_command_with_progress(command, fifo):
     """
     flags = ["--no-progress", "--progress-fd", "3"]
     command_full = [command[0]] + flags + command[1:]
-    command_flat = pmb.helpers.run.flat_cmd(command_full)
+    command_flat = pmb.helpers.run_core.flat_cmd(command_full)
     command_flat = f"exec 3>{fifo}; {command_flat}"
     return ["sh", "-c", command_flat]
 

pmb/helpers/aportupgrade.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Luca Weiss
+# Copyright 2023 Luca Weiss
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import fnmatch
@@ -34,7 +34,7 @@ def init_req_headers() -> None:
         return
     # Generic request headers
     req_headers = {
-        'User-Agent': f'pmbootstrap/{pmb.config.version} aportupgrade'}
+        'User-Agent': f'pmbootstrap/{pmb.__version__} aportupgrade'}
 
     # Request headers specific to GitHub
     req_headers_github = dict(req_headers)

pmb/helpers/args.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import copy
 import os

pmb/helpers/cli.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
 import logging

pmb/helpers/devices.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/helpers/file.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os

pmb/helpers/frontend.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import json
@@ -9,13 +9,14 @@ import sys
 import pmb.aportgen
 import pmb.build
 import pmb.build.autodetect
-import pmb.sideload
 import pmb.chroot
 import pmb.chroot.initfs
 import pmb.chroot.other
+import pmb.ci
 import pmb.config
 import pmb.export
 import pmb.flasher
+import pmb.helpers.aportupgrade
 import pmb.helpers.devices
 import pmb.helpers.git
 import pmb.helpers.lint
@@ -25,13 +26,13 @@ import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.helpers.repo_missing
 import pmb.helpers.run
-import pmb.helpers.aportupgrade
 import pmb.helpers.status
 import pmb.install
 import pmb.install.blockdevice
 import pmb.netboot
 import pmb.parse
 import pmb.qemu
+import pmb.sideload
 
 
 def _parse_flavor(args, autoinstall=True):
@@ -376,17 +377,17 @@ def newapkbuild(args):
 
 def kconfig(args):
     if args.action_kconfig == "check":
+        details = args.kconfig_check_details
+        # Build the components list from cli arguments (--waydroid etc.)
+        components_list = []
+        for name in pmb.parse.kconfig.get_all_component_names():
+            if getattr(args, f"kconfig_check_{name}"):
+                components_list += [name]
+
         # Handle passing a file directly
         if args.file:
-            if pmb.parse.kconfig.check_file(args.package,
-                                            waydroid=args.waydroid,
-                                            nftables=args.nftables,
-                                            containers=args.containers,
-                                            zram=args.zram,
-                                            netboot=args.netboot,
-                                            community=args.community,
-                                            uefi=args.uefi,
-                                            details=True):
+            if pmb.parse.kconfig.check_file(args.package, components_list,
+                                            details=details):
                 logging.info("kconfig check succeeded!")
                 return
             raise RuntimeError("kconfig check failed!")
@@ -413,17 +414,8 @@ def kconfig(args):
                 if "!pmb:kconfigcheck" in apkbuild["options"]:
                     skipped += 1
                     continue
-            if not pmb.parse.kconfig.check(
-                    args, package,
-                    force_waydroid_check=args.waydroid,
-                    force_iwd_check=args.iwd,
-                    force_nftables_check=args.nftables,
-                    force_containers_check=args.containers,
-                    force_zram_check=args.zram,
-                    force_netboot_check=args.netboot,
-                    force_community_check=args.community,
-                    force_uefi_check=args.uefi,
-                    details=True):
+            if not pmb.parse.kconfig.check(args, package, components_list,
+                                           details=details):
                 error = True
 
         # At least one failure
@@ -539,18 +531,25 @@ def work_migrate(args):
 
 
 def log(args):
+    log_testsuite = f"{args.work}/log_testsuite.txt"
+
     if args.clear_log:
         pmb.helpers.run.user(args, ["truncate", "-s", "0", args.log])
-    pmb.helpers.run.user(args, ["tail", "-n", args.lines, "-F", args.log],
-                         output="tui")
+        pmb.helpers.run.user(args, ["truncate", "-s", "0", log_testsuite])
 
+    cmd = ["tail", "-n", args.lines, "-F"]
 
-def log_distccd(args):
-    logpath = "/home/pmos/distccd.log"
-    if args.clear_log:
-        pmb.chroot.user(args, ["truncate", "-s", "0", logpath])
-    pmb.chroot.user(args, ["tail", "-n", args.lines, "-f", logpath],
-                    output="tui")
+    # Follow the testsuite's log file too if it exists. It will be created when
+    # starting a test case that writes to it (git -C test grep log_testsuite).
+    if os.path.exists(log_testsuite):
+        cmd += [log_testsuite]
+
+    # tail writes the last lines of the files to the terminal. Put the regular
+    # log at the end, so that output is visible at the bottom (where the user
+    # looks for an error / what's currently going on).
+    cmd += [args.log]
+
+    pmb.helpers.run.user(args, cmd, output="tui")
 
 
 def zap(args):
@@ -610,3 +609,47 @@ def lint(args):
 def status(args):
     if not pmb.helpers.status.print_status(args, args.details):
         sys.exit(1)
+
+
+def ci(args):
+    topdir = pmb.helpers.git.get_topdir(args, os.getcwd())
+    if not os.path.exists(topdir):
+        logging.error("ERROR: change your current directory to a git"
+                      " repository (e.g. pmbootstrap, pmaports) before running"
+                      " 'pmbootstrap ci'.")
+        exit(1)
+
+    scripts_available = pmb.ci.get_ci_scripts(topdir)
+    scripts_available = pmb.ci.sort_scripts_by_speed(scripts_available)
+    if not scripts_available:
+        logging.error("ERROR: no supported CI scripts found in current git"
+                      " repository, see https://postmarketos.org/pmb-ci")
+        exit(1)
+
+    scripts_selected = {}
+    if args.scripts:
+        if args.all:
+            raise RuntimeError("Combining --all with script names doesn't"
+                               " make sense")
+        for script in args.scripts:
+            if script not in scripts_available:
+                logging.error(f"ERROR: script '{script}' not found in git"
+                              " repository, found these:"
+                              f" {', '.join(scripts_available.keys())}")
+                exit(1)
+            scripts_selected[script] = scripts_available[script]
+    elif args.all:
+        scripts_selected = scripts_available
+
+    if args.fast:
+        for script, script_data in scripts_available.items():
+            if "slow" not in script_data["options"]:
+                scripts_selected[script] = script_data
+
+    if not pmb.helpers.git.clean_worktree(args, topdir):
+        logging.warning("WARNING: this git repository has uncommitted changes")
+
+    if not scripts_selected:
+        scripts_selected = pmb.ci.ask_which_scripts_to_run(scripts_available)
+
+    pmb.ci.run_scripts(args, topdir, scripts_selected)

pmb/helpers/git.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 import logging
@@ -246,3 +246,29 @@ def is_outdated(path):
 
     date_outdated = time.time() - pmb.config.git_repo_outdated
     return date_head <= date_outdated
+
+
+def get_topdir(args, path):
+    """ :returns: a string with the top dir of the git repository, or an
+                  empty string if it's not a git repository. """
+    return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"],
+                                path, output_return=True, check=False).rstrip()
+
+
+def get_files(args, path):
+    """ Get all files inside a git repository, that are either already in the
+        git tree or are not in gitignore. Do not list deleted files. To be used
+        for creating a tarball of the git repository.
+        :param path: top dir of the git repository
+        :returns: all files in a git repository as list, relative to path """
+    ret = []
+    files = pmb.helpers.run.user(args, ["git", "ls-files"], path,
+                                 output_return=True).split("\n")
+    files += pmb.helpers.run.user(args, ["git", "ls-files",
+                                  "--exclude-standard", "--other"], path,
+                                  output_return=True).split("\n")
+    for file in files:
+        if os.path.exists(f"{path}/{file}"):
+            ret += [file]
+
+    return ret

pmb/helpers/http.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import hashlib
 import json

pmb/helpers/lint.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Danct12 <danct12@disroot.org>
+# Copyright 2023 Danct12 <danct12@disroot.org>
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -17,7 +17,6 @@ def check(args, pkgnames):
     :param pkgnames: Names of the packages to lint
     """
     pmb.chroot.apk.install(args, ["atools"])
-    pmb.build.init(args)
 
     # Mount pmaports.git inside the chroot so that we don't have to copy the
     # package folders

pmb/helpers/logging.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os

pmb/helpers/mount.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import pmb.helpers.run

pmb/helpers/other.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -54,9 +54,12 @@ def check_binfmt_misc(args):
     if os.path.exists(path):
         return
 
+    # check=False: this might be built-in instead of being a module
     pmb.helpers.run.root(args, ["modprobe", "binfmt_misc"], check=False)
+
+    # check=False: we check it below and print a more helpful message on error
     pmb.helpers.run.root(args, ["mount", "-t", "binfmt_misc", "none",
-                                "/proc/sys/fs/binfmt_misc"])
+                                "/proc/sys/fs/binfmt_misc"], check=False)
 
     if not os.path.exists(path):
         link = "https://postmarketos.org/binfmt_misc"
@@ -264,16 +267,17 @@ def validate_hostname(hostname):
         return False
 
     # Check that it only contains valid chars
-    if not re.match("^[0-9a-z-]*$", hostname):
+    if not re.match(r"^[0-9a-z-\.]*$", hostname):
         logging.fatal("ERROR: Hostname must only contain letters (a-z),"
-                      " digits (0-9) or minus signs (-)")
+                      " digits (0-9), minus signs (-), or periods (.)")
         return False
 
-    # Check that doesn't begin or end with a minus sign
-    if hostname[:1] == "-" or hostname[-1:] == "-":
+    # Check that doesn't begin or end with a minus sign or period
+    if re.search(r"^-|^\.|-$|\.$", hostname):
         logging.fatal("ERROR: Hostname must not begin or end with a minus"
-                      " sign")
+                      " sign or period")
         return False
+
     return True
 
 

pmb/helpers/package.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with both pmaports and binary package repos. See also:

pmb/helpers/pkgrel_bump.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 

pmb/helpers/pmaports.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with pmaports. See also:

pmb/helpers/repo.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """
 Functions that work with binary package repos. See also:
@@ -43,10 +43,10 @@ def hash(url, length=8):
 def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
     """
     Get a list of repository URLs, as they are in /etc/apk/repositories.
-    :param user_repository: add /mnt/pmbootstrap-packages
+    :param user_repository: add /mnt/pmbootstrap/packages
     :param postmarketos_mirror: add postmarketos mirror URLs
     :param alpine: add alpine mirror URLs
-    :returns: list of mirror strings, like ["/mnt/pmbootstrap-packages",
+    :returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
                                             "http://...", ...]
     """
     ret = []
@@ -59,7 +59,7 @@ def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
 
     # Local user repository (for packages compiled with pmbootstrap)
     if user_repository:
-        ret.append("/mnt/pmbootstrap-packages")
+        ret.append("/mnt/pmbootstrap/packages")
 
     # Upstream postmarketOS binary repository
     if postmarketos_mirror:

pmb/helpers/repo_missing.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 

pmb/helpers/run.py

@@ -1,38 +1,8 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import shlex
 import pmb.helpers.run_core
 
 
-def flat_cmd(cmd, working_dir=None, env={}):
-    """
-    Convert a shell command passed as list into a flat shell string with
-    proper escaping.
-
-    :param cmd: command as list, e.g. ["echo", "string with spaces"]
-    :param working_dir: when set, prepend "cd ...;" to execute the command
-                        in the given working directory
-    :param env: dict of environment variables to be passed to the command, e.g.
-                {"JOBS": "5"}
-    :returns: the flat string, e.g.
-              echo 'string with spaces'
-              cd /home/pmos;echo 'string with spaces'
-    """
-    # Merge env and cmd into escaped list
-    escaped = []
-    for key, value in env.items():
-        escaped.append(key + "=" + shlex.quote(value))
-    for i in range(len(cmd)):
-        escaped.append(shlex.quote(cmd[i]))
-
-    # Prepend working dir
-    ret = " ".join(escaped)
-    if working_dir:
-        ret = "cd " + shlex.quote(working_dir) + ";" + ret
-
-    return ret
-
-
 def user(args, cmd, working_dir=None, output="log", output_return=False,
          check=None, env={}, sudo=False):
     """
@@ -54,7 +24,7 @@ def user(args, cmd, working_dir=None, output="log", output_return=False,
 
     # Add environment variables and run
     if env:
-        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
+        cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
     return pmb.helpers.run_core.core(args, msg, cmd, working_dir, output,
                                      output_return, check, sudo)
 
@@ -71,8 +41,8 @@ def root(args, cmd, working_dir=None, output="log", output_return=False,
     arguments and the return value.
     """
     if env:
-        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
-    cmd = [pmb.config.sudo] + cmd
+        cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
+    cmd = pmb.config.sudo(cmd)
 
     return user(args, cmd, working_dir, output, output_return, check, env,
                 True)

pmb/helpers/run_core.py

@@ -1,13 +1,14 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fcntl
 import logging
+import os
 import selectors
+import shlex
 import subprocess
 import sys
 import threading
 import time
-import os
 import pmb.helpers.run
 
 """ For a detailed description of all output modes, read the description of
@@ -15,6 +16,35 @@ import pmb.helpers.run
     called by core(). """
 
 
+def flat_cmd(cmd, working_dir=None, env={}):
+    """
+    Convert a shell command passed as list into a flat shell string with
+    proper escaping.
+
+    :param cmd: command as list, e.g. ["echo", "string with spaces"]
+    :param working_dir: when set, prepend "cd ...;" to execute the command
+                        in the given working directory
+    :param env: dict of environment variables to be passed to the command, e.g.
+                {"JOBS": "5"}
+    :returns: the flat string, e.g.
+              echo 'string with spaces'
+              cd /home/pmos;echo 'string with spaces'
+    """
+    # Merge env and cmd into escaped list
+    escaped = []
+    for key, value in env.items():
+        escaped.append(key + "=" + shlex.quote(value))
+    for i in range(len(cmd)):
+        escaped.append(shlex.quote(cmd[i]))
+
+    # Prepend working dir
+    ret = " ".join(escaped)
+    if working_dir:
+        ret = "cd " + shlex.quote(working_dir) + ";" + ret
+
+    return ret
+
+
 def sanity_checks(output="log", output_return=False, check=None):
     """
     Raise an exception if the parameters passed to core() don't make sense
@@ -227,10 +257,10 @@ def sudo_timer_iterate():
     Run sudo -v and schedule a new timer to repeat the same.
     """
 
-    if pmb.config.sudo == "sudo":
+    if pmb.config.which_sudo() == "sudo":
         subprocess.Popen(["sudo", "-v"]).wait()
     else:
-        subprocess.Popen([pmb.config.sudo, "true"]).wait()
+        subprocess.Popen(pmb.config.sudo(["true"])).wait()
 
     timer = threading.Timer(interval=60, function=sudo_timer_iterate)
     timer.daemon = True
@@ -288,7 +318,7 @@ def core(args, log_message, cmd, working_dir=None, output="log",
                    output value  | timeout | out to log | out to stdout | wait | pass stdin
                    ------------------------------------------------------------------------
                    "log"         | x       | x          |               | x    |
-                   "stdout"      | x       | x          | x             | x    | x
+                   "stdout"      | x       | x          | x             | x    |
                    "interactive" |         | x          | x             | x    | x
                    "tui"         |         |            | x             | x    | x
                    "background"  |         | x          |               |      |
@@ -310,6 +340,15 @@ def core(args, log_message, cmd, working_dir=None, output="log",
     """
     sanity_checks(output, output_return, check)
 
+    # Preserve proxy environment variables
+    env = {}
+    for var in ["FTP_PROXY", "ftp_proxy", "HTTP_PROXY", "http_proxy",
+                "HTTPS_PROXY", "https_proxy", "HTTP_PROXY_AUTH"]:
+        if var in os.environ:
+            env[var] = os.environ[var]
+    if env:
+        cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
+
     if args.sudo_timer and sudo:
         sudo_timer_start()
 
@@ -338,7 +377,7 @@ def core(args, log_message, cmd, working_dir=None, output="log",
 
         output_timeout = output in ["log", "stdout"] and not disable_timeout
 
-        stdin = subprocess.DEVNULL if output == "log" else None
+        stdin = subprocess.DEVNULL if output in ["log", "stdout"] else None
 
         (code, output_after_run) = foreground_pipe(args, cmd, working_dir,
                                                    output_to_stdout,

pmb/helpers/status.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/helpers/ui.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Clayton Craft
+# Copyright 2023 Clayton Craft
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob

pmb/install/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.install._install import install
 from pmb.install._install import get_kernel_package

pmb/install/_install.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -164,8 +164,8 @@ def create_home_from_skel(args):
 def configure_apk(args):
     """
     Copy over all official keys, and the keys used to compile local packages
-    (unless --no-local-pkgs is set). Then disable the /mnt/pmbootstrap-packages
-    repository.
+    (unless --no-local-pkgs is set). Then copy the corresponding APKINDEX files
+    and remove the /mnt/pmbootstrap/packages repository.
     """
     # Official keys
     pattern = f"{pmb.config.apk_keys_path}/*.pub"
@@ -179,8 +179,15 @@ def configure_apk(args):
     for key in glob.glob(pattern):
         pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"])
 
+    # Copy over the corresponding APKINDEX files from cache
+    index_files = pmb.helpers.repo.apkindex_files(args,
+                                                  arch=args.deviceinfo["arch"],
+                                                  user_repository=False)
+    for f in index_files:
+        pmb.helpers.run.root(args, ["cp", f, rootfs + "/var/cache/apk/"])
+
     # Disable pmbootstrap repository
-    pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap-packages/d",
+    pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap\/packages/d",
                                 rootfs + "/etc/apk/repositories"])
     pmb.helpers.run.user(args, ["cat", rootfs + "/etc/apk/repositories"])
 
@@ -197,7 +204,13 @@ def set_user(args):
     if not pmb.chroot.user_exists(args, args.user, suffix):
         pmb.chroot.root(args, ["adduser", "-D", "-u", "10000", args.user],
                         suffix)
-    groups = pmb.install.ui.get_groups(args) + pmb.config.install_user_groups
+
+    pmaports_cfg = pmb.config.pmaports.read_config(args)
+    groups = []
+    groups += pmaports_cfg.get("install_user_groups",
+                               "audio,input,netdev,plugdev,video,wheel").split(",")
+    groups += pmb.install.ui.get_groups(args)
+
     for group in groups:
         pmb.chroot.root(args, ["addgroup", "-S", group], suffix,
                         check=False)
@@ -347,6 +360,26 @@ def setup_keymap(args):
         logging.info("NOTE: No valid keymap specified for device")
 
 
+def setup_timezone(args):
+    suffix = f"rootfs_{args.device}"
+
+    arch = args.deviceinfo["arch"]
+    alpine_conf = pmb.helpers.package.get(args, "alpine-conf", arch)
+    version = alpine_conf["version"].split("-r")[0]
+
+    setup_tz_cmd = ["setup-timezone"]
+    # setup-timezone will, by default, copy the timezone to /etc/zoneinfo
+    # and disregard tzdata, to save space. If we actually have tzdata
+    # installed, make sure that setup-timezone makes use of it, since
+    # there's no space to be saved.
+    if "tzdata" in pmb.chroot.apk.installed(args, suffix):
+        setup_tz_cmd += ["-i"]
+    if not pmb.parse.version.check_string(version, ">=3.14.0"):
+        setup_tz_cmd += ["-z"]
+    setup_tz_cmd += [args.timezone]
+    pmb.chroot.root(args, setup_tz_cmd, suffix)
+
+
 def setup_hostname(args):
     """
     Set the hostname and update localhost address in /etc/hosts
@@ -370,6 +403,25 @@ def setup_hostname(args):
     pmb.chroot.root(args, ["sed", "-i", "-e", regex, "/etc/hosts"], suffix)
 
 
+def setup_appstream(args):
+    """
+    If alpine-appstream-downloader has been downloaded, execute it to have
+    update AppStream data on new installs
+    """
+    suffix = "rootfs_" + args.device
+    installed_pkgs = pmb.chroot.apk.installed(args, suffix)
+
+    if "alpine-appstream-downloader" not in installed_pkgs or args.offline:
+        return
+
+    pmb.chroot.root(args, ["alpine-appstream-downloader",
+                           "/mnt/appstream-data"], suffix)
+    pmb.chroot.root(args, ["mkdir", "-p", "/var/lib/swcatalog"], suffix)
+    pmb.chroot.root(args, ["cp", "-r", "/mnt/appstream-data/icons",
+                           "/mnt/appstream-data/xml",
+                           "-t", "/var/lib/swcatalog"], suffix)
+
+
 def disable_sshd(args):
     if not args.no_sshd:
         return
@@ -560,7 +612,7 @@ def write_cgpt_kpart(args, layout, suffix):
     :param layout: partition layout from get_partition_layout()
     :param suffix: of the chroot, which holds the image file to be flashed
     """
-    if not args.deviceinfo["cgpt_kpart"]:
+    if not args.deviceinfo["cgpt_kpart"] or not args.install_cgpt:
         return
 
     device_rootfs = mount_device_rootfs(args, suffix)
@@ -648,6 +700,74 @@ def get_partition_layout(reserve, kernel):
     return ret
 
 
+def get_uuid(args, partition):
+    """
+    Get UUID of a partition
+
+    :param partition: block device for getting UUID from
+    """
+    return pmb.chroot.root(
+        args,
+        [
+            "blkid",
+            "-s", "UUID",
+            "-o", "value",
+            partition,
+        ],
+        output_return=True
+    ).rstrip()
+
+
+def create_crypttab(args, layout, suffix):
+    """
+    Create /etc/crypttab config
+
+    :param layout: partition layout from get_partition_layout()
+    :param suffix: of the chroot, which crypttab will be created to
+    """
+
+    luks_uuid = get_uuid(args, f"/dev/installp{layout['root']}")
+
+    crypttab = f"root UUID={luks_uuid} none luks\n"
+
+    open(f"{args.work}/chroot_{suffix}/tmp/crypttab", "w").write(crypttab)
+    pmb.chroot.root(args, ["mv", "/tmp/crypttab", "/etc/crypttab"], suffix)
+
+
+def create_fstab(args, layout, suffix):
+    """
+    Create /etc/fstab config
+
+    :param layout: partition layout from get_partition_layout()
+    :param suffix: of the chroot, which fstab will be created to
+    """
+
+    # Do not install fstab into target rootfs when using on-device
+    # installer. Provide fstab only to installer suffix
+    if args.on_device_installer and "rootfs_" in suffix:
+        return
+
+    boot_dev = f"/dev/installp{layout['boot']}"
+    root_dev = f"/dev/installp{layout['root']}"
+
+    boot_mount_point = f"UUID={get_uuid(args, boot_dev)}"
+    root_mount_point = "/dev/mapper/root" if args.full_disk_encryption \
+        else f"UUID={get_uuid(args, root_dev)}"
+
+    boot_filesystem = args.deviceinfo["boot_filesystem"] or "ext2"
+    root_filesystem = pmb.install.get_root_filesystem(args)
+
+    fstab = f"""
+# <file system> <mount point> <type> <options> <dump> <pass>
+{root_mount_point} / {root_filesystem} defaults 0 0
+{boot_mount_point} /boot {boot_filesystem} defaults 0 0
+""".lstrip()
+
+    with open(f"{args.work}/chroot_{suffix}/tmp/fstab", "w") as f:
+        f.write(fstab)
+    pmb.chroot.root(args, ["mv", "/tmp/fstab", "/etc/fstab"], suffix)
+
+
 def install_system_image(args, size_reserve, suffix, step, steps,
                          boot_label="pmOS_boot", root_label="pmOS_root",
                          split=False, sdcard=None):
@@ -666,12 +786,13 @@ def install_system_image(args, size_reserve, suffix, step, steps,
     logging.info(f"*** ({step}/{steps}) PREPARE INSTALL BLOCKDEVICE ***")
     pmb.chroot.shutdown(args, True)
     (size_boot, size_root) = get_subpartitions_size(args, suffix)
-    layout = get_partition_layout(size_reserve, args.deviceinfo["cgpt_kpart"])
+    layout = get_partition_layout(size_reserve, args.deviceinfo["cgpt_kpart"] \
+             and args.install_cgpt)
     if not args.rsync:
         pmb.install.blockdevice.create(args, size_boot, size_root,
                                        size_reserve, split, sdcard)
         if not split:
-            if args.deviceinfo["cgpt_kpart"]:
+            if args.deviceinfo["cgpt_kpart"] and args.install_cgpt:
                 pmb.install.partition_cgpt(
                     args, layout, size_boot, size_reserve)
             else:
@@ -681,6 +802,22 @@ def install_system_image(args, size_reserve, suffix, step, steps,
 
     pmb.install.format(args, layout, boot_label, root_label, sdcard)
 
+    # Create /etc/fstab and /etc/crypttab
+    logging.info("(native) create /etc/fstab")
+    create_fstab(args, layout, suffix)
+    if args.full_disk_encryption:
+        logging.info("(native) create /etc/crypttab")
+        create_crypttab(args, layout, suffix)
+
+    # Run mkinitfs to pass UUIDs to cmdline
+    logging.info(f"({suffix}) mkinitfs")
+    pmb.chroot.root(args, ["mkinitfs"], suffix)
+
+    # Clean up after running mkinitfs in chroot
+    pmb.helpers.mount.umount_all(args, f"{args.work}/chroot_{suffix}")
+    pmb.helpers.run.root(args, ["rm", f"{args.work}/chroot_{suffix}/in-pmbootstrap"])
+    pmb.chroot.remove_mnt_pmbootstrap(args, suffix)
+
     # Just copy all the files
     logging.info(f"*** ({step + 1}/{steps}) FILL INSTALL BLOCKDEVICE ***")
     copy_files_from_chroot(args, suffix)
@@ -867,7 +1004,7 @@ def install_on_device_installer(args, step, steps):
            "ONDEV_CHANNEL_DESCRIPTION": channel_cfg["description"],
            "ONDEV_CHANNEL_MIRRORDIR_ALPINE": channel_cfg["mirrordir_alpine"],
            "ONDEV_CIPHER": args.cipher,
-           "ONDEV_PMBOOTSTRAP_VERSION": pmb.config.version,
+           "ONDEV_PMBOOTSTRAP_VERSION": pmb.__version__,
            "ONDEV_UI": args.ui}
     pmb.chroot.root(args, ["ondev-prepare"], suffix_installer, env=env)
 
@@ -995,24 +1132,22 @@ def create_device_rootfs(args, step, steps):
     setup_keymap(args)
 
     # Set timezone
-    arch = args.deviceinfo["arch"]
-    package = pmb.helpers.package.get(args, "alpine-conf", arch)
-    version = package["version"].split("-r")[0]
-
-    if not pmb.parse.version.check_string(version, ">=3.14.0"):
-        pmb.chroot.root(args, ["setup-timezone", "-z", args.timezone], suffix)
-    else:
-        pmb.chroot.root(args, ["setup-timezone", args.timezone], suffix)
+    setup_timezone(args)
 
     # Set locale
     if locale_is_set:
-        pmb.chroot.root(args, ["sed", "-i",
-                               f"s/LANG=C.UTF-8/LANG={args.locale}/",
-                               "/etc/profile.d/locale.sh"], suffix)
+        # 10locale-pmos.sh gets sourced before 20locale.sh from
+        # alpine-baselayout by /etc/profile. Since they don't override the
+        # locale if it exists, it warranties we have preference
+        line = f"export LANG=${{LANG:-{shlex.quote(args.locale)}}}"
+        pmb.chroot.root(args, ["sh", "-c", f"echo {shlex.quote(line)}"
+                               " > /etc/profile.d/10locale-pmos.sh"], suffix)
 
     # Set the hostname as the device name
     setup_hostname(args)
 
+    setup_appstream(args)
+
     disable_sshd(args)
     disable_firewall(args)
 

pmb/install/blockdevice.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os

pmb/install/format.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -103,7 +103,7 @@ def format_and_mount_root(args, device, root_label, sdcard):
             mkfs_root_args = ["mkfs.ext4", "-O", "^metadata_csum", "-F",
                               "-q", "-L", root_label]
             # When we don't know the file system size before hand like
-            # with non-block devices, we need to explicitely set a number of
+            # with non-block devices, we need to explicitly set a number of
             # inodes. See #1717 and #1845 for details
             if not sdcard:
                 mkfs_root_args = mkfs_root_args + ["-N", "100000"]

pmb/install/losetup.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import json

pmb/install/partition.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -97,6 +97,15 @@ def partition(args, layout, size_boot, size_reserve):
         ["set", str(layout["boot"]), "boot", "on"]
     ]
 
+    # Not strictly necessary if the device doesn't use EFI boot, but marking
+    # it as an ESP will cover all situations where the device does use EFI
+    # boot. Marking it as ESP is helpful for EFI fw when it's looking for EFI
+    # system partitions. It's assumed that setting this bit is unlikely to
+    # cause problems for other situations, like when using Legacy BIOS boot
+    # or u-boot.
+    if partition_type.lower() == "gpt":
+        commands += [["set", str(layout["boot"]), "esp", "on"]]
+
     for command in commands:
         pmb.chroot.root(args, ["parted", "-s", "/dev/install"] +
                         command, check=False)
@@ -112,7 +121,7 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
     :param size_reserve: empty partition between root and boot in MiB (pma#463)
     """
 
-    pmb.chroot.root(args, ["apk", "add", "cgpt"])
+    pmb.chroot.apk.install(args, ["cgpt"], build=False)
 
     cgpt = {
         'kpart_start': args.deviceinfo["cgpt_kpart_start"],
@@ -136,27 +145,6 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
     commands = [
         ["parted", "-s", "/dev/install", "mktable", "gpt"],
         ["cgpt", "create", "/dev/install"],
-        [
-            "cgpt", "add",
-            # pmOS_boot is second partition, the first will be ChromeOS kernel
-            # partition
-            "-i", str(layout["boot"]),  # Partition number
-            "-t", "data",
-            "-b", boot_part_start,
-            "-s", s_boot,
-            "-l", "pmOS_boot",
-            "/dev/install"
-        ],
-        # Mark this partition as bootable for u-boot
-        [
-            "parted",
-            "-s", "/dev/install",
-            "set", str(layout["boot"]),
-            "boot", "on"
-        ],
-        # For some reason cgpt switches all flags to 0 after marking
-        # any partition as bootable, so create ChromeOS kernel partition
-        # only after pmOS_boot is created and marked as bootable
         [
             "cgpt", "add",
             "-i", str(layout["kernel"]),
@@ -169,11 +157,23 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
             "-P", "10",  # Priority flag
             "/dev/install"
         ],
+        [
+            "cgpt", "add",
+            # pmOS_boot is second partition, the first will be ChromeOS kernel
+            # partition
+            "-i", str(layout["boot"]),  # Partition number
+            "-t", "efi", # Mark this partition as bootable for u-boot
+            "-b", boot_part_start,
+            "-s", s_boot,
+            "-l", "pmOS_boot",
+            "/dev/install"
+        ],
     ]
 
     dev_size = pmb.chroot.root(
         args, ["blockdev", "--getsz", "/dev/install"], output_return=True)
-    root_size = str(int(dev_size) - int(s_root_start) - 1024)
+    # 33: Sec GPT table (32) + Sec GPT header (1)
+    root_size = str(int(dev_size) - int(s_root_start) - 33)
 
     commands += [
         [

pmb/install/recovery.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Attila Szollosi
+# Copyright 2023 Attila Szollosi
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 
@@ -35,7 +35,8 @@ def create_zip(args, suffix):
         "ISOREC": method == "heimdall-isorec",
         "KERNEL_PARTLABEL": vars["$PARTITION_KERNEL"],
         "INITFS_PARTLABEL": vars["$PARTITION_INITFS"],
-        "SYSTEM_PARTLABEL": vars["$PARTITION_SYSTEM"],
+        # Name is still "SYSTEM", not "ROOTFS" in the recovery installer
+        "SYSTEM_PARTLABEL": vars["$PARTITION_ROOTFS"],
         "INSTALL_PARTITION": args.recovery_install_partition,
         "CIPHER": args.cipher,
         "FDE": args.full_disk_encryption,

pmb/install/ui.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Dylan Van Assche
+# Copyright 2023 Dylan Van Assche
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 

pmb/netboot/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Mark Hargreaves, Luca Weiss
+# Copyright 2023 Mark Hargreaves, Luca Weiss
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -20,7 +20,7 @@ def start_nbd_server(args, ip="172.16.42.2", port=9999):
 
     chroot = f"{args.work}/chroot_native"
 
-    rootfs_path = f"/mnt/pmbootstrap-netboot/{args.device}.img"
+    rootfs_path = f"/mnt/pmbootstrap/netboot/{args.device}.img"
     if not os.path.exists(chroot + rootfs_path) or args.replace:
         rootfs_path2 = f"/home/pmos/rootfs/{args.device}.img"
         if not os.path.exists(chroot + rootfs_path2):

pmb/parse/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.parse.arguments import arguments
 from pmb.parse._apkbuild import apkbuild

pmb/parse/_apkbuild.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -320,8 +320,7 @@ def apkbuild(path, check_pkgver=True, check_pkgname=True):
 
     # Sanity check: pkgver
     if check_pkgver:
-        if ("-r" in ret["pkgver"] or not
-           pmb.parse.version.validate(ret["pkgver"])):
+        if not pmb.parse.version.validate(ret["pkgver"]):
             logging.info(
                 "NOTE: Valid pkgvers are described here: "
                 "https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#pkgver")

pmb/parse/apkindex.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import collections
 import logging
@@ -308,7 +308,7 @@ def providers(args, package, arch=None, must_exist=True, indexes=None):
                                     " higher)")
                     continue
 
-            # Add the provier to ret
+            # Add the provider to ret
             logging.verbose(package + ": provided by: " + provider_pkgname +
                             "-" + version + " in " + path)
             ret[provider_pkgname] = provider
@@ -339,7 +339,7 @@ def provider_highest_priority(providers, pkgname):
 
     if priority_providers:
         logging.debug(
-            f"{pkgname}: picked provider(s) with higest priority "
+            f"{pkgname}: picked provider(s) with highest priority "
             f"{max_priority}: {', '.join(priority_providers.keys())}")
         return priority_providers
 

pmb/parse/arch.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fnmatch
 import platform
@@ -12,8 +12,9 @@ def alpine_native():
         "i686": "x86",
         "x86_64": "x86_64",
         "aarch64": "aarch64",
+        "arm64": "aarch64",
         "armv6l": "armhf",
-        "armv7l": "armv7"
+        "armv7l": "armv7",
     }
     if machine in mapping:
         return mapping[machine]
@@ -36,6 +37,7 @@ def from_chroot_suffix(args, suffix):
 def alpine_to_qemu(arch):
     """
     Convert the architecture to the string used in the QEMU packaging.
+    This corresponds to the package name of e.g. qemu-system-aarch64.
     """
 
     mapping = {
@@ -44,6 +46,7 @@ def alpine_to_qemu(arch):
         "armhf": "arm",
         "armv7": "arm",
         "aarch64": "aarch64",
+        "riscv64": "riscv64",
     }
     for pattern, arch_qemu in mapping.items():
         if fnmatch.fnmatch(arch, pattern):
@@ -61,7 +64,8 @@ def alpine_to_kernel(arch):
         "aarch64*": "arm64",
         "arm*": "arm",
         "ppc*": "powerpc",
-        "s390*": "s390"
+        "s390*": "s390",
+        "riscv64*": "riscv",
     }
     for pattern, arch_kernel in mapping.items():
         if fnmatch.fnmatch(arch, pattern):
@@ -78,6 +82,9 @@ def alpine_to_hostspec(arch):
         "armel": "armv5-alpine-linux-musleabi",
         "armhf": "armv6-alpine-linux-musleabihf",
         "armv7": "armv7-alpine-linux-musleabihf",
+        "loongarch32": "loongarch32-alpine-linux-musl",
+        "loongarchx32": "loongarchx32-alpine-linux-musl",
+        "loongarch64": "loongarch64-alpine-linux-musl",
         "mips": "mips-alpine-linux-musl",
         "mips64": "mips64-alpine-linux-musl",
         "mipsel": "mipsel-alpine-linux-musl",
@@ -85,6 +92,8 @@ def alpine_to_hostspec(arch):
         "ppc": "powerpc-alpine-linux-musl",
         "ppc64": "powerpc64-alpine-linux-musl",
         "ppc64le": "powerpc64le-alpine-linux-musl",
+        "riscv32": "riscv32-alpine-linux-musl",
+        "riscv64": "riscv64-alpine-linux-musl",
         "s390x": "s390x-alpine-linux-musl",
         "x86": "i586-alpine-linux-musl",
         "x86_64": "x86_64-alpine-linux-musl",

pmb/parse/arguments.py

@@ -1,4 +1,4 @@
-# Copyright 2022 Oliver Smith
+# Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import argparse
 import copy
@@ -79,6 +79,8 @@ def arguments_install(subparser):
                      " installation - will be handled in PLAIN TEXT during"
                      " install and may be logged to the logfile, do not use an"
                      " important password!")
+    ret.add_argument("--no-cgpt", help="do not use cgpt partition table",
+                     dest="install_cgpt", action="store_false", default=True)
 
     # Image type
     group_desc = ret.add_argument_group(
@@ -250,13 +252,13 @@ def arguments_flasher(subparser):
                              " default boot image partition ")
 
     # Flash rootfs
-    flash_rootfs = sub.add_parser("flash_rootfs", aliases=["flash_system"],
+    flash_rootfs = sub.add_parser("flash_rootfs",
                                   help="flash the rootfs to a partition on the"
                                   " device (partition layout does not get"
                                   " changed)")
     flash_rootfs.add_argument("--partition", default=None,
                               help="partition to flash the rootfs to (defaults"
-                                   " to deviceinfo_flash_*_partition_system,"
+                                   " to deviceinfo_flash_*_partition_rootfs,"
                                    " 'userdata' on Android may have more"
                                    " space)")
 
@@ -366,6 +368,8 @@ def arguments_qemu(subparser):
     ret.add_argument("--host-qemu", dest="host_qemu", action='store_true',
                      help="Use the host system's qemu")
 
+    ret.add_argument("--efi", action="store_true",
+                     help="Use EFI boot (default: direct kernel image boot)")
     return ret
 
 
@@ -464,23 +468,14 @@ def arguments_kconfig(subparser):
     check.add_argument("--arch", choices=arch_choices, dest="arch")
     check.add_argument("--file", action="store_true", help="check a file"
                        " directly instead of a config in a package")
-    check.add_argument("--waydroid", action="store_true", help="check"
-                       " options needed for waydroid too")
-    check.add_argument("--iwd", action="store_true", help="check"
-                       " options needed for iwd too")
-    check.add_argument("--nftables", action="store_true", help="check"
-                       " options needed for nftables too")
-    check.add_argument("--containers", action="store_true",
-                       help="check options needed for containers too")
-    check.add_argument("--zram", action="store_true", help="check"
-                       " options needed for zram support too")
-    check.add_argument("--netboot", action="store_true", help="check"
-                       " options needed for netbooting too")
-    check.add_argument("--community", action="store_true", help="check"
-                       " options needed for various features, required for"
-                       " community/main devices")
-    check.add_argument("--uefi", action="store_true", help="check"
-                       " options needed for uefi too")
+    check.add_argument("--no-details", action="store_false",
+                       dest="kconfig_check_details",
+                       help="print one generic error per component instead of"
+                            " listing each option that needs to be adjusted")
+    for name in pmb.parse.kconfig.get_all_component_names():
+        check.add_argument(f"--{name}", action="store_true",
+                           dest=f"kconfig_check_{name}",
+                           help=f"check options needed for {name} too")
     add_kernel_arg(check)
 
     # "pmbootstrap kconfig edit"
@@ -546,6 +541,21 @@ def arguments_netboot(subparser):
     return ret
 
 
+def arguments_ci(subparser):
+    ret = subparser.add_parser("ci", help="run continuous integration scripts"
+                                          " locally of git repo in current"
+                                          " directory")
+    script_args = ret.add_mutually_exclusive_group()
+    script_args.add_argument("-a", "--all", action="store_true",
+                             help="run all scripts")
+    script_args.add_argument("-f", "--fast", action="store_true",
+                             help="run fast scripts only")
+    ret.add_argument("scripts", nargs="*", metavar="script",
+                     help="name of the CI script to run, depending on the git"
+                          " repository")
+    return ret
+
+
 def package_completer(prefix, action, parser=None, parsed_args=None):
     args = parsed_args
     pmb.config.merge_with_args(args)
@@ -580,8 +590,8 @@ def add_packages_arg(subparser, name="packages", *args, **kwargs):
         arg.completer = package_completer
 
 
-def add_kernel_arg(subparser, name="package", *args, **kwargs):
-    arg = subparser.add_argument("package", nargs='?', help="kernel package"
+def add_kernel_arg(subparser, name="package", nargs="?", *args, **kwargs):
+    arg = subparser.add_argument("package", nargs=nargs, help="kernel package"
                                  " (e.g. linux-postmarketos-allwinner)")
     if argcomplete:
         arg.completer = kernel_completer
@@ -595,7 +605,7 @@ def arguments():
 
     # Other
     parser.add_argument("-V", "--version", action="version",
-                        version=pmb.config.version)
+                        version=pmb.__version__)
     parser.add_argument("-c", "--config", dest="config",
                         default=pmb.config.defaults["config"],
                         help="path to pmbootstrap.cfg file (default in"
@@ -603,7 +613,6 @@ def arguments():
     parser.add_argument("--config-channels",
                         help="path to channels.cfg (which is by default"
                              " read from pmaports.git, origin/master branch)")
-    parser.add_argument("-d", "--port-distccd", dest="port_distccd")
     parser.add_argument("-mp", "--mirror-pmOS", dest="mirrors_postmarketos",
                         help="postmarketOS mirror, disable with: -mp='',"
                              " specify multiple with: -mp='one' -mp='two',"
@@ -644,16 +653,6 @@ def arguments():
     # Compiler
     parser.add_argument("--no-ccache", action="store_false",
                         dest="ccache", help="do not cache the compiled output")
-    parser.add_argument("--no-crossdirect", action="store_true",
-                        help="Don't use the new, faster 'crossdirect' method,"
-                             " use the old 'distcc-sshd' method instead. Use"
-                             " if crossdirect broke something. This option"
-                             " and the legacy 'distcc-sshd' code will be"
-                             " removed soon if no problems turn up.")
-    parser.add_argument("--distcc-nofallback", action="store_false",
-                        help="when using the cross compiler via distcc fails,"
-                             "do not fall back to compiling slowly with QEMU",
-                        dest="distcc_fallback")
     parser.add_argument("--no-cross", action="store_false", dest="cross",
                         help="disable cross compiler, build only with QEMU and"
                              " gcc (slow!)")
@@ -693,17 +692,14 @@ def arguments():
     arguments_newapkbuild(sub)
     arguments_lint(sub)
     arguments_status(sub)
+    arguments_ci(sub)
 
     # Action: log
     log = sub.add_parser("log", help="follow the pmbootstrap logfile")
-    log_distccd = sub.add_parser(
-        "log_distccd",
-        help="follow the distccd logfile")
-    for action in [log, log_distccd]:
-        action.add_argument("-n", "--lines", default="60",
-                            help="count of initial output lines")
-        action.add_argument("-c", "--clear", help="clear the log",
-                            action="store_true", dest="clear_log")
+    log.add_argument("-n", "--lines", default="60",
+                     help="count of initial output lines")
+    log.add_argument("-c", "--clear", help="clear the log",
+                     action="store_true", dest="clear_log")
 
     # Action: zap
     zap = sub.add_parser("zap", help="safely delete chroot folders")
@@ -839,9 +835,21 @@ def arguments():
     build.add_argument("-n", "--no-depends", action="store_true",
                        help="never build dependencies, abort instead",
                        dest="no_depends")
+    build.add_argument("--go-mod-cache", action="store_true", default=None,
+                       help="for go packages: Usually they should bundle the"
+                            " dependency sources instead of downloading them"
+                            " at build time. But if they don't (e.g. with"
+                            " pmbootstrap build --src), then this option can"
+                            " be used to let GOMODCACHE point into"
+                            " pmbootstrap's work dir to only download"
+                            " dependencies once. (default: true with --src,"
+                            " false otherwise)")
+    build.add_argument("--no-go-mod-cache",
+                       action="store_false", dest="go_mod_cache", default=None,
+                       help="don't set GOMODCACHE")
     build.add_argument("--envkernel", action="store_true",
                        help="Create an apk package from the build output of"
-                       " a kernel compiled with envkernel.sh.")
+                       " a kernel compiled locally on the host or with envkernel.sh.")
     add_packages_arg(build, nargs="+")
 
     # Action: deviceinfo_parse
@@ -893,4 +901,9 @@ def arguments():
     setattr(args, "from_argparse", copy.deepcopy(args))
     setattr(args.from_argparse, "from_argparse", args.from_argparse)
     pmb.helpers.args.init(args)
+
+    if getattr(args, "go_mod_cache", None) is None:
+        gomodcache = True if getattr(args, "src", None) else False
+        setattr(args, "go_mod_cache", gomodcache)
+
     return args