pmb: chroot: init: explicitly install the keys package

The alpine keys are explicitly installed by the alpine-base dependency.
This adds the postmarketos-base to /etc/apk/world for new chroots so
they don't get purged when the chroot gets cleaned.

.ci/codespell.sh

@@ -1,20 +0,0 @@
-#!/bin/sh -ex
-# SPDX-License-Identifier: GPL-3.0-or-later
-# Copyright 2023 Oliver Smith
-# Description: find typos
-# https://postmarketos.org/pmb-ci
-
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add \
-		py3-codespell
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-set -x
-
-# -L: words to ignore
-codespell \
-	-L crate \
-	-L hda \
-	.

.ci/docs.sh

@@ -1,26 +0,0 @@
-#!/bin/sh -e
-# Description: create documentation with sphinx
-# Options: native
-# https://postmarketos.org/pmb-ci
-
-
-# Ensure sphinx_rtd_theme is installed
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add \
-		py3-sphinx_rtd_theme \
-		py3-sphinxcontrib-autoprogram
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-# Require sphinx to be installed on the host system
-if [ -z "$(command -v sphinx-build)" ]; then
-	echo "ERROR: sphinx-build command not found, make sure it is in your PATH."
-	exit 1
-fi
-
-sphinx-build \
-	docs \
-	public \
-
-#	-E -a -v -T

.ci/mypy.sh

@@ -1,13 +0,0 @@
-#!/bin/sh -e
-# Description: static type checking for python scripts
-# https://postmarketos.org/pmb-ci
-
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add py3-argcomplete py3-mypy
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-set -x
-
-mypy pmbootstrap.py

.ci/note.sh

@@ -1,7 +0,0 @@
-#!/bin/sh -e
-
-printf "==============================================\n"
-printf "PROTIP: use"
-printf " \e[1;33mpmbootstrap ci\e[0m"
-printf " to run these scripts locally.\n"
-printf "==============================================\n"

.ci/prepare.sh

@@ -0,0 +1,36 @@
+#!/bin/sh -e
+# Install pmbootstrap depends, set up pmos user with sudo
+if [ "$(id -u)" != 0 ]; then
+	echo "ERROR: this script is meant to be executed in the gitlab-ci"
+	echo "environment only."
+	exit 1
+fi
+
+topdir="$(realpath "$(dirname "$0")/..")"
+cd "$topdir"
+
+ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
+
+apk add -q \
+	git \
+	openssl \
+	py3-pytest \
+	py3-pytest-cov \
+	sudo
+
+adduser -D pmos
+chown -R pmos:pmos .
+echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+
+su pmos -c "git config --global user.email postmarketos-ci@localhost"
+su pmos -c "git config --global user.name postmarketOS_CI"
+
+echo "Initializing pmbootstrap"
+if ! su pmos -c "yes '' | pmbootstrap \
+		--details-to-stdout \
+		init \
+		>/tmp/pmb_init 2>&1"; then
+	cat /tmp/pmb_init
+	exit 1
+fi
+echo ""

.ci/pytest.sh

@@ -1,39 +1,6 @@
 #!/bin/sh -e
-# Description: run pmbootstrap python testsuite
+topdir="$(realpath "$(dirname "$0")/..")"
-# Options: native slow
+cd "$topdir"
-# https://postmarketos.org/pmb-ci
-
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add \
-		git \
-		openssl \
-		py3-pytest \
-		py3-pytest-cov \
-		sudo
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-# Require pytest to be installed on the host system
-if [ -z "$(command -v pytest)" ]; then
-	echo "ERROR: pytest command not found, make sure it is in your PATH."
-	exit 1
-fi
-
-# Use pytest-cov if it is installed to display code coverage
-cov_arg=""
-if python -c "import pytest_cov" >/dev/null 2>&1; then
-	cov_arg="--cov=pmb"
-fi
-
-echo "Initializing pmbootstrap..."
-if ! yes '' | ./pmbootstrap.py \
-		--details-to-stdout \
-		init \
-		>/tmp/pmb_init 2>&1; then
-	cat /tmp/pmb_init
-	exit 1
-fi
 
 # Make sure that the work folder format is up to date, and that there are no
 # mounts from aborted test cases (#1595)
@@ -54,19 +21,4 @@ if ! [ -e "$deviceinfo" ]; then
 	exit 1
 fi
 
-# Make sure pmaports is clean, some of the tests will fail otherwise
+pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"
-if [ -n "$(git -C "$pmaports" status --porcelain)" ]; then
-	echo "ERROR: pmaports dir is not clean"
-	exit 1
-fi
-
-echo "Running pytest..."
-echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
-pytest \
-	--color=yes \
-	-vv \
-	-x \
-	$cov_arg \
-	test \
-		-m "not skip_ci" \
-		"$@"

.ci/ruff.sh

@@ -1,19 +0,0 @@
-#!/bin/sh -e
-# Description: lint all python scripts
-# https://postmarketos.org/pmb-ci
-
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add ruff
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-set -x
-
-# __init__.py with additional ignore:
-# F401: imported, but not used
-# shellcheck disable=SC2046
-ruff check --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py')
-
-# Check all other files
-ruff check --exclude=__init__.py .

.ci/shellcheck.sh

@@ -1,15 +0,0 @@
-#!/bin/sh -e
-# Description: lint all shell scripts
-# https://postmarketos.org/pmb-ci
-
-if [ "$(id -u)" = 0 ]; then
-	set -x
-	apk -q add shellcheck
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
-
-find . -name '*.sh' |
-while read -r file; do
-	echo "shellcheck: $file"
-	shellcheck "$file"
-done

.ci/vermin.sh

@@ -1,23 +1,17 @@
 #!/bin/sh -e
-# Description: verify that we don't use too new python features
+_vermin() {
-# https://postmarketos.org/pmb-ci
+	if ! vermin -q "$@" >/dev/null 2>&1; then
-
+		vermin -vv "$@"
-if [ "$(id -u)" = 0 ]; then
+	fi
-	set -x
+}
-	apk -q add vermin
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
-fi
 
 # shellcheck disable=SC2046
-vermin \
+_vermin \
-	-t=3.7- \
+	-t=3.6- \
 	--backport argparse \
 	--backport configparser \
 	--backport enum \
 	--backport typing \
-	--lint \
-	--no-parse-comments \
-	--eval-annotations \
 	$(find . -name '*.py' \
 		-a -not -path "./.venv/*" \
 		-a -not -path "./venv/*")

.gitignore

@@ -19,7 +19,6 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-public/
 env/
 develop-eggs/
 dist/
@@ -119,8 +118,3 @@ ENV/
 
 # JetBrains IDEs (PyCharm, etc)
 .idea
-
-# KDevelop
-.kdev4/
-*.kdev4
-*.kate*

.gitlab-ci.yml

@@ -1,74 +1,66 @@
-image: alpine:edge
+---
+# Author: Clayton Craft <clayton@craftyguy.net>
 
-# The mr-settings check needs to run in a MR specific context. With this block,
+image: alpine:latest
-# the whole pipeline runs in that context for MRs. Otherwise we would have two
-# pipelines for MRs.
-workflow:
-  rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
-    - if: $CI_COMMIT_BRANCH == 'master'
-    - if: $CI_COMMIT_BRANCH == 'wip'
-    - if: $CI_COMMIT_BRANCH == 'docs'
 
-before_script: &global_before_scripts
+# defaults for "only"
-  - ".ci/note.sh"
+# We need to run the CI jobs in a "merge request specific context", if CI is
-  - apk upgrade -U
+# running in a merge request. Otherwise the environment variable that holds the
-  - "adduser -D build"
+# merge request ID is not available. This means, we must set the "only"
+# variable accordingly - and if we only do it for one job, all other jobs will
+# not get executed. So have the defaults here, and use them in all jobs that
+# should run on both the master branch, and in merge requests.
+# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
+.only-default: &only-default
+  only:
+    - master
+    - merge_requests
+    - tags
 
-stages:
+static-code-analysis:
-  - lint
+  <<: *only-default
-  - test
+  before_script:
-
+    - .ci/prepare.sh
-codespell:
-  stage: lint
   script:
-    - ".ci/codespell.sh"
+    - su pmos -c "test/static_code_analysis.sh"
-
-pytest:
-  stage: test
-  script:
-    - "apk -q add git"
-    - "su build -c 'git config --global user.email ci@ci'"
-    - "su build -c 'git config --global user.name CI'"
-    - "echo 'build ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers"
-    - ".ci/pytest.sh"
-
-ruff:
-  stage: lint
-  script:
-    - ".ci/ruff.sh"
-
-shellcheck:
-  stage: lint
-  script:
-    - ".ci/shellcheck.sh"
 
 vermin:
-  stage: lint
+  image: alpine:latest
-  script:
+  <<: *only-default
-    - ".ci/vermin.sh"
-
-mr-settings:
-  stage: lint
-  rules:
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
   before_script:
-    - *global_before_scripts
+  - "apk -q add py3-pip"
-    - "apk -q add python3"
+  - "pip3 -q --disable-pip-version-check install vermin"
-    - "wget -q 'https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py'"
   script:
-    - "python3 ./check_mr_settings.py"
+  - ".ci/vermin.sh"
 
-mypy:
+# MR settings
-  stage: lint
+# (Checks for "Allow commits from members who can merge to the target branch")
+mr-settings:
+  only:
+    - merge_requests
+  before_script:
+    - apk -q add python3
   script:
-    - ".ci/mypy.sh"
+    - wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
+    - python3 ./check_mr_settings.py
 
-docs:
+pytest:
-  stage: lint
+  <<: *only-default
+  before_script:
+    - .ci/prepare.sh
   script:
-  - apk add py3-sphinx py3-sphinx_rtd_theme
+    - su pmos -c .ci/pytest.sh
-  - ".ci/docs.sh"
+  after_script:
+     # Move logs so it can be saved as artifacts
+    - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
+    - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
+    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
+    - "dmesg > $CI_PROJECT_DIR/dmesg.txt"
   artifacts:
+    when: always
     paths:
-    - public
+      - "log.txt"
+      - "log_testsuite.txt"
+      - "dmesg.txt"
+      - "pmbootstrap.cfg"
+    expire_in: 1 week

CONTRIBUTING.md

@@ -1,26 +1,26 @@
-## Contributing
+## Reporting issues
-pmbootstrap development is being discussed in
+* Consider joining the [chat](https://wiki.postmarketos.org/wiki/Matrix_and_IRC) for instant help.
-[#postmarketOS-devel](https://wiki.postmarketos.org/wiki/Matrix_and_IRC).
+* Maybe your question is answered in the [wiki](https://wiki.postmarketos.org/) somewhere. [Search](https://wiki.postmarketos.org/index.php?search=&title=Special%3ASearch&go=Go) first!
+* Otherwise, just ask what you want to know. We're happy if we can help you and glad that you're using `pmbootstrap`!
 
-### CI scripts
+## Development
-Use `pmbootstrap ci` inside your `pmbootstrap.git` dir, to run all CI scripts
-locally.
 
-### Coding style
+See pmbootstrap's [Development Guide](https://wiki.postmarketos.org/wiki/Development_guide).
-A lot of the coding style is enforced by the CI scripts.
 
-#### Python
+### Contributing code changes
-* Use [PEP8](https://www.python.org/dev/peps/pep-0008/).
+* [Fork](https://docs.gitlab.com/ee/gitlab-basics/fork-project.html) this repository, commit your changes and then make a [Merge Request](https://docs.gitlab.com/ee/workflow/merge_requests.html).
-* Max line length: 80-100 characters (use 80 for comments and most code lines
+* Please test your code before submitting a Merge Request.
-  except when 100 makes much more sense; try to keep it consistent with
-  existing code).
-* Use [f-strings](https://peps.python.org/pep-0498/) for any new or modified
-  code, instead of any of the other string formatting methods.
-* pmbootstrap should run on any Linux distribution, so we support all active
-  Python versions (see [here](https://www.python.org/downloads/)).
-* Docstrings below functions are formatted in `reST` style:
 
-```python
+### Shell scripting
+* We don't write scripts for `bash`, but for `busybox`'s `ash` shell, which is POSIX compliant (plus very few features from `bash`).
+* Use `shellcheck` to test your changes for issues before submitting. There is even an [online](https://www.shellcheck.net) version.
+* We're looking into automatizing this more, some files already get checked automatically by the [static code analysis script](test/static_code_analysis.sh).
+
+### Python
+* We use the [PEP8](https://www.python.org/dev/peps/pep-0008/) standard for Python code. Don't worry, you don't need to read all that, just run the `autopep8` program on your changed code, and confirm with the [static code analysis script](test/static_code_analysis.sh) that everything is PEP8 compliant. *This script will run automatically on Travis CI when you make a Merge Request, and it must pass for your code to get accepted.*
+* We use the `reST` style for `docstrings` below functions (to comment what individual functions are doing, you'll see those when browsing through the code). Please stick to this format, and try to describe the important parameters and return values at least. Example from [here](https://stackoverflow.com/a/24385103):
+
+```Python
 """
 This is a reST style.
 
@@ -31,123 +31,7 @@ This is a reST style.
 """
 ```
 
-#### Shell scripts
+* If it is feasible for you, try to run the testsuite on code that you have changed. The `test/test_build.py` case will build full cross-compilers for `aarch64` and `armhf`, so it may take a long time. Testcases can be started with `pytest` and it's planned to run that automatically when making a new Merge Request (see #64).
-* Must be POSIX compliant, so busybox ash can interpret them. (Exception: the
-  `local` keyword can also be used, to give variables a local scope inside
-  functions).
 
-### Code patterns
 
-#### The `args` variable
+**If you need any help, don't hesitate to open an [issue](https://gitlab.com/postmarketOS/pmbootstrap/issues) and ask!**
-This contains the arguments passed to pmbootstrap, and some additional data.
-See `pmb/helpers/args.py` for details. This is a legacy construct, see
-[#1879](https://gitlab.com/postmarketOS/pmbootstrap/-/issues/1879).
-
-#### Executing commands
-Use one of the following functions instead of Python's built-in `subprocess`:
-
-* `pmb.helpers.run.user()`
-* `pmb.helpers.run.root()`
-* `pmb.chroot.user()`
-* `pmb.chroot.root()`
-
-These functions call `pmb.helpers.run_core.core()` internally to write to the
-log file (that you can read with `pmbootstrap log`) and timeout when there is
-no output. A lot of function parameters are passed through to `core()` as well,
-see its docstring for a detailed description of what these parameters do.
-
-##### Using shell syntax
-The passed commands do not run inside a shell. If you need to use shell syntax,
-wrap your command with `sh -c` and use `shutil.quote` on the parameters (if
-they contain untrusted input):
-
-```py
-# Does not work, the command does not run in a shell!
-pmb.chroot.root(args, ["echo", "test", ">", "/tmp/test"])
-
-# Use this instead (assuming untrusted input for text, dest)
-text = "test"
-dest = "/tmp/test"
-shell_cmd = f"echo {shutil.quote(text)} > {shutil.quote(dest)}"
-pmb.chroot.root(args, ["sh", "-c", shell_cmd])
-```
-
-If you need to run many commands in a shell at once, write them into a
-temporary shell script and execute that with one of the `pmb` command
-functions.
-
-#### Writing files to the chroot
-The users in the chroots (`root` and `pmos`) have different user IDs than the
-user of the host system. Therefore we can't just write a file to anywhere in
-the chroot. Use one of the following methods.
-
-##### Short files
-```py
-pmb.chroot.user(args, ["sh", "-c", f"echo {shlex.quote(hostname)}"
-                       " > /etc/hostname"], suffix)
-```
-
-##### Long files
-Write to a temp dir first with python code, then move and chown the file.
-
-```py
-with open("tmp/somefile", "w") as handle:
-    handle.write("Some long file")
-    handle.write("with multiple")
-    handle.write("lines here")
-pmb.chroot.root(args, ["mv", "/tmp/somefile", "/etc/somefile"])
-pmb.chroot.root(args, ["chown", "root:root", "/etc/somefile"], suffix)
-```
-
-### Manual testing
-
-#### APKBUILD parser
-Besides the python tests, it's a good idea to let the APKBUILD parsing code run
-over all APKBUILDs that we have in pmaports.git, before and after making
-changes. This makes it easy to spot regressions.
-
-```
-$ pmbootstrap apkbuild_parse > /tmp/new
-$ git checkout master
-$ pmbootstrap apkbuild_parse > /tmp/old
-$ colordiff /tmp/old /tmp/new | less -R
-```
-
-### Debugging
-
-#### Tab completion
-When tab completion breaks, commands-line `pmbootstrap build <TAB>` will simply
-not return the expected list of packages anymore. Exceptions are not printed.
-To change this behavior and get the exceptions, adjust the
-`eval "$(register-python-argcomplete pmbootstrap)"` line in your shell's rc
-file.
-
-```
-$ register-python-argcomplete3 pmbootstrap
-
-_python_argcomplete() {
-    local IFS=$'\013'
-    local SUPPRESS_SPACE=0
-    if compopt +o nospace 2> /dev/null; then
-        SUPPRESS_SPACE=1
-    fi
-    COMPREPLY=( $(IFS="$IFS" \
-                  COMP_LINE="$COMP_LINE" \
-                  COMP_POINT="$COMP_POINT" \
-                  COMP_TYPE="$COMP_TYPE" \
-                  _ARGCOMPLETE_COMP_WORDBREAKS="$COMP_WORDBREAKS" \
-                  _ARGCOMPLETE=1 \
-                  _ARGCOMPLETE_SUPPRESS_SPACE=$SUPPRESS_SPACE \
-                  "$1" 8>&1 9>&2 1>/dev/null 2>/dev/null) )
-    if [[ $? != 0 ]]; then
-        unset COMPREPLY
-    elif [[ $SUPPRESS_SPACE == 1 ]] && [[ "$COMPREPLY" =~ [=/:]$ ]]; then
-        compopt -o nospace
-    fi
-}
-complete -o nospace -o default -F _python_argcomplete "pmbootstrap"
-```
-
-Copy the whole output of the command to your shell's rc file instead of the
-eval line, but remove `1>/dev/null 2>/dev/null`. Then it will print exceptions
-to the shell.

README.md

@@ -1,49 +1,22 @@
 # pmbootstrap
+[**Introduction**](https://postmarketos.org/blog/2017/05/26/intro/) | [**Security Warning**](https://ollieparanoid.github.io/post/security-warning/) | [**Devices**](https://wiki.postmarketos.org/wiki/Devices)
 
-Sophisticated chroot/build/flash tool to develop and install
+Sophisticated chroot/build/flash tool to develop and install [postmarketOS](https://postmarketos.org).
-[postmarketOS](https://postmarketos.org).
 
-## Development
+Package build scripts live in the [`pmaports`](https://gitlab.com/postmarketOS/pmaports) repository now.
-
-Find the location of the upstream repository for pmbootstrap on the
-[postmarketOS homepage](https://postmarketos.org/source-code/).
-
-Run CI scripts locally with:
-```
-$ pmbootstrap ci
-```
-
-Run a single test file:
-```
-$ pytest -vv ./test/test_keys.py
-```
-
-## Issues
-
-Issues are being tracked
-[here](https://gitlab.com/postmarketOS/pmbootstrap/-/issues).
 
 ## Requirements
-* Linux distribution on the host system (`x86`, `x86_64`, `aarch64` or `armv7`)
+* 2 GB of RAM recommended for compiling
-  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
+* Linux distribution on the host system (`x86`, `x86_64`, or `aarch64`)
-    does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
+  * [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
+  * Kernels based on the grsec patchset [do **not** work](https://github.com/postmarketOS/pmbootstrap/issues/107)
   * [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
-  * Note: kernel versions between 5.8.8 and 6.0 might 
+* Python 3.6+
-    [have issues with parted](https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2309).
-* Python 3.7+
 * OpenSSL
 * git
-* ps
-* tar
 
 ## Usage Examples
-Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for
+Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for in-depth coverage of topics such as [porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The help output (`pmbootstrap -h`) has detailed usage instructions for every command. Read on for some generic examples of what can be done with `pmbootstrap`.
-in-depth coverage of topics such as
-[porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device)
-or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The
-help output (`pmbootstrap -h`) has detailed usage instructions for every
-command. Read on for some generic examples of what can be done with
-`pmbootstrap`.
 
 ### Installing pmbootstrap
 <https://wiki.postmarketos.org/wiki/Installing_pmbootstrap>
@@ -87,28 +60,7 @@ $ pmbootstrap checksum hello-world
 
 Generate a template for a new package:
 ```
-$ pmbootstrap newapkbuild "https://gitlab.com/postmarketOS/tinydm/-/archive/1.2.0/tinydm-1.2.0.tar.gz"
+$ pmbootstrap newapkbuild "https://gitlab.com/postmarketOS/osk-sdl/-/archive/0.52/osk-sdl-0.52.tar.bz2"
-```
-
-#### Default architecture
-
-Packages will be compiled for the architecture of the device running
-pmbootstrap by default. For example, if your `x86_64` PC runs pmbootstrap, it
-would build a package for `x86_64` with this command:
-```
-$ pmbootstrap build hello-world
-```
-
-If you would rather build for the target device selected in `pmbootstrap init`
-by default, then use the `build_default_device_arch` option:
-```
-$ pmbootstrap config build_default_device_arch True
-```
-
-If your target device is `pine64-pinephone` for example, pmbootstrap will now
-build this package for `aarch64`:
-```
-$ pmbootstrap build hello-world
 ```
 
 ### Chroots
@@ -128,9 +80,7 @@ $ pmbootstrap zap
 ```
 
 ### Device Porting Assistance
-Analyze Android
+Analyze Android [`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also works with recovery OS images like TWRP):
-[`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also
-works with recovery OS images like TWRP):
 ```
 $ pmbootstrap bootimg_analyze ~/Downloads/twrp-3.2.1-0-fp2.img
 ```
@@ -158,7 +108,7 @@ $ pmbootstrap install --fde
 
 Update existing installation on SD card:
 ```
-$ pmbootstrap install --disk=/dev/mmcblk0 --rsync
+$ pmbootstrap install --sdcard=/dev/mmcblk0 --rsync
 ```
 
 Run the image in QEMU:
@@ -194,16 +144,14 @@ List pmaports that don't have a binary package:
 $ pmbootstrap repo_missing --arch=armhf --overview
 ```
 
-Increase the `pkgrel` for each aport where the binary package has outdated
+Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps):
-dependencies (e.g. after soname bumps):
 ```
 $ pmbootstrap pkgrel_bump --auto
 ```
 
-Generate cross-compiler aports based on the latest version from Alpine's
+Generate cross-compiler aports based on the latest version from Alpine's aports:
-aports:
 ```
-$ pmbootstrap aportgen gcc-armhf
+$ pmbootstrap aportgen binutils-armhf gcc-armhf
 ```
 
 Manually rebuild package index:
@@ -242,32 +190,44 @@ $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hel
 $ pmbootstrap stats --arch=armhf
 ```
 
-### Use alternative sudo
+`distccd` log:
+```
+$ pmbootstrap log_distccd
+```
 
-pmbootstrap supports `doas` and `sudo`.
+## Development
-If multiple sudo implementations are installed, pmbootstrap will use `doas`.
+### Requirements for running tests
-You can set the `PMB_SUDO` environmental variable to define the sudo
+* [Shellcheck](https://shellcheck.net/)
-implementation you want to use.
 
-### Select SSH keys to include and make authorized in new images
+You also need to install the following python packages (pip can be useful if you distribution hasn't got them packaged):
+* `pytest`
+* `pytest-cov`
+* `flake8`
 
-If the config file option `ssh_keys` is set to `True` (it defaults to `False`),
+On Alpine Linux it can be done with:
-then all files matching the glob `~/.ssh/id_*.pub` will be placed in
+```shell
-`~/.ssh/authorized_keys` in the user's home directory in newly-built images.
+$ sudo apk add grep shellcheck py3-pytest py3-pytest-cov py3-flake8
+```
 
-Sometimes, for example if you have a large number of SSH keys, you may wish to
+### Running linters
-select a different set of public keys to include in an image. To do this, set
+The easiest way is to run the same script CI runs:
-the `ssh_key_glob` configuration parameter in the pmbootstrap config file to a
+```shell
-string containing a glob that is to match the file or files you wish to
+$ ./test/static_code_analysis.sh
-include.
+```
 
-For example, a `~/.config/pmbootstrap.cfg` may contain:
+### Running tests
+You can now run `pytest -vv` inside the pmbootstrap folder to run all available tests.
 
-    [pmbootstrap]
+CI runs slightly reduces set of tests (it skips tests that require running qemu) by this:
-    # ...
+```shell
-    ssh_keys = True
+$ .ci/pytest.sh
-    ssh_key_glob = ~/.ssh/postmarketos-dev.pub
+```
-    # ...
+This is the easiest way to do the same as CI.
+
+Alternatively you can run a single test file if you wish:
+```shell
+$ pytest -vv ./test/test_keys.py
+```
 
 ## License
 [GPLv3](LICENSE)

docs/api/modules.rst

@@ -1,9 +0,0 @@
-pmboostrap code documentation
-=============================
-
-.. note:: This is the internal API, not a public one.
-
-.. toctree::
-   :maxdepth: 4
-
-   pmb

docs/api/pmb.aportgen.rst

@@ -1,69 +0,0 @@
-pmb.aportgen package
-====================
-
-Submodules
-----------
-
-pmb.aportgen.busybox\_static module
------------------------------------
-
-.. automodule:: pmb.aportgen.busybox_static
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.core module
-------------------------
-
-.. automodule:: pmb.aportgen.core
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.device module
---------------------------
-
-.. automodule:: pmb.aportgen.device
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.gcc module
------------------------
-
-.. automodule:: pmb.aportgen.gcc
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.grub\_efi module
------------------------------
-
-.. automodule:: pmb.aportgen.grub_efi
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.linux module
--------------------------
-
-.. automodule:: pmb.aportgen.linux
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.aportgen.musl module
-------------------------
-
-.. automodule:: pmb.aportgen.musl
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.aportgen
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.build.rst

@@ -1,69 +0,0 @@
-pmb.build package
-=================
-
-Submodules
-----------
-
-pmb.build.autodetect module
----------------------------
-
-.. automodule:: pmb.build.autodetect
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.checksum module
--------------------------
-
-.. automodule:: pmb.build.checksum
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.envkernel module
---------------------------
-
-.. automodule:: pmb.build.envkernel
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.init module
----------------------
-
-.. automodule:: pmb.build.init
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.kconfig module
-------------------------
-
-.. automodule:: pmb.build.kconfig
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.newapkbuild module
-----------------------------
-
-.. automodule:: pmb.build.newapkbuild
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.build.other module
-----------------------
-
-.. automodule:: pmb.build.other
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.build
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.chroot.rst

@@ -1,109 +0,0 @@
-pmb.chroot package
-==================
-
-Submodules
-----------
-
-pmb.chroot.apk module
----------------------
-
-.. automodule:: pmb.chroot.apk
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.apk\_static module
------------------------------
-
-.. automodule:: pmb.chroot.apk_static
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.binfmt module
-------------------------
-
-.. automodule:: pmb.chroot.binfmt
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.init module
-----------------------
-
-.. automodule:: pmb.chroot.init
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.initfs module
-------------------------
-
-.. automodule:: pmb.chroot.initfs
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.initfs\_hooks module
--------------------------------
-
-.. automodule:: pmb.chroot.initfs_hooks
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.mount module
------------------------
-
-.. automodule:: pmb.chroot.mount
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.other module
------------------------
-
-.. automodule:: pmb.chroot.other
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.root module
-----------------------
-
-.. automodule:: pmb.chroot.root
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.shutdown module
---------------------------
-
-.. automodule:: pmb.chroot.shutdown
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.user module
-----------------------
-
-.. automodule:: pmb.chroot.user
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.chroot.zap module
----------------------
-
-.. automodule:: pmb.chroot.zap
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.chroot
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.ci.rst

@@ -1,10 +0,0 @@
-pmb.ci package
-==============
-
-Module contents
----------------
-
-.. automodule:: pmb.ci
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.config.rst

@@ -1,70 +0,0 @@
-pmb.config package
-==================
-
-Submodules
-----------
-
-pmb.config.init module
-----------------------
-
-.. automodule:: pmb.config.init
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.config.load module
-----------------------
-
-.. automodule:: pmb.config.load
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.config.merge\_with\_args module
------------------------------------
-
-.. automodule:: pmb.config.merge_with_args
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.config.pmaports module
---------------------------
-
-.. automodule:: pmb.config.pmaports
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.config.save module
-----------------------
-
-.. automodule:: pmb.config.save
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.config.sudo module
-----------------------
-
-.. automodule:: pmb.config.sudo
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:
-
-pmb.config.workdir module
--------------------------
-
-.. automodule:: pmb.config.workdir
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.config
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.export.rst

@@ -1,37 +0,0 @@
-pmb.export package
-==================
-
-Submodules
-----------
-
-pmb.export.frontend module
---------------------------
-
-.. automodule:: pmb.export.frontend
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.export.odin module
-----------------------
-
-.. automodule:: pmb.export.odin
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.export.symlinks module
---------------------------
-
-.. automodule:: pmb.export.symlinks
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.export
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.flasher.rst

@@ -1,45 +0,0 @@
-pmb.flasher package
-===================
-
-Submodules
-----------
-
-pmb.flasher.frontend module
----------------------------
-
-.. automodule:: pmb.flasher.frontend
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.flasher.init module
------------------------
-
-.. automodule:: pmb.flasher.init
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.flasher.run module
-----------------------
-
-.. automodule:: pmb.flasher.run
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.flasher.variables module
-----------------------------
-
-.. automodule:: pmb.flasher.variables
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.flasher
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.helpers.rst

@@ -1,189 +0,0 @@
-pmb.helpers package
-===================
-
-Submodules
-----------
-
-pmb.helpers.apk module
-----------------------
-
-.. automodule:: pmb.helpers.apk
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.aportupgrade module
--------------------------------
-
-.. automodule:: pmb.helpers.aportupgrade
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.args module
------------------------
-
-.. automodule:: pmb.helpers.args
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.cli module
-----------------------
-
-.. automodule:: pmb.helpers.cli
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.devices module
---------------------------
-
-.. automodule:: pmb.helpers.devices
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.file module
------------------------
-
-.. automodule:: pmb.helpers.file
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.frontend module
----------------------------
-
-.. automodule:: pmb.helpers.frontend
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.git module
-----------------------
-
-.. automodule:: pmb.helpers.git
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.http module
------------------------
-
-.. automodule:: pmb.helpers.http
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.lint module
------------------------
-
-.. automodule:: pmb.helpers.lint
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.logging module
---------------------------
-
-.. automodule:: pmb.helpers.logging
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.mount module
-------------------------
-
-.. automodule:: pmb.helpers.mount
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.other module
-------------------------
-
-.. automodule:: pmb.helpers.other
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.package module
---------------------------
-
-.. automodule:: pmb.helpers.package
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.pkgrel\_bump module
--------------------------------
-
-.. automodule:: pmb.helpers.pkgrel_bump
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.pmaports module
----------------------------
-
-.. automodule:: pmb.helpers.pmaports
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.repo module
------------------------
-
-.. automodule:: pmb.helpers.repo
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.repo\_missing module
---------------------------------
-
-.. automodule:: pmb.helpers.repo_missing
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.run module
-----------------------
-
-.. automodule:: pmb.helpers.run
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.run\_core module
-----------------------------
-
-.. automodule:: pmb.helpers.run_core
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.status module
--------------------------
-
-.. automodule:: pmb.helpers.status
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.helpers.ui module
----------------------
-
-.. automodule:: pmb.helpers.ui
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.helpers
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.install.rst

@@ -1,61 +0,0 @@
-pmb.install package
-===================
-
-Submodules
-----------
-
-pmb.install.blockdevice module
-------------------------------
-
-.. automodule:: pmb.install.blockdevice
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.install.format module
--------------------------
-
-.. automodule:: pmb.install.format
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.install.losetup module
---------------------------
-
-.. automodule:: pmb.install.losetup
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.install.partition module
-----------------------------
-
-.. automodule:: pmb.install.partition
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.install.recovery module
----------------------------
-
-.. automodule:: pmb.install.recovery
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.install.ui module
----------------------
-
-.. automodule:: pmb.install.ui
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.install
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.netboot.rst

@@ -1,10 +0,0 @@
-pmb.netboot package
-===================
-
-Module contents
----------------
-
-.. automodule:: pmb.netboot
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.parse.rst

@@ -1,93 +0,0 @@
-pmb.parse package
-=================
-
-Submodules
-----------
-
-pmb.parse.apkindex module
--------------------------
-
-.. automodule:: pmb.parse.apkindex
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.arch module
----------------------
-
-.. automodule:: pmb.parse.arch
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.arguments module
---------------------------
-
-.. automodule:: pmb.parse.arguments
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.binfmt\_info module
------------------------------
-
-.. automodule:: pmb.parse.binfmt_info
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.bootimg module
-------------------------
-
-.. automodule:: pmb.parse.bootimg
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.cpuinfo module
-------------------------
-
-.. automodule:: pmb.parse.cpuinfo
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.depends module
-------------------------
-
-.. automodule:: pmb.parse.depends
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.deviceinfo module
----------------------------
-
-.. automodule:: pmb.parse.deviceinfo
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.kconfig module
-------------------------
-
-.. automodule:: pmb.parse.kconfig
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-pmb.parse.version module
-------------------------
-
-.. automodule:: pmb.parse.version
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.parse
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.qemu.rst

@@ -1,21 +0,0 @@
-pmb.qemu package
-================
-
-Submodules
-----------
-
-pmb.qemu.run module
--------------------
-
-.. automodule:: pmb.qemu.run
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
-Module contents
----------------
-
-.. automodule:: pmb.qemu
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/api/pmb.rst

@@ -1,31 +0,0 @@
-pmb package
-===========
-
-Subpackages
------------
-
-.. toctree::
-   :maxdepth: 4
-
-   pmb.aportgen
-   pmb.build
-   pmb.chroot
-   pmb.ci
-   pmb.config
-   pmb.export
-   pmb.flasher
-   pmb.helpers
-   pmb.install
-   pmb.netboot
-   pmb.parse
-   pmb.qemu
-   pmb.sideload
-
-Module contents
----------------
-
-.. automodule:: pmb
-   :members:
-   :undoc-members:
-   :show-inheritance:
-   :noindex:

docs/api/pmb.sideload.rst

@@ -1,10 +0,0 @@
-pmb.sideload package
-====================
-
-Module Contents
----------------
-
-.. automodule:: pmb.sideload
-   :members:
-   :undoc-members:
-   :show-inheritance:

docs/conf.py

@@ -1,61 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-import sys
-import os
-import datetime
-
-
-sys.path.insert(0, os.path.abspath('..'))  # Allow modules to be found
-from pmb import __version__
-
-on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
-
-if not on_rtd:  # only import and set the theme if we're building docs locally
-    import sphinx_rtd_theme
-
-    html_theme = 'sphinx_rtd_theme'
-    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-
-project = 'pmbootstrap'
-copyright = str(datetime.date.today().year) + ', postmarketOS developers'
-author = 'postmarketOS developers'
-
-release = __version__
-version = '.'.join(release.split('.')[:3])
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.autosummary', 'sphinx.ext.doctest', 'sphinxcontrib.autoprogram']
-
-templates_path = ['_templates']
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# -- Options for HTML output -------------------------------------------------
-html_theme = 'sphinx_rtd_theme'
-html_favicon = 'https://wiki.postmarketos.org/favicon.ico'
-
-html_theme_options = {'style_nav_header_background': '008b69',}
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'pmboostrapdoc'
-
-html_theme_options = {
-    'display_version': True,
-    'style_external_links': True,
-    }
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    ('index', 'pmbootstrap', 'pmbootstrap Documentation',
-     ['postmarketOS Developers'], 1)
-]

docs/index.rst

@@ -1,38 +0,0 @@
-Welcome to pmbootstrap's documentation!
-=======================================
-
-pmboostrap is the central command-line application for postmarketOS development. Among other things,
-it allows building packages, creating installation images and flashing themx to your device. If you just want to install 
-postmarketOS, read the `Installation`_ wiki article first since you might not need pmbootstrap depeing on the method.
-
-For the latest releases please check the `repository`_. 
-  
-In case of any problems that is also the place to check the `issue-tracker`_.
-
-For further information, please check out the `postmarketOS-wiki`_.
-
-
-.. toctree::
-   :maxdepth: 3
-   :caption: Contents:
-
-   installation
-   usage
-   api/modules
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
-
-
-*Note:* This documentation is currently a work-in-progress, your feedback and contributions are very welcome!
-
-.. _postmarketOS-wiki: https://wiki.postmarketos.org/wiki/Main_Page
-.. _issue-tracker: https://gitlab.com/postmarketOS/pmbootstrap/-/issues
-.. _repository: https://gitlab.com/postmarketOS/pmbootstrap/
-.. _Installation: https://wiki.postmarketos.org/wiki/Installation

docs/installation.rst

@@ -1,67 +0,0 @@
-
-Installation
-============
-
-pmbootstrap runs on pretty much any Linux distribution with python3, openssl and git installed. It uses Alpine Linux chroots internally to avoid installing software on the host machine. If you don't run Linux on your PC, see :ref:`other-os`. 
-
-On Linux
---------
-
-From package manager
-^^^^^^^^^^^^^^^^^^^^
-
-.. code-block::
-
-   Alpine Linux, postmarketOS:
-   # apk add pmbootstrap
-   Arch Linux:
-   # pacman -S pmbootstrap
-   Debian:
-   # apt install pmbootstrap
-   Fedora:
-   # dnf install pmbootstrap
-   Void Linux:
-   # xbps-install -S pmbootstrap
-   Gentoo:
-   # emerge -va app-eselect/eselect-repository
-   # eselect repository enable guru
-   # emaint -r guru sync
-   # emerge -va dev-util/pmbootstrap
-   Nix/NixOS
-   # nix run nixpkgs#pmbootstrap
-
-.. note::
-   Fixed release distributions, i.e. Debian, may freeze pmbootstrap version. Consider installing it from git if you want the latest features and bug fixes.
-
-From git
-^^^^^^^^
-Follow this section if your Linux distribution doesn't have pmbootstrap packaged, or its version of pmbootstrap is too old, or you would like to change the code. Run the following to clone and install pmbootstrap from git. 
-
-
-.. code-block::
-   
-   $ git clone --depth=1 https://gitlab.com/postmarketOS/pmbootstrap.git
-   $ mkdir -p ~/.local/bin
-   $ ln -s "$PWD/pmbootstrap/pmbootstrap.py" ~/.local/bin/pmbootstrap
-   $ pmbootstrap --version
-   2.1.0
-
-If this returns something like `pmbootstrap: command not found instead` of a version number, ensure that `~/.local/bin` is in your `PATH`. For example by adding the following to your `~/.profile` (zsh: `~/.zprofile`) followed by `source ~/.profile` to update your environment. 
-
-.. code-block::
-
-   PATH="$HOME/.local/bin:$PATH"
-
-Then open a new terminal and try again. 
-
-.. _other-os:
-
-On other operating systems
---------------------------
-
-Running pmbootstrap on other operating systems than Linux is not supported. If you run another OS, consider setting up a virtual machine with Linux. 
-
-Some people also made it work with WSL, see the `Windows FAQ`_ in the pmOS-Wiki. 
-But again, it's not officially supported - we recommend getting some sort of Linux install instead and running it there.
-
-.. _Windows FAQ: https://wiki.postmarketos.org/wiki/Windows_FAQ

docs/usage.rst

@@ -1,59 +0,0 @@
-
-#####
-Usage
-#####
-
-pmbootstrap offers many options and actions and is normally ran from a shell.
-
-
-Before pmbootstrap can be used, a number of configuration questions need to be answered. The sections below go into detail for the various questions. 
-
-.. code-block:: shell
-
-  $ pmboostrap init
-
-If you already ran this before, run the following to update your local clone of pmaports.git instead, before moving straight onto the installation step: 
-
-.. code-block:: shell
-
-   $ pmbootstrap pull
-
-For further details on the different actions please see below and refer to the wiki-arcticle on `pmbootstrap`_.
-
-.. autoprogram:: pmb.parse:get_parser()
-   :prog: pmbootstrap
-   :groups:
-
-Requirements
-============
-
-pmbootstrap requires the following:
-
-  * Linux distribution on the host system (`x86`, `x86_64`, `aarch64` or `armv7`)
-    
-    .. note::
-       Windows subsystem for `Linux (WSL)`_ does **not** work! Please use `VirtualBox`_ instead.
-
-
-  * Linux kernel 3.17 or higher (`oldkernel`_)
-
-    .. note::
-       Kernel version 5.8 - 6.0 might have issues with loop-devices
-
-
-  * Python 3.7+
-  * OpenSSL
-  * git
-  * ps
-  * tar
-  * sudo or doas
-
-
-.. _pmbootstrap: https://wiki.postmarketos.org/wiki/Pmbootstrap#Using_pmbootstrap
-
-.. _Linux (WSL): https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux
-
-.. _virtualbox: https://www.virtualbox.org/
-
-.. _oldkernel: https://postmarketos.org/oldkernel
-

helpers/envkernel.fish

@@ -15,13 +15,8 @@ for arg in $argv
 end
 
 # Fish compatibility code from envkernel.sh
-set envkernel_fish (status filename)
+set script_dir (dirname (status filename))
-set script_dir (dirname "$envkernel_fish")
 sh "$script_dir/envkernel.sh" $argv --fish 1>| read -z fishcode
-set pmbootstrap_dir (realpath "$script_dir/..")
-if not test -e "$pmbootstrap_dir/pmbootstrap.py"
-	set -e pmbootstrap_dir
-end
 
 # Verbose output (enable with: 'set ENVKERNEL_FISH_VERBOSE 1')
 if [ "$ENVKERNEL_FISH_VERBOSE" = "1" ]
@@ -32,30 +27,3 @@ end
 
 # Execute generated code
 echo -e "$fishcode" | source -
-
-# Set prompt
-if test -z "$ENVKERNEL_DISABLE_PROMPT"
-    functions -c fish_prompt _old_fish_prompt
-
-    function fish_prompt
-        set -l old_status $status
-        printf "[envkernel] "
-        echo "exit $old_status" | .
-        _old_fish_prompt
-    end
-end
-
-# Deactivate
-function deactivate
-	if functions -q _old_fish_prompt
-		functions -e fish_prompt
-		functions -c _old_fish_prompt fish_prompt
-		functions -e _old_fish_prompt
-	end
-	functions -e make kernelroot pmbootstrap pmbroot
-	functions -e deactivate reactivate
-	set -e envkernel_fish script_dir pmbootstrap_dir
-end
-
-# Reactivate
-alias reactivate "deactivate; pushd '$PWD'; . '$envkernel_fish'; popd"

helpers/envkernel.sh

@@ -40,51 +40,37 @@ clean_kernel_src_dir() {
 }
 
 
-export_envkernel_sh() {
-	# Get script location
-	# See also: <https://stackoverflow.com/a/29835459>
-	# shellcheck disable=SC3054
-	if [ -n "${BASH_SOURCE[0]}" ]; then
-		envkernel_sh="$(realpath "$BASH_SOURCE")"
-	else
-		envkernel_sh="$1"
-	fi
-	export envkernel_sh
-}
-
-
 export_pmbootstrap_dir() {
 	if [ -n "$pmbootstrap_dir" ]; then
 		return 0;
 	fi
 
-	# Get pmbootstrap dir based on this script's location, if it's
+	# Get pmbootstrap dir based on this script's location
-	# in a pmbootstrap source tree
+	# See also: <https://stackoverflow.com/a/29835459>
-	pmbootstrap_dir="$(realpath "$(dirname "${envkernel_sh}")/..")"
+	# shellcheck disable=SC3054
-	if [ -e "$pmbootstrap_dir/pmbootstrap.py" ]; then
+	if [ -n "${BASH_SOURCE[0]}" ]; then
-		export pmbootstrap_dir
+		script_dir="$(dirname "${BASH_SOURCE[0]}")"
 	else
-		unset pmbootstrap_dir
+		script_dir="$(dirname "$1")"
+	fi
+
+	# Fail with debug information
+	# shellcheck disable=SC2155
+	export pmbootstrap_dir=$(realpath "$script_dir/..")
+	if ! [ -e "$pmbootstrap_dir/pmbootstrap.py" ]; then
+		echo "ERROR: Failed to get the script's location with your shell."
+		echo "Please adjust export_pmbootstrap_dir in envkernel.sh. Debug info:"
+		echo "\$1: $1"
+		echo "\$pmbootstrap_dir: $pmbootstrap_dir"
+		return 1
 	fi
 }
 
 
 set_alias_pmbootstrap() {
-	if [ -n "$pmbootstrap_dir" ] \
+	pmbootstrap="$pmbootstrap_dir"/pmbootstrap.py
-			&& [ -e "$pmbootstrap_dir/pmbootstrap.py" ]; then
+	# shellcheck disable=SC2139
-		pmbootstrap="$pmbootstrap_dir"/pmbootstrap.py
+	alias pmbootstrap="\"$pmbootstrap\""
-		# shellcheck disable=SC2139
-		alias pmbootstrap="\"$pmbootstrap\""
-	elif [ -n "$(command -v pmbootstrap)" ]; then
-		pmbootstrap="$(command -v pmbootstrap)"
-	else
-		echo "ERROR: pmbootstrap not found!"
-		echo "If you're loading envkernel.sh from a pmbootstrap source tree,"
-		echo "please check export_pmbootstrap_dir in envkernel.sh. Otherwise "
-		echo "please make sure 'pmbootstrap' is on your PATH."
-		return 1
-	fi
-
 	if [ -e "${XDG_CONFIG_HOME:-$HOME/.config}"/pmbootstrap.cfg ]; then
 		"$pmbootstrap" work_migrate
 	else
@@ -134,9 +120,7 @@ initialize_chroot() {
 	# shellcheck disable=SC3057
 	arch_substr="${host_arch:0:3}"
 	if [ "$arch" = "$host_arch" ] || \
-		{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; } || \
+		{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; }; then
-		{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; } || \
-		{ [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; }; then
 		need_cross_compiler=0
 	fi
 
@@ -183,9 +167,7 @@ initialize_chroot() {
 		musl-dev \
 		ncurses-dev \
 		perl \
-		py3-dt-schema \
 		sed \
-		yamllint \
 		yaml-dev \
 		xz || return 1
 
@@ -233,14 +215,9 @@ set_alias_make() {
 		cross_compiler="/usr/bin/$prefix-"
 	fi
 
-	if [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; then
-		cc=$hostcc
-	fi
-
 	# Build make command
 	cmd="echo '*** pmbootstrap envkernel.sh active for $PWD! ***';"
 	cmd="$cmd pmbootstrap -q chroot --user --"
-	cmd="$cmd CCACHE_DISABLE=1"
 	cmd="$cmd ARCH=$arch"
 	if [ "$need_cross_compiler" = 1 ]; then
 		cmd="$cmd CROSS_COMPILE=$cross_compiler"
@@ -256,12 +233,11 @@ set_alias_make() {
 	cmd="_run_script() {"
 	cmd="$cmd echo '*** pmbootstrap envkernel.sh active for $PWD! ***';"
 	cmd="$cmd _script=\"\$1\";"
-	cmd="$cmd shift;"
 	cmd="$cmd if [ -e \"\$_script\" ]; then"
 	cmd="$cmd 	echo \"Running \$_script in the chroot native /mnt/linux/\";"
 	cmd="$cmd 	pmbootstrap -q chroot --user -- sh -c \"cd /mnt/linux;"
 	cmd="$cmd 	srcdir=/mnt/linux builddir=/mnt/linux/.output tmpdir=/tmp/envkernel"
-	cmd="$cmd 	./\"\$_script\" \\\"\\\$@\\\"\" \"sh\" \"\$@\";"
+	cmd="$cmd 	./\"\$_script\"\";"
 	cmd="$cmd else"
 	cmd="$cmd 	echo \"ERROR: \$_script not found.\";"
 	cmd="$cmd fi;"
@@ -274,10 +250,8 @@ set_alias_make() {
 
 
 set_alias_pmbroot_kernelroot() {
-	if [ -n "$pmbootstrap_dir" ]; then
+	# shellcheck disable=SC2139
-		# shellcheck disable=SC2139
+	alias pmbroot="cd '$pmbootstrap_dir'"
-		alias pmbroot="cd '$pmbootstrap_dir'"
-	fi
 	# shellcheck disable=SC2139
 	alias kernelroot="cd '$PWD'"
 }
@@ -285,7 +259,7 @@ set_alias_pmbroot_kernelroot() {
 
 cross_compiler_version() {
 	if [ "$need_cross_compiler" = 1 ]; then
-		"$pmbootstrap" chroot --user -- "${cross_compiler}gcc"  --version \
+		pmbootstrap chroot --user -- "${cross_compiler}gcc"  --version \
 			2> /dev/null | grep "^.*gcc " | \
 			awk -F'[()]' '{ print $1 "("$2")" }'
 	else
@@ -309,12 +283,8 @@ update_prompt() {
 set_deactivate() {
 	cmd="_deactivate() {"
 	cmd="$cmd unset POSTMARKETOS_ENVKERNEL_ENABLED;"
-	if [ -n "$pmbootstrap_dir" ]; then
+	cmd="$cmd unalias make kernelroot pmbootstrap pmbroot run-script;"
-		cmd="$cmd unalias pmbootstrap pmbroot;"
-	fi
-	cmd="$cmd unalias make kernelroot run-script;"
 	cmd="$cmd unalias deactivate reactivate;"
-	cmd="$cmd unset pmbootstrap pmbootstrap_dir;"
 	cmd="$cmd if [ -n \"\$_OLD_PS1\" ]; then"
 	cmd="$cmd   export PS1=\"\$_OLD_PS1\";"
 	cmd="$cmd   unset _OLD_PS1;"
@@ -331,12 +301,12 @@ set_deactivate() {
 
 set_reactivate() {
 	# shellcheck disable=SC2139
-	alias reactivate="deactivate; pushd '$PWD'; . '$envkernel_sh'; popd"
+	alias reactivate="deactivate; pushd '$PWD'; . '$pmbootstrap_dir'/helpers/envkernel.sh; popd"
 }
 
 check_and_deactivate() {
 	if [ "$POSTMARKETOS_ENVKERNEL_ENABLED" = 1 ]; then
-		# we already are running in envkernel
+		# we already are runnning in envkernel
 		deactivate
 	fi
 }
@@ -345,7 +315,9 @@ check_and_deactivate() {
 print_usage() {
 	# shellcheck disable=SC3054
 	if [ -n "${BASH_SOURCE[0]}" ]; then
-		echo "usage: source $(basename "$(realpath "$BASH_SOURCE")")"
+		echo "usage: source $(basename "${BASH_SOURCE[0]}")"
+	else
+		echo "usage: source $(basename "$1")"
 	fi
 	echo "optional arguments:"
 	echo "    --fish        Print fish alias syntax (internally used)"
@@ -396,8 +368,7 @@ main() {
 	if check_and_deactivate \
 		&& check_kernel_folder \
 		&& clean_kernel_src_dir \
-		&& export_envkernel_sh "$1" \
+		&& export_pmbootstrap_dir "$1" \
-		&& export_pmbootstrap_dir \
 		&& set_alias_pmbootstrap \
 		&& export_chroot_device_deviceinfo \
 		&& check_device \
@@ -419,13 +390,8 @@ main() {
 		echo " * output folder:  $PWD/.output"
 		echo " * architecture:   $arch ($device is $deviceinfo_arch)"
 		echo " * cross compile:  $(cross_compiler_version)"
-		if [ -n "$pmbootstrap_dir" ]; then
+		echo " * aliases: make, kernelroot, pmbootstrap, pmbroot," \
-			echo " * aliases: make, kernelroot, pmbootstrap, pmbroot," \
+			"run-script (see 'type make' etc.)"
-				"run-script (see 'type make' etc.)"
-		else
-			echo " * aliases: make, kernelroot, run-script"\
-				"(see 'type make' etc.)"
-		fi
 		echo " * run 'deactivate' to revert all env changes"
 	else
 		# Failure
@@ -437,9 +403,9 @@ main() {
 
 # Print fish alias syntax (when called from envkernel.fish)
 fish_compat() {
-	[ "$1" = "--fish" ] || return 0
+	[ "$1" = "--fish" ] || return
 	for name in make kernelroot pmbootstrap pmbroot; do
-		alias "$name" >/dev/null 2>&1 && echo "alias $(alias "$name" | sed 's/=/ /')"
+		echo "alias $(alias $name | sed 's/=/ /')"
 	done
 }
 

pmb/__init__.py

@@ -1,13 +1,10 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 # PYTHON_ARGCOMPLETE_OK
 import sys
 import logging
 import os
 import traceback
-from argparse import Namespace
-
-from pmb.helpers.exceptions import BuildFailedError, NonBugError
 
 from . import config
 from . import parse
@@ -17,27 +14,6 @@ from .helpers import logging as pmb_logging
 from .helpers import mount
 from .helpers import other
 
-# pmbootstrap version
-__version__ = "2.3.1"
-
-# Python version check
-version = sys.version_info
-if version < (3, 9):
-    print("You need at least Python 3.9 to run pmbootstrap")
-    print("(You are running it with Python " + str(version.major) +
-          "." + str(version.minor) + ")")
-    sys.exit()
-
-
-def print_log_hint(args: Namespace) -> None:
-    # Hints about the log file (print to stdout only)
-    log_hint = "Run 'pmbootstrap log' for details."
-    if not args or not os.path.exists(args.log):
-        log_hint += (" Alternatively you can use '--details-to-stdout' to get more"
-                     " output, e.g. 'pmbootstrap --details-to-stdout init'.")
-    print()
-    print(log_hint)
-
 
 def main():
     # Wrap everything to display nice error messages
@@ -47,11 +23,8 @@ def main():
         args = parse.arguments()
         os.umask(0o22)
 
-        # Store script invocation command
-        os.environ["PMBOOTSTRAP_CMD"] = sys.argv[0]
-
         # Sanity checks
-        other.check_grsec()
+        other.check_grsec(args)
         if not args.as_root and os.geteuid() == 0:
             raise RuntimeError("Do not run pmbootstrap as root!")
 
@@ -83,19 +56,6 @@ def main():
                          " shutdown' as necessary)")
         logging.info("DONE!")
 
-    except KeyboardInterrupt:
-        print("\nCaught KeyboardInterrupt, exiting …")
-        sys.exit(130)  # SIGINT(2) + 128
-
-    except NonBugError as exception:
-        logging.error(f"ERROR: {exception}")
-        return 2
-
-    except BuildFailedError as exception:
-        logging.error(f"ERROR: {exception}")
-        print_log_hint(args)
-        return 3
-
     except Exception as e:
         # Dump log to stdout when args (and therefore logging) init failed
         if not args:
@@ -105,13 +65,13 @@ def main():
         logging.info("See also: <https://postmarketos.org/troubleshooting>")
         logging.debug(traceback.format_exc())
 
-        print_log_hint(args)
+        # Hints about the log file (print to stdout only)
-        print()
+        log_hint = "Run 'pmbootstrap log' for details."
-        print("Before you report this error, ensure that pmbootstrap is "
+        if not args or not os.path.exists(args.log):
-              "up to date.")
+            log_hint += (" Alternatively you can use '--details-to-stdout' to"
-        print("Find the latest version here:"
+                         " get more output, e.g. 'pmbootstrap"
-              " https://gitlab.com/postmarketOS/pmbootstrap/-/tags")
+                         " --details-to-stdout init'.")
-        print(f"Your version: {__version__}")
+        print(log_hint)
         return 1
 
 

pmb/aportgen/__init__.py

@@ -1,7 +1,8 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
+import pmb.aportgen.binutils
 import pmb.aportgen.busybox_static
 import pmb.aportgen.device
 import pmb.aportgen.gcc
@@ -12,21 +13,6 @@ import pmb.config
 import pmb.helpers.cli
 
 
-def get_cross_package_arches(pkgname):
-    """
-    Get the arches for which we want to build cross packages.
-    
-    :param pkgname: package name, e.g. "gcc-aarch64", "gcc-x86_64"
-    
-    :returns: string of architecture(s) (space separated)
-    
-    """
-    if pkgname.endswith("-x86_64"):
-        return "aarch64"
-    else:
-        return "x86_64"
-
-
 def properties(pkgname):
     """
     Get the `pmb.config.aportgen` properties for the aport generator, based on
@@ -35,9 +21,7 @@ def properties(pkgname):
     Example: "musl-armhf" => ("musl", "cross", {"confirm_overwrite": False})
 
     :param pkgname: package name
-    
     :returns: (prefix, folder, options)
-    
     """
     for folder, options in pmb.config.aportgen.items():
         for prefix in options["prefixes"]:

pmb/aportgen/binutils.py

@@ -0,0 +1,57 @@
+# Copyright 2021 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+import pmb.aportgen.core
+import pmb.helpers.git
+import pmb.helpers.run
+
+
+def generate(args, pkgname):
+    # Copy original aport
+    arch = pkgname.split("-")[1]
+    upstream = pmb.aportgen.core.get_upstream_aport(args, "binutils")
+    pmb.helpers.run.user(args, ["cp", "-r", upstream, args.work + "/aportgen"])
+
+    # Rewrite APKBUILD
+    fields = {
+        "pkgname": pkgname,
+        "pkgdesc": f"Tools necessary to build programs for {arch} targets",
+        "arch": args.arch_native,
+        "makedepends_build": "",
+        "makedepends_host": "",
+        "makedepends": "gettext libtool autoconf automake bison texinfo",
+        "subpackages": "",
+    }
+
+    replace_functions = {
+        "build": """
+            _target="$(arch_to_hostspec """ + arch + """)"
+            "$builddir"/configure \\
+                --build="$CBUILD" \\
+                --target=$_target \\
+                --with-lib-path=/usr/lib \\
+                --prefix=/usr \\
+                --with-sysroot=/usr/$_target \\
+                --enable-ld=default \\
+                --enable-gold=yes \\
+                --enable-plugins \\
+                --enable-deterministic-archives \\
+                --disable-multilib \\
+                --disable-werror \\
+                --disable-nls
+            make
+        """,
+        "package": """
+            make install DESTDIR="$pkgdir"
+
+            # remove man, info folders
+            rm -rf "$pkgdir"/usr/share
+        """,
+        "libs": None,
+        "gold": None,
+    }
+
+    replace_simple = {"\tsubpackages=*": "\tsubpackages=\"\""}
+
+    pmb.aportgen.core.rewrite(args, pkgname, "main/binutils", fields,
+                              "binutils", replace_functions, replace_simple,
+                              remove_indent=8)

pmb/aportgen/busybox_static.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -27,7 +27,6 @@ def generate(args, pkgname):
     channel_cfg = pmb.config.pmaports.read_config_channel(args)
     mirrordir = channel_cfg["mirrordir_alpine"]
     apkbuild_path = f"{args.work}/chroot_native/{tempdir}/APKBUILD"
-    apk_name = f"busybox-static-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk"
     with open(apkbuild_path, "w", encoding="utf-8") as handle:
         apkbuild = f"""\
             # Automatically generated aport, do not edit!
@@ -47,7 +46,7 @@ def generate(args, pkgname):
 
             url="http://busybox.net"
             license="GPL2"
-            arch="{pmb.aportgen.get_cross_package_arches(pkgname)}"
+            arch="{args.arch_native}"
             options="!check !strip"
             pkgdesc="Statically linked Busybox for $_arch"
             _target="$(arch_to_hostspec $_arch)"
@@ -59,7 +58,7 @@ def generate(args, pkgname):
             package() {{
                 mkdir -p "$pkgdir/usr/$_target"
                 cd "$pkgdir/usr/$_target"
-                tar -xf $srcdir/{apk_name}
+                tar -xf $srcdir/busybox-static-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk
                 rm .PKGINFO .SIGN.*
             }}
         """
@@ -67,7 +66,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init_abuild_minimal(args)
+    pmb.build.init(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/core.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import fnmatch
 import logging
@@ -21,7 +21,6 @@ def format_function(name, body, remove_indent=4):
     """
     Format the body of a shell function passed to rewrite() below, so it fits
     the format of the original APKBUILD.
-
     :param remove_indent: Maximum number of spaces to remove from the
         beginning of each line of the function body.
     """
@@ -57,16 +56,16 @@ def rewrite(args, pkgname, path_original="", fields={}, replace_pkgname=None,
 
     :param path_original: The original path of the automatically generated
         aport.
-    :param fields: key-value pairs of fields that shall be changed in the
+    :param fields: key-value pairs of fields, that shall be changed in the
         APKBUILD. For example: {"pkgdesc": "my new package", "subpkgs": ""}
     :param replace_pkgname: When set, $pkgname gets replaced with that string
         in every line.
     :param replace_functions: Function names and new bodies, for example:
         {"build": "return 0"}
         The body can also be None (deletes the function)
-    :param replace_simple: Lines that fnmatch the pattern, get
+    :param replace_simple: Lines, that fnmatch the pattern, get
         replaced/deleted. Example: {"*test*": "# test", "*mv test.bin*": None}
-    :param below_header: String that gets directly placed below the header.
+    :param below_header: String, that gets directly placed below the header.
     :param remove_indent: Number of spaces to remove from function body
         provided to replace_functions.
 
@@ -166,22 +165,18 @@ def get_upstream_aport(args, pkgname, arch=None):
     pmb.helpers.git.clone(args, "aports_upstream")
     aports_upstream_path = args.work + "/cache_git/aports_upstream"
 
-    if getattr(args, "fork_alpine_retain_branch", False):
+    # Checkout branch
-        logging.info("Not changing aports branch as --fork-alpine-retain-branch was "
+    channel_cfg = pmb.config.pmaports.read_config_channel(args)
-                     "used.")
+    branch = channel_cfg["branch_aports"]
-    else:
+    logging.info(f"Checkout aports.git branch: {branch}")
-        # Checkout branch
+    if pmb.helpers.run.user(args, ["git", "checkout", branch],
-        channel_cfg = pmb.config.pmaports.read_config_channel(args)
+                            aports_upstream_path, check=False):
-        branch = channel_cfg["branch_aports"]
+        logging.info("NOTE: run 'pmbootstrap pull' and try again")
-        logging.info(f"Checkout aports.git branch: {branch}")
+        logging.info("NOTE: if it still fails, your aports.git was cloned with"
-        if pmb.helpers.run.user(args, ["git", "checkout", branch],
+                     " an older version of pmbootstrap, as shallow clone."
-                                aports_upstream_path, check=False):
+                     " Unshallow it, or remove it and let pmbootstrap clone it"
-            logging.info("NOTE: run 'pmbootstrap pull' and try again")
+                     f" again: {aports_upstream_path}")
-            logging.info("NOTE: if it still fails, your aports.git was cloned with"
+        raise RuntimeError("Branch checkout failed.")
-                         " an older version of pmbootstrap, as shallow clone."
-                         " Unshallow it, or remove it and let pmbootstrap clone it"
-                         f" again: {aports_upstream_path}")
-            raise RuntimeError("Branch checkout failed.")
 
     # Search package
     paths = glob.glob(aports_upstream_path + "/*/" + pkgname)
@@ -194,7 +189,7 @@ def get_upstream_aport(args, pkgname, arch=None):
     aport_path = paths[0]
 
     # Parse APKBUILD
-    apkbuild = pmb.parse.apkbuild(f"{aport_path}/APKBUILD",
+    apkbuild = pmb.parse.apkbuild(args, aport_path + "/APKBUILD",
                                   check_pkgname=False)
     apkbuild_version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
 
@@ -207,20 +202,19 @@ def get_upstream_aport(args, pkgname, arch=None):
 
     # Compare version (return when equal)
     compare = pmb.parse.version.compare(apkbuild_version, package["version"])
-
+    if compare == 0:
-    # APKBUILD > binary: this is fine
-    if compare == 1:
-        logging.info(f"NOTE: {pkgname} {arch} binary package has a lower"
-                     f" version {package['version']} than the APKBUILD"
-                     f" {apkbuild_version}")
         return aport_path
 
-    # APKBUILD < binary: aports.git is outdated
+    # Different version message
-    if compare == -1:
+    logging.error("ERROR: Package '" + pkgname + "' has a different version in"
-        logging.warning("WARNING: Package '" + pkgname + "' has a lower version in"
                   " local checkout of Alpine's aports (" + apkbuild_version +
                   ") compared to Alpine's binary package (" +
                   package["version"] + ")!")
-        logging.info("NOTE: You can update your local checkout with: 'pmbootstrap pull'")
 
-    return aport_path
+    # APKBUILD < binary
+    if compare == -1:
+        raise RuntimeError("You can update your local checkout with: "
+                           "'pmbootstrap pull'")
+    # APKBUILD > binary
+    raise RuntimeError("You can force an update of your binary package"
+                       " APKINDEX files with: 'pmbootstrap update'")

pmb/aportgen/device.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -9,14 +9,14 @@ import pmb.parse.apkindex
 import pmb.parse.bootimg
 
 
-def ask_for_architecture():
+def ask_for_architecture(args):
     architectures = pmb.config.build_device_architectures
     # Don't show armhf, new ports shouldn't use this architecture
     if "armhf" in architectures:
         architectures.remove("armhf")
     while True:
-        ret = pmb.helpers.cli.ask("Device architecture", architectures,
+        ret = pmb.helpers.cli.ask(args, "Device architecture", architectures,
-                                  "aarch64", complete=architectures)
+                                  architectures[0], complete=architectures)
         if ret in architectures:
             return ret
         logging.fatal("ERROR: Invalid architecture specified. If you want to"
@@ -25,14 +25,14 @@ def ask_for_architecture():
                       " pmb/config/__init__.py.")
 
 
-def ask_for_manufacturer():
+def ask_for_manufacturer(args):
     logging.info("Who produced the device (e.g. LG)?")
-    return pmb.helpers.cli.ask("Manufacturer", None, None, False)
+    return pmb.helpers.cli.ask(args, "Manufacturer", None, None, False)
 
 
-def ask_for_name(manufacturer):
+def ask_for_name(args, manufacturer):
     logging.info("What is the official name (e.g. Google Nexus 5)?")
-    ret = pmb.helpers.cli.ask("Name", None, None, False)
+    ret = pmb.helpers.cli.ask(args, "Name", None, None, False)
 
     # Always add the manufacturer
     if not ret.startswith(manufacturer) and \
@@ -41,19 +41,19 @@ def ask_for_name(manufacturer):
     return ret
 
 
-def ask_for_year():
+def ask_for_year(args):
     # Regex from https://stackoverflow.com/a/12240826
     logging.info("In what year was the device released (e.g. 2012)?")
-    return pmb.helpers.cli.ask("Year", None, None, False,
+    return pmb.helpers.cli.ask(args, "Year", None, None, False,
                                validation_regex=r'^[1-9]\d{3,}$')
 
 
-def ask_for_chassis():
+def ask_for_chassis(args):
     types = pmb.config.deviceinfo_chassis_types
 
     logging.info("What type of device is it?")
     logging.info("Valid types are: " + ", ".join(types))
-    return pmb.helpers.cli.ask("Chassis", None, None, True,
+    return pmb.helpers.cli.ask(args, "Chassis", None, None, True,
                                validation_regex='|'.join(types),
                                complete=types)
 
@@ -68,13 +68,12 @@ def ask_for_external_storage(args):
                                    " other external storage medium?")
 
 
-def ask_for_flash_method():
+def ask_for_flash_method(args):
     while True:
         logging.info("Which flash method does the device support?")
-        method = pmb.helpers.cli.ask("Flash method",
+        method = pmb.helpers.cli.ask(args, "Flash method",
                                      pmb.config.flash_methods,
-                                     "none",
+                                     pmb.config.flash_methods[0])
-                                     complete=pmb.config.flash_methods)
 
         if method in pmb.config.flash_methods:
             if method == "heimdall":
@@ -85,7 +84,7 @@ def ask_for_flash_method():
                     logging.info("<https://wiki.postmarketos.org/wiki"
                                  "/Deviceinfo_flash_methods#Isorec_or_bootimg"
                                  ".3F>")
-                    heimdall_type = pmb.helpers.cli.ask("Type",
+                    heimdall_type = pmb.helpers.cli.ask(args, "Type",
                                                         heimdall_types,
                                                         heimdall_types[0])
                     if heimdall_type in heimdall_types:
@@ -107,7 +106,7 @@ def ask_for_bootimg(args):
                  " 'pmbootstrap bootimg_analyze').")
 
     while True:
-        response = pmb.helpers.cli.ask("Path", None, "", False)
+        response = pmb.helpers.cli.ask(args, "Path", None, "", False)
         path = os.path.expanduser(response)
         if not path:
             return None
@@ -117,54 +116,33 @@ def ask_for_bootimg(args):
             logging.fatal("ERROR: " + str(e) + ". Please try again.")
 
 
-def generate_deviceinfo_fastboot_content(bootimg=None):
+def generate_deviceinfo_fastboot_content(args, bootimg=None):
     if bootimg is None:
         bootimg = {"cmdline": "",
                    "qcdt": "false",
+                   "mtk_mkimage": "false",
                    "dtb_second": "false",
                    "base": "",
                    "kernel_offset": "",
                    "ramdisk_offset": "",
                    "second_offset": "",
                    "tags_offset": "",
-                   "pagesize": "2048",
+                   "pagesize": "2048"}
-                   "mtk_label_kernel": "",
-                   "mtk_label_ramdisk": ""}
 
     content = f"""\
         deviceinfo_kernel_cmdline="{bootimg["cmdline"]}"
         deviceinfo_generate_bootimg="true"
         deviceinfo_bootimg_qcdt="{bootimg["qcdt"]}"
+        deviceinfo_bootimg_mtk_mkimage="{bootimg["mtk_mkimage"]}"
         deviceinfo_bootimg_dtb_second="{bootimg["dtb_second"]}"
         deviceinfo_flash_pagesize="{bootimg["pagesize"]}"
         """
 
-    if "qcdt_type" in bootimg.keys():
-        content += f"""\
-        deviceinfo_bootimg_qcdt_type="{bootimg["qcdt_type"]}"
-        """
-
-    if "mtk_label_kernel" in bootimg.keys():
-        content += f"""\
-        deviceinfo_mtk_label_kernel="{bootimg["mtk_label_kernel"]}"
-        """
-    if "mtk_label_ramdisk" in bootimg.keys():
-        content += f"""\
-        deviceinfo_mtk_label_ramdisk="{bootimg["mtk_label_ramdisk"]}"
-        """
-
     if "header_version" in bootimg.keys():
         content += f"""\
         deviceinfo_header_version="{bootimg["header_version"]}"
         """
-
+    else:
-        if bootimg["header_version"] == "2":
-            content += f"""\
-            deviceinfo_append_dtb="false"
-            deviceinfo_flash_offset_dtb="{bootimg["dtb_offset"]}"
-            """
-
-    if "base" in bootimg.keys():
         content += f"""\
         deviceinfo_flash_offset_base="{bootimg["base"]}"
         deviceinfo_flash_offset_kernel="{bootimg["kernel_offset"]}"
@@ -193,12 +171,15 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
         deviceinfo_codename="{codename}"
         deviceinfo_year="{year}"
         deviceinfo_dtb=""
+        deviceinfo_modules_initfs=""
         deviceinfo_arch="{arch}"
 
         # Device related
         deviceinfo_chassis="{chassis}"
         deviceinfo_keyboard="{"true" if has_keyboard else "false"}"
         deviceinfo_external_storage="{external_storage}"
+        deviceinfo_screen_width="800"
+        deviceinfo_screen_height="600"
 
         # Bootloader related
         deviceinfo_flash_method="{flash_method}"
@@ -206,13 +187,13 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
 
     content_heimdall_bootimg = """\
         deviceinfo_flash_heimdall_partition_kernel=""
-        deviceinfo_flash_heimdall_partition_rootfs=""
+        deviceinfo_flash_heimdall_partition_system=""
         """
 
     content_heimdall_isorec = """\
         deviceinfo_flash_heimdall_partition_kernel=""
         deviceinfo_flash_heimdall_partition_initfs=""
-        deviceinfo_flash_heimdall_partition_rootfs=""
+        deviceinfo_flash_heimdall_partition_system=""
         """
 
     content_0xffff = """\
@@ -224,9 +205,9 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
         """
 
     if flash_method == "fastboot":
-        content += generate_deviceinfo_fastboot_content(bootimg)
+        content += generate_deviceinfo_fastboot_content(args, bootimg)
     elif flash_method == "heimdall-bootimg":
-        content += generate_deviceinfo_fastboot_content(bootimg)
+        content += generate_deviceinfo_fastboot_content(args, bootimg)
         content += content_heimdall_bootimg
     elif flash_method == "heimdall-isorec":
         content += content_heimdall_isorec
@@ -243,27 +224,6 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
             handle.write(line.lstrip() + "\n")
 
 
-def generate_modules_initfs(args):
-    content = """\
-    # Remove this file if unnecessary (CHANGEME!)
-    # This file shall contain a list of modules to be included in the initramfs,
-    # so that they are available in early boot stages. In general, it should
-    # include modules to support unlocking FDE (touchscreen, panel, etc),
-    # USB networking, and telnet in the debug-shell.
-    # The format is one module name per line. Lines starting with the character
-    # '#', and empty lines are ignored. If there are multiple kernel variants
-    # with different initramfs module requirements, one modules-initfs.$variant
-    # file should be created for each of them.
-    """
-
-    # Write to file
-    pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"])
-    path = args.work + "/aportgen/modules-initfs"
-    with open(path, "w", encoding="utf-8") as handle:
-        for line in content.rstrip().split("\n"):
-            handle.write(line.lstrip() + "\n")
-
-
 def generate_apkbuild(args, pkgname, name, arch, flash_method):
     # Dependencies
     depends = ["postmarketos-base",
@@ -272,6 +232,7 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
         depends.append("mkbootimg")
     if flash_method == "0xffff":
         depends.append("uboot-tools")
+    depends.append("mesa-dri-gallium")
 
     # Whole APKBUILD
     depends.sort()
@@ -280,7 +241,7 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
         # Reference: <https://postmarketos.org/devicepkg>
         pkgname={pkgname}
         pkgdesc="{name}"
-        pkgver=1
+        pkgver=0.1
         pkgrel=0
         url="https://postmarketos.org"
         license="MIT"
@@ -290,10 +251,7 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
             {depends}
         "
         makedepends="devicepkg-dev"
-        source="
+        source="deviceinfo"
-            deviceinfo
-            modules-initfs
-        "
 
         build() {{
             devicepkg_build $startdir $pkgname
@@ -315,14 +273,14 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
 
 
 def generate(args, pkgname):
-    arch = ask_for_architecture()
+    arch = ask_for_architecture(args)
-    manufacturer = ask_for_manufacturer()
+    manufacturer = ask_for_manufacturer(args)
-    name = ask_for_name(manufacturer)
+    name = ask_for_name(args, manufacturer)
-    year = ask_for_year()
+    year = ask_for_year(args)
-    chassis = ask_for_chassis()
+    chassis = ask_for_chassis(args)
     has_keyboard = ask_for_keyboard(args)
     has_external_storage = ask_for_external_storage(args)
-    flash_method = ask_for_flash_method()
+    flash_method = ask_for_flash_method(args)
     bootimg = None
     if flash_method in ["fastboot", "heimdall-bootimg"]:
         bootimg = ask_for_bootimg(args)
@@ -330,5 +288,4 @@ def generate(args, pkgname):
     generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
                         chassis, has_keyboard, has_external_storage,
                         flash_method, bootimg)
-    generate_modules_initfs(args)
     generate_apkbuild(args, pkgname, name, arch, flash_method)

pmb/aportgen/gcc.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.helpers.git
@@ -23,21 +23,22 @@ def generate(args, pkgname):
                          " gcc6.")
     pmb.helpers.run.user(args, ["cp", "-r", upstream, f"{args.work}/aportgen"])
 
-    # Rewrite APKBUILD
+    # Rewrite APKBUILD (only building for native covers most use cases and
+    # saves a lot of build time, can be changed on demand)
     fields = {
         "pkgname": pkgname,
         "pkgdesc": f"Stage2 cross-compiler for {arch}",
-        "arch": pmb.aportgen.get_cross_package_arches(pkgname),
+        "arch": args.arch_native,
-        "depends": f"binutils-{arch} mpc1",
+        "depends": f"isl binutils-{arch} mpc1",
         "makedepends_build": "gcc g++ bison flex texinfo gawk zip"
                              " gmp-dev mpfr-dev mpc1-dev zlib-dev",
         "makedepends_host": "linux-headers gmp-dev mpfr-dev mpc1-dev isl-dev"
                             f" zlib-dev musl-dev-{arch} binutils-{arch}",
-        "subpackages": "",
+        "subpackages": f"g++-{arch}:gpp" if prefix == "gcc" else "",
 
         # gcc6: options is already there, so we need to replace it and not only
         # set it below the header like done below.
-        "options": "!strip",
+        "options": "!strip !tracedeps",
 
         "LIBGOMP": "false",
         "LIBGCC": "false",
@@ -45,11 +46,6 @@ def generate(args, pkgname):
         "LIBITM": "false",
     }
 
-    # Latest gcc only, not gcc4 and gcc6
-    if prefix == "gcc":
-        fields["subpackages"] = f"g++-{arch}:gpp" \
-                                f" libstdc++-dev-{arch}:libcxx_dev"
-
     below_header = "CTARGET_ARCH=" + arch + """
         CTARGET="$(arch_to_hostspec ${CTARGET_ARCH})"
         LANG_D=false
@@ -58,7 +54,7 @@ def generate(args, pkgname):
         LANG_GO=false
         LANG_FORTRAN=false
         LANG_ADA=false
-        options="!strip"
+        options="!strip !tracedeps"
 
         # abuild doesn't try to tries to install "build-base-$CTARGET_ARCH"
         # when this variable matches "no*"
@@ -82,9 +78,6 @@ def generate(args, pkgname):
         # use CBUILDROOT as sysroot. In the original APKBUILD this is a local
         # variable, but we make it a global one.
         '*_cross_configure=*': None,
-
-        # Do not build foreign arch libgcc, we use the one from Alpine (#2168)
-        '_libgcc=true*': '_libgcc=false',
     }
 
     pmb.aportgen.core.rewrite(args, pkgname, based_on, fields,

pmb/aportgen/grub_efi.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Nick Reitemeyer, Oliver Smith
+# Copyright 2021 Nick Reitemeyer, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -27,7 +27,6 @@ def generate(args, pkgname):
     channel_cfg = pmb.config.pmaports.read_config_channel(args)
     mirrordir = channel_cfg["mirrordir_alpine"]
     apkbuild_path = f"{args.work}/chroot_native/{tempdir}/APKBUILD"
-    apk_name = f'"$srcdir/grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk"'
     with open(apkbuild_path, "w", encoding="utf-8") as handle:
         apkbuild = f"""\
             # Automatically generated aport, do not edit!
@@ -43,13 +42,13 @@ def generate(args, pkgname):
             pkgdesc="GRUB $_arch EFI files for every architecture"
             url="https://www.gnu.org/software/grub/"
             license="GPL-3.0-or-later"
-            arch="{pmb.config.arch_native}"
+            arch="{args.arch_native}"
             source="grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk::$_mirror/{mirrordir}/main/$_arch/grub-efi-$pkgver-r$pkgrel.apk"
 
             package() {{
                 mkdir -p "$pkgdir"
                 cd "$pkgdir"
-                tar -xf {apk_name}
+                tar -xf "$srcdir/grub-efi-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk"
                 rm .PKGINFO .SIGN.*
             }}
         """
@@ -57,7 +56,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init_abuild_minimal(args)
+    pmb.build.init(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/aportgen/linux.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.helpers.run
 import pmb.aportgen.core
@@ -10,7 +10,7 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
     device = "-".join(pkgname.split("-")[1:])
     carch = pmb.parse.arch.alpine_to_kernel(deviceinfo["arch"])
 
-    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex",
+    makedepends = ["bash", "bc", "bison", "devicepkg-dev", "flex",
                    "openssl-dev", "perl"]
 
     build = """
@@ -19,20 +19,13 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
                 KBUILD_BUILD_VERSION="$((pkgrel + 1 ))-postmarketOS\""""
 
     package = """
-            downstreamkernel_package "$builddir" "$pkgdir" "$_carch\" \\
+            downstreamkernel_package "$builddir" "$pkgdir" "$_carch" "$_flavor" "$_outdir\""""
-                "$_flavor" "$_outdir\""""
-
-    if deviceinfo.get("header_version") == "2":
-        package += """
-
-            make dtbs_install O="$_outdir" ARCH="$_carch" \\
-                INSTALL_DTBS_PATH="$pkgdir\"/boot/dtbs"""
 
     if deviceinfo["bootimg_qcdt"] == "true":
         build += """\n
             # Master DTB (deviceinfo_bootimg_qcdt)"""
         vendors = ["spreadtrum", "exynos", "other"]
-        soc_vendor = pmb.helpers.cli.ask("SoC vendor", vendors,
+        soc_vendor = pmb.helpers.cli.ask(args, "SoC vendor", vendors,
                                          vendors[-1], complete=vendors)
         if soc_vendor == "spreadtrum":
             makedepends.append("dtbtool-sprd")
@@ -43,10 +36,9 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
         elif soc_vendor == "exynos":
             codename = "-".join(pkgname.split("-")[2:])
             makedepends.append("dtbtool-exynos")
-            build += """
+            build += f"""
             dtbTool-exynos -o "$_outdir/arch/$_carch/boot"/dt.img \\
-                $(find "$_outdir/arch/$_carch/boot/dts/\""""
+                $(find "$_outdir/arch/$_carch/boot/dts/" -name *{codename}*.dtb)"""
-            build += f" -name *{codename}*.dtb)"
         else:
             makedepends.append("dtbtool")
             build += """

pmb/aportgen/musl.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.aportgen.core
 import pmb.build
@@ -27,8 +27,6 @@ def generate(args, pkgname):
     channel_cfg = pmb.config.pmaports.read_config_channel(args)
     mirrordir = channel_cfg["mirrordir_alpine"]
     apkbuild_path = f"{args.work}/chroot_native/{tempdir}/APKBUILD"
-    apk_name = f"$srcdir/musl-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk"
-    apk_dev_name = f"$srcdir/musl-dev-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk"
     with open(apkbuild_path, "w", encoding="utf-8") as handle:
         apkbuild = f"""\
             # Automatically generated aport, do not edit!
@@ -42,7 +40,7 @@ def generate(args, pkgname):
             pkgname={pkgname}
             pkgver={pkgver}
             pkgrel={pkgrel}
-            arch="{pmb.aportgen.get_cross_package_arches(pkgname)}"
+            arch="{args.arch_native}"
             subpackages="musl-dev-{arch}:package_dev"
 
             _arch="{arch}"
@@ -65,7 +63,7 @@ def generate(args, pkgname):
                 cd "$pkgdir/usr/$_target"
                 # Use 'busybox tar' to avoid 'tar: Child returned status 141'
                 # on some machines (builds.sr.ht, gitlab-ci). See pmaports#26.
-                busybox tar -xf {apk_name}
+                busybox tar -xf $srcdir/musl-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk
                 rm .PKGINFO .SIGN.*
             }}
             package_dev() {{
@@ -73,12 +71,11 @@ def generate(args, pkgname):
                 cd "$subpkgdir/usr/$_target"
                 # Use 'busybox tar' to avoid 'tar: Child returned status 141'
                 # on some machines (builds.sr.ht, gitlab-ci). See pmaports#26.
-                busybox tar -xf {apk_dev_name}
+                busybox tar -xf $srcdir/musl-dev-$pkgver-r$pkgrel-$_arch-{mirrordir}.apk
                 rm .PKGINFO .SIGN.*
 
-                # symlink everything from /usr/$_target/usr/*
+                # symlink everything from /usr/$_target/usr/* to /usr/$_target/*
-                # to /usr/$_target/* so the cross-compiler gcc does not fail
+                # so the cross-compiler gcc does not fail to build.
-                # to build.
                 for _dir in include lib; do
                     mkdir -p "$subpkgdir/usr/$_target/$_dir"
                     cd "$subpkgdir/usr/$_target/usr/$_dir"
@@ -93,7 +90,7 @@ def generate(args, pkgname):
             handle.write(line[12:].replace(" " * 4, "\t") + "\n")
 
     # Generate checksums
-    pmb.build.init_abuild_minimal(args)
+    pmb.build.init(args)
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
     pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
     pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

pmb/build/__init__.py

@@ -1,9 +1,9 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-from pmb.build.init import init, init_abuild_minimal, init_compiler
+from pmb.build.init import init, init_compiler
 from pmb.build.envkernel import package_kernel
-from pmb.build.kconfig import menuconfig
+from pmb.build.menuconfig import menuconfig
 from pmb.build.newapkbuild import newapkbuild
 from pmb.build.other import copy_to_buildpath, is_necessary, \
     index_repo
-from pmb.build._package import BootstrapStage, mount_pmaports, package
+from pmb.build._package import mount_pmaports, package

pmb/build/_package.py

@@ -1,7 +1,6 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
-import enum
 import logging
 import os
 
@@ -9,45 +8,34 @@ import pmb.build
 import pmb.build.autodetect
 import pmb.chroot
 import pmb.chroot.apk
+import pmb.chroot.distccd
 import pmb.helpers.pmaports
 import pmb.helpers.repo
 import pmb.parse
 import pmb.parse.arch
-from pmb.helpers.exceptions import BuildFailedError
 
 
-class BootstrapStage(enum.IntEnum):
+def skip_already_built(args, pkgname, arch):
     """
-    Pass a BOOTSTRAP= environment variable with the given value to abuild. See
+    Check if the package was already built in this session, and add it
-    bootstrap_1 etc. at https://postmarketos.org/pmaports.cfg for details.
+    to the cache in case it was not built yet.
-    """
-    NONE = 0
-    # We don't need explicit representations of the other numbers.
-
-
-def skip_already_built(pkgname, arch):
-    """Check if the package was already built in this session.
-
-    Add it to the cache in case it was not built yet.
 
     :returns: True when it can be skipped or False
     """
-    if arch not in pmb.helpers.other.cache["built"]:
+    if arch not in args.cache["built"]:
-        pmb.helpers.other.cache["built"][arch] = []
+        args.cache["built"][arch] = []
-    if pkgname in pmb.helpers.other.cache["built"][arch]:
+    if pkgname in args.cache["built"][arch]:
         logging.verbose(pkgname + ": already checked this session,"
                         " no need to build it or its dependencies")
         return True
-
+    args.cache["built"][arch].append(pkgname)
-    logging.verbose(f"{pkgname}: marking as already built")
-    pmb.helpers.other.cache["built"][arch].append(pkgname)
     return False
 
 
 def get_apkbuild(args, pkgname, arch):
-    """Parse the APKBUILD path for pkgname.
+    """
-
+    Parse the APKBUILD path for pkgname. When there is none, try to find it in
-    When there is none, try to find it in the binary package APKINDEX files or raise an exception.
+    the binary package APKINDEX files or raise an exception.
 
     :param pkgname: package name to be built, as specified in the APKBUILD
     :returns: None or parsed APKBUILD
@@ -66,8 +54,8 @@ def get_apkbuild(args, pkgname, arch):
 
 
 def check_build_for_arch(args, pkgname, arch):
-    """Check if pmaport can be built or exists as binary for a specific arch.
+    """
-
+    Check if pmaport can be built or exists as binary for a specific arch.
     :returns: * True when it can be built
               * False when it can't be built, but exists in a binary repo
                 (e.g. temp/mesa can't be built for x86_64, but Alpine has it)
@@ -100,10 +88,9 @@ def check_build_for_arch(args, pkgname, arch):
 
 
 def get_depends(args, apkbuild):
-    """Alpine's abuild always builds/installs the "depends" and "makedepends" of a package
+    """
-    before building it.
+    Alpine's abuild always builds/installs the "depends" and "makedepends"
-
+    of a package before building it. We used to only care about "makedepends"
-    We used to only care about "makedepends"
     and it's still possible to ignore the depends with --ignore-depends.
 
     :returns: list of dependency pkgnames (eg. ["sdl2", "sdl2_net"])
@@ -127,7 +114,8 @@ def get_depends(args, apkbuild):
 
 
 def build_depends(args, apkbuild, arch, strict):
-    """Get and build dependencies with verbose logging messages.
+    """
+    Get and build dependencies with verbose logging messages.
 
     :returns: (depends, depends_built)
     """
@@ -142,9 +130,6 @@ def build_depends(args, apkbuild, arch, strict):
     if "no_depends" in args and args.no_depends:
         pmb.helpers.repo.update(args, arch)
         for depend in depends:
-            # Ignore conflicting dependencies
-            if depend.startswith("!"):
-                continue
             # Check if binary package is missing
             if not pmb.parse.apkindex.package(args, depend, arch, False):
                 raise RuntimeError("Missing binary package for dependency '" +
@@ -162,8 +147,6 @@ def build_depends(args, apkbuild, arch, strict):
     else:
         # Build the dependencies
         for depend in depends:
-            if depend.startswith("!"):
-                continue
             if package(args, depend, arch, strict=strict):
                 depends_built += [depend]
         logging.verbose(pkgname + ": build dependencies: done, built: " +
@@ -173,7 +156,9 @@ def build_depends(args, apkbuild, arch, strict):
 
 
 def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built):
-    """Check if a build is necessary, and warn if it is not, but there were dependencies built.
+    """
+    Check if a build is necessary, and warn if it is not, but there were
+    dependencies built.
 
     :returns: True or False
     """
@@ -195,13 +180,12 @@ def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built):
 
 def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
                   suffix="native", skip_init_buildenv=False, src=None):
-    """Build all dependencies.
+    """
-
+    Build all dependencies, check if we need to build at all (otherwise we've
-    Check if we need to build at all (otherwise we've
     just initialized the build environment for nothing) and then setup the
     whole build environment (abuild, gcc, dependencies, cross-compiler).
 
-    :param cross: None, "native", or "crossdirect"
+    :param cross: None, "native", "distcc", or "crossdirect"
     :param skip_init_buildenv: can be set to False to avoid initializing the
                                build environment. Use this when building
                                something during initialization of the build
@@ -212,7 +196,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
 
     depends_arch = arch
     if cross == "native":
-        depends_arch = pmb.config.arch_native
+        depends_arch = args.arch_native
 
     # Build dependencies
     depends, built = build_depends(args, apkbuild, depends_arch, strict)
@@ -225,10 +209,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     if not skip_init_buildenv:
         pmb.build.init(args, suffix)
         pmb.build.other.configure_abuild(args, suffix)
-        if args.ccache:
+        pmb.build.other.configure_ccache(args, suffix)
-            pmb.build.other.configure_ccache(args, suffix)
-            if "rust" in depends or "cargo" in depends:
-                pmb.chroot.apk.install(args, ["sccache"], suffix)
     if not strict and "pmb:strict" not in apkbuild["options"] and len(depends):
         pmb.chroot.apk.install(args, depends, suffix)
     if src:
@@ -237,18 +218,33 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     # Cross-compiler init
     if cross:
         pmb.build.init_compiler(args, depends, cross, arch)
+    if cross == "distcc":
+        pmb.chroot.distccd.start(args, arch)
     if cross == "crossdirect":
         pmb.chroot.mount_native_into_foreign(args, suffix)
 
     return True
 
 
-def get_pkgver(original_pkgver, original_source=False, now=None):
+def get_gcc_version(args, arch):
-    """Get the original pkgver when using the original source.
+    """
+    Get the GCC version for a specific arch from parsing the right APKINDEX.
+    We feed this to ccache, so it knows the right GCC version, when
+    cross-compiling in a foreign arch chroot with distcc. See the "using
+    ccache with other compiler wrappers" section of their man page:
+    <https://linux.die.net/man/1/ccache>
+    :returns: a string like "6.4.0-r5"
+    """
+    return pmb.parse.apkindex.package(args, "gcc-" + arch,
+                                      args.arch_native)["version"]
 
-    Otherwise, get the pkgver with an appended suffix of current date and time.
+
-    For example: ``_p20180218550502``
+def get_pkgver(original_pkgver, original_source=False, now=None):
-    When appending the suffix, an existing suffix (e.g. ``_git20171231``) gets
+    """
+    Get the original pkgver when using the original source. Otherwise, get the
+    pkgver with an appended suffix of current date and time. For example:
+        _p20180218550502
+    When appending the suffix, an existing suffix (e.g. _git20171231) gets
     replaced.
 
     :param original_pkgver: unmodified pkgver from the package's APKBUILD.
@@ -267,14 +263,15 @@ def get_pkgver(original_pkgver, original_source=False, now=None):
 
 
 def override_source(args, apkbuild, pkgver, src, suffix="native"):
-    """Mount local source inside chroot and append new functions (prepare() etc.)
+    """
+    Mount local source inside chroot and append new functions (prepare() etc.)
     to the APKBUILD to make it use the local source.
     """
     if not src:
         return
 
     # Mount source in chroot
-    mount_path = "/mnt/pmbootstrap/source-override/"
+    mount_path = "/mnt/pmbootstrap-source-override/"
     mount_path_outside = args.work + "/chroot_" + suffix + mount_path
     pmb.helpers.mount.bind(args, src, mount_path_outside, umount=True)
 
@@ -344,7 +341,8 @@ def mount_pmaports(args, destination, suffix="native"):
 
 
 def link_to_git_dir(args, suffix):
-    """ Make ``/home/pmos/build/.git`` point to the .git dir from pmaports.git, with a
+    """
+    Make /home/pmos/build/.git point to the .git dir from pmaports.git, with a
     symlink so abuild does not fail (#1841).
 
     abuild expects the current working directory to be a subdirectory of a
@@ -372,15 +370,14 @@ def link_to_git_dir(args, suffix):
 
 
 def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
-               suffix="native", src=None, bootstrap_stage=BootstrapStage.NONE):
+               suffix="native", src=None):
     """
     Set up all environment variables and construct the abuild command (all
     depending on the cross-compiler method and target architecture), copy
     the aport to the chroot and execute abuild.
 
-    :param cross: None, "native", or "crossdirect"
+    :param cross: None, "native", "distcc", or "crossdirect"
     :param src: override source used to build the package with a local folder
-    :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild
     :returns: (output, cmd, env), output is the destination apk path relative
               to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
               used by the test case, and they are the full abuild command and
@@ -408,31 +405,24 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
         hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
         env["CROSS_COMPILE"] = hostspec + "-"
         env["CC"] = hostspec + "-gcc"
+    if cross == "distcc":
+        env["CCACHE_PREFIX"] = "distcc"
+        env["CCACHE_PATH"] = f"/usr/lib/arch-bin-masquerade/{arch}:/usr/bin"
+        env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
+        env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
+        env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
+                             args.port_distccd)
+        env["DISTCC_BACKOFF_PERIOD"] = "0"
+        if not args.distcc_fallback:
+            env["DISTCC_FALLBACK"] = "0"
+        if args.verbose:
+            env["DISTCC_VERBOSE"] = "1"
     if cross == "crossdirect":
         env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
                                 pmb.config.chroot_path])
     if not args.ccache:
         env["CCACHE_DISABLE"] = "1"
 
-    # Use sccache without crossdirect (crossdirect uses it via rustc.sh)
-    if args.ccache and cross != "crossdirect":
-        env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
-
-    # Cache binary objects from go in this path (like ccache)
-    env["GOCACHE"] = "/home/pmos/.cache/go-build"
-
-    # Cache go modules (git repositories). Usually these should be bundled and
-    # it should not be required to download them at build time, in that case
-    # the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But
-    # e.g. when using --src they are not bundled, in that case it makes sense
-    # to point GOMODCACHE at pmbootstrap's work dir so the modules are only
-    # downloaded once.
-    if args.go_mod_cache:
-        env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
-
-    if bootstrap_stage:
-        env["BOOTSTRAP"] = str(bootstrap_stage)
-
     # Build the abuild command
     cmd = ["abuild", "-D", "postmarketOS"]
     if strict or "pmb:strict" in apkbuild["options"]:
@@ -454,7 +444,9 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
 
 
 def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
-    """Various finishing tasks that need to be done after a build."""
+    """
+    Various finishing tasks that need to be done after a build.
+    """
     # Verify output file
     channel = pmb.config.pmaports.read_config(args)["channel"]
     path = f"{args.work}/packages/{channel}/{output}"
@@ -464,22 +456,18 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
     # Clear APKINDEX cache (we only parse APKINDEX files once per session and
     # cache the result for faster dependency resolving, but after we built a
     # package we need to parse it again)
-    pmb.parse.apkindex.clear_cache(f"{args.work}/packages/{channel}"
+    pmb.parse.apkindex.clear_cache(args, f"{args.work}/packages/{channel}"
-                                   f"/{arch}/APKINDEX.tar.gz")
+                                         f"/{arch}/APKINDEX.tar.gz")
 
     # Uninstall build dependencies (strict mode)
     if strict or "pmb:strict" in apkbuild["options"]:
         logging.info("(" + suffix + ") uninstall build dependencies")
         pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build",
                         env={"SUDO_APK": "abuild-apk --no-progress"})
-        # If the build depends contain postmarketos-keys or postmarketos-base,
-        # abuild will have removed the postmarketOS repository key (pma#1230)
-        pmb.chroot.init_keys(args)
 
 
 def package(args, pkgname, arch=None, force=False, strict=False,
-            skip_init_buildenv=False, src=None,
+            skip_init_buildenv=False, src=None):
-            bootstrap_stage=BootstrapStage.NONE):
     """
     Build a package and its dependencies with Alpine Linux' abuild.
 
@@ -498,17 +486,12 @@ def package(args, pkgname, arch=None, force=False, strict=False,
                                something during initialization of the build
                                environment (e.g. qemu aarch64 bug workaround)
     :param src: override source used to build the package with a local folder
-    :param bootstrap_stage: pass a BOOTSTRAP= env var with the value to abuild
     :returns: None if the build was not necessary
               output path relative to the packages folder ("armhf/ab-1-r2.apk")
     """
-    logging.verbose(f"{pkgname}: running pmb.build._package.package")
-
     # Once per session is enough
-    arch = arch or pmb.config.arch_native
+    arch = arch or args.arch_native
-    # the order of checks here is intentional,
+    if skip_already_built(args, pkgname, arch):
-    # skip_already_built() has side effects!
-    if skip_already_built(pkgname, arch) and not force:
         return
 
     # Only build when APKBUILD exists
@@ -519,17 +502,14 @@ def package(args, pkgname, arch=None, force=False, strict=False,
     # Detect the build environment (skip unnecessary builds)
     if not check_build_for_arch(args, pkgname, arch):
         return
-    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
+    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
     cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
     if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
                          skip_init_buildenv, src):
         return
 
-    try:
+    # Build and finish up
-        # Build and finish up
+    (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
-        (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
+                                    suffix, src)
-                                        suffix, src, bootstrap_stage)
-    except RuntimeError:
-        raise BuildFailedError(f"Build for {arch}/{pkgname} failed!")
     finish(args, apkbuild, arch, output, strict, suffix)
     return output

pmb/build/autodetect.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -40,7 +40,7 @@ def arch(args, pkgname):
     :returns: arch string like "x86_64" or "armhf". Preferred order, depending
               on what is supported by the APKBUILD:
               * native arch
-              * device arch (this will be preferred instead if build_default_device_arch is true)
+              * device arch
               * first arch in the APKBUILD
     """
     aport = pmb.helpers.pmaports.find(args, pkgname)
@@ -48,21 +48,14 @@ def arch(args, pkgname):
     if ret:
         return ret
 
-    apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
+    apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
     arches = apkbuild["arch"]
+    if "noarch" in arches or "all" in arches or args.arch_native in arches:
+        return args.arch_native
 
-    if args.build_default_device_arch:
+    arch_device = args.deviceinfo["arch"]
-        preferred_arch = args.deviceinfo["arch"]
+    if arch_device in arches:
-        preferred_arch_2nd = pmb.config.arch_native
+        return arch_device
-    else:
-        preferred_arch = pmb.config.arch_native
-        preferred_arch_2nd = args.deviceinfo["arch"]
-
-    if "noarch" in arches or "all" in arches or preferred_arch in arches:
-        return preferred_arch
-
-    if preferred_arch_2nd in arches:
-        return preferred_arch_2nd
 
     try:
         return apkbuild["arch"][0]
@@ -70,8 +63,8 @@ def arch(args, pkgname):
         return None
 
 
-def suffix(apkbuild, arch):
+def suffix(args, apkbuild, arch):
-    if arch == pmb.config.arch_native:
+    if arch == args.arch_native:
         return "native"
 
     if "pmb:cross-native" in apkbuild["options"]:
@@ -82,14 +75,14 @@ def suffix(apkbuild, arch):
 
 def crosscompile(args, apkbuild, arch, suffix):
     """
-        :returns: None, "native", "crossdirect"
+        :returns: None, "native", "crossdirect" or "distcc"
     """
     if not args.cross:
         return None
-    if not pmb.parse.arch.cpu_emulation_required(arch):
+    if not pmb.parse.arch.cpu_emulation_required(args, arch):
         return None
     if suffix == "native":
         return "native"
-    if "!pmb:crossdirect" in apkbuild["options"]:
+    if args.no_crossdirect or "!pmb:crossdirect" in apkbuild["options"]:
-        return None
+        return "distcc"
     return "crossdirect"

pmb/build/checksum.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 
@@ -9,8 +9,8 @@ import pmb.helpers.pmaports
 
 
 def update(args, pkgname):
-    """Fetch all sources and update the checksums in the APKBUILD."""
+    """ Fetch all sources and update the checksums in the APKBUILD. """
-    pmb.build.init_abuild_minimal(args)
+    pmb.build.init(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) generate checksums for " + pkgname)
     pmb.chroot.user(args, ["abuild", "checksum"],
@@ -23,8 +23,8 @@ def update(args, pkgname):
 
 
 def verify(args, pkgname):
-    """Fetch all sources and verify their checksums."""
+    """ Fetch all sources and verify their checksums. """
-    pmb.build.init_abuild_minimal(args)
+    pmb.build.init(args)
     pmb.build.copy_to_buildpath(args, pkgname)
     logging.info("(native) verify checksums for " + pkgname)
 

pmb/build/envkernel.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Robert Yang
+# Copyright 2021 Robert Yang
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -12,13 +12,14 @@ import pmb.helpers.pmaports
 import pmb.parse
 
 
-def match_kbuild_out(word):
+def match_kbuild_out(args, word):
-    """Look for paths in the following formats:
+    """
+    Look for paths in the following formats:
       "<prefix>/<kbuild_out>/arch/<arch>/boot"
       "<prefix>/<kbuild_out>/include/config/kernel.release"
 
     :param word: space separated string cut out from a line from an APKBUILD
-                 function body that might be the kbuild output path
+                 function body, that might be the kbuild output path
     :returns: kernel build output directory.
               empty string when a separate build output directory isn't used.
               None, when no output directory is found.
@@ -46,25 +47,21 @@ def match_kbuild_out(word):
     return "" if out_dir is None else out_dir.strip("/")
 
 
-def find_kbuild_output_dir(function_body):
+def find_kbuild_output_dir(args, function_body):
-    """Guess what the kernel build output directory is.
+    """
-
+    Guess what the kernel build output directory is. Parses each line of the
-    Parses each line of the function word by word, looking for paths which
+    function word by word, looking for paths which contain the kbuild output
-    contain the kbuild output directory.
+    directory.
 
     :param function_body: contents of a function from the kernel APKBUILD
     :returns: kbuild output dir
               None, when output dir is not found
     """
+
     guesses = []
     for line in function_body:
         for item in line.split():
-            # Guess that any APKBUILD using downstreamkernel_package
+            kbuild_out = match_kbuild_out(args, item)
-            # uses the default kbuild out directory.
-            if item == "downstreamkernel_package":
-                guesses.append("")
-                break
-            kbuild_out = match_kbuild_out(item)
             if kbuild_out is not None:
                 guesses.append(kbuild_out)
                 break
@@ -85,9 +82,11 @@ def find_kbuild_output_dir(function_body):
 
 
 def modify_apkbuild(args, pkgname, aport):
-    """Modify kernel APKBUILD to package build output from envkernel.sh."""
+    """
+    Modify kernel APKBUILD to package build output from envkernel.sh
+    """
     apkbuild_path = aport + "/APKBUILD"
-    apkbuild = pmb.parse.apkbuild(apkbuild_path)
+    apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
     if os.path.exists(args.work + "/aportgen"):
         pmb.helpers.run.user(args, ["rm", "-r", args.work + "/aportgen"])
 
@@ -118,20 +117,10 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     build_path = "/home/pmos/build"
     kbuild_out_source = "/mnt/linux/.output"
 
-    # If the kernel was cross-compiled on the host rather than with the envkernel
-    # helper, we can still use the envkernel logic to package the artifacts for
-    # development, making it easy to quickly sideload a new kernel or pmbootstrap
-    # to create a boot image.
-
-    pmb.helpers.mount.bind(args, ".", f"{chroot}/mnt/linux")
-
     if not os.path.exists(chroot + kbuild_out_source):
-        raise RuntimeError("No '.output' dir found in your kernel source dir. "
+        raise RuntimeError("No '.output' dir found in your kernel source dir."
-                           "Compile the " + args.device + " kernel first and "
+                           "Compile the " + args.device + " kernel with "
-                           "then try again. See https://postmarketos.org/envkernel"
+                           "envkernel.sh first, then try again.")
-                           "for details. If building on your host and only using "
-                           "--envkernel for packaging, make sure you have O=.output "
-                           "as an argument to make.")
 
     # Create working directory for abuild
     pmb.build.copy_to_buildpath(args, pkgname)
@@ -153,14 +142,11 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
     # Create the apk package
     env = {"CARCH": arch,
            "CHOST": arch,
-           "CBUILD": pmb.config.arch_native,
+           "CBUILD": args.arch_native,
            "SUDO_APK": "abuild-apk --no-progress"}
     cmd = ["abuild", "rootpkg"]
     pmb.chroot.user(args, cmd, working_dir=build_path, env=env)
 
-    # Clean up bindmount
-    pmb.helpers.mount.umount_all(args, f"{chroot}/mnt/linux")
-
     # Clean up symlinks
     if build_output != "":
         if os.path.islink(chroot + "/mnt/linux/" + build_output) and \
@@ -170,7 +156,10 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
 
 
 def package_kernel(args):
-    """Frontend for 'pmbootstrap build --envkernel': creates a package from envkernel output."""
+    """
+    Frontend for 'pmbootstrap build --envkernel': creates a package from
+    envkernel output.
+    """
     pkgname = args.packages[0]
     if len(args.packages) > 1 or not pkgname.startswith("linux-"):
         raise RuntimeError("--envkernel needs exactly one linux-* package as "
@@ -182,20 +171,18 @@ def package_kernel(args):
     apkbuild_path = args.work + "/aportgen/APKBUILD"
 
     arch = args.deviceinfo["arch"]
-    apkbuild = pmb.parse.apkbuild(apkbuild_path, check_pkgname=False)
+    apkbuild = pmb.parse.apkbuild(args, apkbuild_path, check_pkgname=False)
     if apkbuild["_outdir"]:
         kbuild_out = apkbuild["_outdir"]
     else:
         function_body = pmb.parse.function_body(aport + "/APKBUILD", "package")
-        kbuild_out = find_kbuild_output_dir(function_body)
+        kbuild_out = find_kbuild_output_dir(args, function_body)
-    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
+    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
 
     # Install package dependencies
     depends, _ = pmb.build._package.build_depends(
-        args, apkbuild, pmb.config.arch_native, strict=False)
+        args, apkbuild, args.arch_native, strict=False)
     pmb.build.init(args, suffix)
-    if pmb.parse.arch.cpu_emulation_required(arch):
-        depends.append("binutils-" + arch)
     pmb.chroot.apk.install(args, depends, suffix)
 
     output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +
@@ -203,9 +190,5 @@ def package_kernel(args):
     message = "(" + suffix + ") build " + output
     logging.info(message)
 
-    try:
+    run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out)
-        run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out)
-    except Exception as e:
-        pmb.helpers.mount.umount_all(args, f"{args.work}/chroot_native/mnt/linux")
-        raise e
     pmb.build.other.index_repo(args, arch)

pmb/build/init.py

@@ -1,25 +1,25 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import glob
-import logging
 import os
-import pathlib
+import logging
+import glob
 
 import pmb.build
 import pmb.config
 import pmb.chroot
 import pmb.chroot.apk
 import pmb.helpers.run
-import pmb.parse.arch
 
 
-def init_abuild_minimal(args, suffix="native"):
+def init(args, suffix="native"):
-    """Initialize a minimal chroot with abuild where one can do 'abuild checksum'."""
+    # Check if already initialized
-    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_abuild_init_done"
+    marker = "/var/local/pmbootstrap_chroot_build_init_done"
-    if os.path.exists(marker):
+    if os.path.exists(args.work + "/chroot_" + suffix + marker):
         return
 
-    pmb.chroot.apk.install(args, ["abuild"], suffix, build=False)
+    # Initialize chroot, install packages
+    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
+                           build=False)
 
     # Fix permissions
     pmb.chroot.root(args, ["chown", "root:abuild",
@@ -27,24 +27,6 @@ def init_abuild_minimal(args, suffix="native"):
     pmb.chroot.root(args, ["chmod", "g+w",
                            "/var/cache/distfiles"], suffix)
 
-    # Add user to group abuild
-    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
-
-    pathlib.Path(marker).touch()
-
-
-def init(args, suffix="native"):
-    """Initialize a chroot for building packages with abuild."""
-    marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_build_init_done"
-    if os.path.exists(marker):
-        return
-
-    init_abuild_minimal(args, suffix)
-
-    # Initialize chroot, install packages
-    pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
-                           build=False)
-
     # Generate package signing keys
     chroot = args.work + "/chroot_" + suffix
     if not os.path.exists(args.work + "/config_abuild/abuild.conf"):
@@ -54,44 +36,35 @@ def init(args, suffix="native"):
 
         # Copy package signing key to /etc/apk/keys
         for key in glob.glob(chroot +
-                             "/mnt/pmbootstrap/abuild-config/*.pub"):
+                             "/mnt/pmbootstrap-abuild-config/*.pub"):
             key = key[len(chroot):]
             pmb.chroot.root(args, ["cp", key, "/etc/apk/keys/"], suffix)
 
-    apk_arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
+    # Add gzip wrapper, that converts '-9' to '-1'
-
+    if not os.path.exists(chroot + "/usr/local/bin/gzip"):
-    # Add apk wrapper that runs native apk and lies about arch
+        with open(chroot + "/tmp/gzip_wrapper.sh", "w") as handle:
-    if pmb.parse.arch.cpu_emulation_required(apk_arch) and \
+            content = """
-            not os.path.exists(chroot + "/usr/local/bin/abuild-apk"):
-        with open(chroot + "/tmp/apk_wrapper.sh", "w") as handle:
-            content = f"""
                 #!/bin/sh
-
+                # Simple wrapper, that converts -9 flag for gzip to -1 for
-                # With !pmb:crossdirect, cross compilation is entriely done
+                # speed improvement with abuild. FIXME: upstream to abuild
-                # in QEMU, no /native dir gets mounted inside the foreign arch
+                # with a flag!
-                # chroot.
-                if ! [ -d /native ]; then
-                    exec /usr/bin/abuild-apk "$@"
-                fi
-
-                export LD_PRELOAD_PATH=/native/usr/lib:/native/lib
                 args=""
                 for arg in "$@"; do
-                    if [ "$arg" == "--print-arch" ]; then
+                    [ "$arg" == "-9" ] && arg="-1"
-                        echo "{apk_arch}"
-                        exit 0
-                    fi
                     args="$args $arg"
                 done
-                /native/usr/bin/abuild-apk $args
+                /bin/gzip $args
             """
             lines = content.split("\n")[1:]
             for i in range(len(lines)):
                 lines[i] = lines[i][16:]
             handle.write("\n".join(lines))
-        pmb.chroot.root(args, ["cp", "/tmp/apk_wrapper.sh",
+        pmb.chroot.root(args, ["cp", "/tmp/gzip_wrapper.sh",
-                               "/usr/local/bin/abuild-apk"], suffix)
+                               "/usr/local/bin/gzip"], suffix)
-        pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/abuild-apk"], suffix)
+        pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix)
+
+    # Add user to group abuild
+    pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
 
     # abuild.conf: Don't clean the build folder after building, so we can
     # inspect it afterwards for debugging
@@ -104,11 +77,12 @@ def init(args, suffix="native"):
                            "s/^ERROR_CLEANUP=.*/ERROR_CLEANUP=''/",
                            "/etc/abuild.conf"], suffix)
 
-    pathlib.Path(marker).touch()
+    # Mark the chroot as initialized
+    pmb.chroot.root(args, ["touch", marker], suffix)
 
 
 def init_compiler(args, depends, cross, arch):
-    cross_pkgs = ["ccache-cross-symlinks", "abuild"]
+    cross_pkgs = ["ccache-cross-symlinks"]
     if "gcc4" in depends:
         cross_pkgs += ["gcc4-" + arch]
     elif "gcc6" in depends:
@@ -120,11 +94,6 @@ def init_compiler(args, depends, cross, arch):
     if cross == "crossdirect":
         cross_pkgs += ["crossdirect"]
         if "rust" in depends or "cargo" in depends:
-            if args.ccache:
+            cross_pkgs += ["rust"]
-                cross_pkgs += ["sccache"]
-            # crossdirect for rust installs all build dependencies in the
-            # native chroot too, as some of them can be required for building
-            # native macros / build scripts
-            cross_pkgs += depends
 
     pmb.chroot.apk.install(args, cross_pkgs)

pmb/build/menuconfig.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -14,16 +14,14 @@ import pmb.helpers.run
 import pmb.parse
 
 
-def get_arch(apkbuild):
+def get_arch(args, apkbuild):
-    """Take the architecture from the APKBUILD or complain if it's ambiguous.
+    """
-
+    Take the architecture from the APKBUILD or complain if it's ambiguous. This
-    This function only gets called if --arch is not set.
+    function only gets called if --arch is not set.
 
     :param apkbuild: looks like: {"pkgname": "linux-...",
                                   "arch": ["x86_64", "armhf", "aarch64"]}
-
+                     or: {"pkgname": "linux-...", "arch": ["armhf"]}
-    or: {"pkgname": "linux-...", "arch": ["armhf"]}
-
     """
     pkgname = apkbuild["pkgname"]
 
@@ -42,8 +40,8 @@ def get_arch(apkbuild):
 
 
 def get_outputdir(args, pkgname, apkbuild):
-    """Get the folder for the kernel compilation output.
+    """
-
+    Get the folder for the kernel compilation output.
     For most APKBUILDs, this is $builddir. But some older ones still use
     $srcdir/build (see the discussion in #1551).
     """
@@ -80,26 +78,20 @@ def get_outputdir(args, pkgname, apkbuild):
                        " template with: pmbootstrap aportgen " + pkgname)
 
 
-def extract_and_patch_sources(args, pkgname, arch):
+def menuconfig(args, pkgname):
-    pmb.build.copy_to_buildpath(args, pkgname)
-    logging.info("(native) extract kernel source")
-    pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
-    logging.info("(native) apply patches")
-    pmb.chroot.user(args, ["abuild", "prepare"], "native",
-                    "/home/pmos/build", output="interactive",
-                    env={"CARCH": arch})
-
-
-def menuconfig(args, pkgname, use_oldconfig):
     # Pkgname: allow omitting "linux-" prefix
-    if not pkgname.startswith("linux-"):
+    if pkgname.startswith("linux-"):
+        pkgname_ = pkgname.split("linux-")[1]
+        logging.info("PROTIP: You can simply do 'pmbootstrap kconfig edit " +
+                     pkgname_ + "'")
+    else:
         pkgname = "linux-" + pkgname
 
     # Read apkbuild
     aport = pmb.helpers.pmaports.find(args, pkgname)
-    apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
+    apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
-    arch = args.arch or get_arch(apkbuild)
+    arch = args.arch or get_arch(args, apkbuild)
-    suffix = pmb.build.autodetect.suffix(apkbuild, arch)
+    suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
     cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
     hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
 
@@ -107,34 +99,32 @@ def menuconfig(args, pkgname, use_oldconfig):
     pmb.build.init(args, suffix)
     if cross:
         pmb.build.init_compiler(args, [], cross, arch)
-
     depends = apkbuild["makedepends"]
+    kopt = "menuconfig"
     copy_xauth = False
-
+    if args.xconfig:
-    if use_oldconfig:
+        depends += ["qt5-qtbase-dev", "font-noto"]
-        kopt = "oldconfig"
+        kopt = "xconfig"
+        copy_xauth = True
+    elif args.nconfig:
+        kopt = "nconfig"
+        depends += ["ncurses-dev"]
     else:
-        kopt = "menuconfig"
+        depends += ["ncurses-dev"]
-        if args.xconfig:
-            depends += ["qt5-qtbase-dev", "font-noto"]
-            kopt = "xconfig"
-            copy_xauth = True
-        elif args.nconfig:
-            kopt = "nconfig"
-            depends += ["ncurses-dev"]
-        else:
-            depends += ["ncurses-dev"]
-
     pmb.chroot.apk.install(args, depends)
 
     # Copy host's .xauthority into native
     if copy_xauth:
         pmb.chroot.other.copy_xauthority(args)
 
-    extract_and_patch_sources(args, pkgname, arch)
+    # Patch and extract sources
-
+    pmb.build.copy_to_buildpath(args, pkgname)
-    # Check for background color variable
+    logging.info("(native) extract kernel source")
-    color = os.environ.get("MENUCONFIG_COLOR")
+    pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
+    logging.info("(native) apply patches")
+    pmb.chroot.user(args, ["abuild", "prepare"], "native",
+                    "/home/pmos/build", output="interactive",
+                    env={"CARCH": arch})
 
     # Run make menuconfig
     outputdir = get_outputdir(args, pkgname, apkbuild)
@@ -145,8 +135,6 @@ def menuconfig(args, pkgname, use_oldconfig):
     if cross:
         env["CROSS_COMPILE"] = f"{hostspec}-"
         env["CC"] = f"{hostspec}-gcc"
-    if color:
-        env["MENUCONFIG_COLOR"] = color
     pmb.chroot.user(args, ["make", kopt], "native",
                     outputdir, output="tui", env=env)
 
@@ -163,4 +151,8 @@ def menuconfig(args, pkgname, use_oldconfig):
     pmb.build.checksum.update(args, pkgname)
 
     # Check config
-    pmb.parse.kconfig.check(args, apkbuild["_flavor"], details=True)
+    pmb.parse.kconfig.check(args, apkbuild["_flavor"], force_anbox_check=False,
+                            force_nftables_check=False,
+                            force_containers_check=False,
+                            force_zram_check=False,
+                            details=True)

pmb/build/newapkbuild.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import os
@@ -25,7 +25,7 @@ def newapkbuild(args, folder, args_passed, force=False):
 
     # Paths for copying
     source_apkbuild = glob_result[0]
-    pkgname = pmb.parse.apkbuild(source_apkbuild, False)["pkgname"]
+    pkgname = pmb.parse.apkbuild(args, source_apkbuild, False)["pkgname"]
     target = args.aports + "/" + folder + "/" + pkgname
 
     # Move /home/pmos/build/$pkgname/* to /home/pmos/build/*

pmb/build/other.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -28,35 +28,27 @@ def copy_to_buildpath(args, package, suffix="native"):
         pmb.chroot.root(args, ["rm", "-rf", "/home/pmos/build"], suffix)
 
     # Copy aport contents with resolved symlinks
-    pmb.helpers.run.root(args, ["mkdir", "-p", build])
+    pmb.helpers.run.root(args, ["cp", "-rL", aport + "/", build])
-    for entry in os.listdir(aport):
-        # Don't copy those dirs, as those have probably been generated by running `abuild`
-        # on the host system directly and not cleaning up after itself.
-        # Those dirs might contain broken symlinks and cp fails resolving them.
-        if entry in ["src", "pkg"]:
-            logging.warn(f"WARNING: Not copying {entry}, looks like a leftover from abuild")
-            continue
-        pmb.helpers.run.root(args, ["cp", "-rL", f"{aport}/{entry}", f"{build}/{entry}"])
-
     pmb.chroot.root(args, ["chown", "-R", "pmos:pmos",
                            "/home/pmos/build"], suffix)
 
 
 def is_necessary(args, arch, apkbuild, indexes=None):
-    """Check if the package has already been built.
+    """
-
+    Check if the package has already been built. Compared to abuild's check,
-    Compared to abuild's check, this check also works for different architectures.
+    this check also works for different architectures.
 
     :param arch: package target architecture
     :param apkbuild: from pmb.parse.apkbuild()
     :param indexes: list of APKINDEX.tar.gz paths
     :returns: boolean
     """
+    # Get package name, version, define start of debug message
     package = apkbuild["pkgname"]
-    version_pmaports = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
+    version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
     msg = "Build is necessary for package '" + package + "': "
 
-    # Get version from APKINDEX
+    # Get old version from APKINDEX
     index_data = pmb.parse.apkindex.package(args, package, arch, False,
                                             indexes)
     if not index_data:
@@ -71,25 +63,29 @@ def is_necessary(args, arch, apkbuild, indexes=None):
         return False
 
     # a) Binary repo has a newer version
-    version_binary = index_data["version"]
+    version_old = index_data["version"]
-    if pmb.parse.version.compare(version_binary, version_pmaports) == 1:
+    if pmb.parse.version.compare(version_old, version_new) == 1:
-        logging.warning(f"WARNING: about to install {package} {version_binary}"
+        logging.warning("WARNING: package {}: aport version {} is lower than"
-                        f" (local pmaports: {version_pmaports}, consider"
+                        " {} from the binary repository. {} will be used when"
-                        " 'pmbootstrap pull')")
+                        " installing {}. See also:"
+                        " <https://postmarketos.org/warning-repo2>"
+                        "".format(package, version_new, version_old,
+                                  version_old, package))
         return False
 
-    # b) Local pmaports has a newer version
+    # b) Aports folder has a newer version
-    if version_pmaports != version_binary:
+    if version_new != version_old:
-        logging.debug(f"{msg}binary package out of date (binary: "
+        logging.debug(f"{msg}Binary package out of date (binary: "
-                      f"{version_binary}, local pmaports: {version_pmaports})")
+                      f"{version_old}, aport: {version_new})")
         return True
 
-    # Local pmaports and binary repo have the same version
+    # Aports and binary repo have the same version.
     return False
 
 
 def index_repo(args, arch=None):
-    """Recreate the APKINDEX.tar.gz for a specific repo, and clear the parsing
+    """
+    Recreate the APKINDEX.tar.gz for a specific repo, and clear the parsing
     cache for that file for the current pmbootstrap session (to prevent
     rebuilding packages twice, in case the rebuild takes less than a second).
 
@@ -121,11 +117,12 @@ def index_repo(args, arch=None):
                 pmb.chroot.user(args, command, working_dir=path_repo_chroot)
         else:
             logging.debug("NOTE: Can't build index for: " + path)
-        pmb.parse.apkindex.clear_cache(f"{path}/APKINDEX.tar.gz")
+        pmb.parse.apkindex.clear_cache(args, path + "/APKINDEX.tar.gz")
 
 
 def configure_abuild(args, suffix, verify=False):
-    """Set the correct JOBS count in ``abuild.conf``.
+    """
+    Set the correct JOBS count in abuild.conf
 
     :param verify: internally used to test if changing the config has worked.
     """
@@ -146,11 +143,12 @@ def configure_abuild(args, suffix, verify=False):
                                 suffix)
                 configure_abuild(args, suffix, True)
             return
-    pmb.chroot.root(args, ["sed", "-i", f"$ a\\{prefix}{args.jobs}", "/etc/abuild.conf"], suffix)
+    raise RuntimeError("Could not find " + prefix + " line in " + path)
 
 
 def configure_ccache(args, suffix="native", verify=False):
-    """Set the maximum ccache size.
+    """
+    Set the maximum ccache size
 
     :param verify: internally used to test if changing the config has worked.
     """

pmb/chroot/__init__.py

@@ -1,7 +1,7 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-from pmb.chroot.init import init, init_keys, UsrMerge
+from pmb.chroot.init import init
-from pmb.chroot.mount import mount, mount_native_into_foreign, remove_mnt_pmbootstrap
+from pmb.chroot.mount import mount, mount_native_into_foreign
 from pmb.chroot.root import root
 from pmb.chroot.user import user
 from pmb.chroot.user import exists as user_exists

pmb/chroot/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -14,20 +14,18 @@ import pmb.parse.depends
 import pmb.parse.version
 
 
-def update_repository_list(args, suffix="native", postmarketos_mirror=True,
+def update_repository_list(args, suffix="native", check=False):
-                           check=False):
     """
     Update /etc/apk/repositories, if it is outdated (when the user changed the
     --mirror-alpine or --mirror-pmOS parameters).
 
-    :param postmarketos_mirror: add postmarketos mirror URLs
     :param check: This function calls it self after updating the
                   /etc/apk/repositories file, to check if it was successful.
                   Only for this purpose, the "check" parameter should be set to
                   True.
     """
     # Skip if we already did this
-    if suffix in pmb.helpers.other.cache["apk_repository_list_updated"]:
+    if suffix in args.cache["apk_repository_list_updated"]:
         return
 
     # Read old entries or create folder structure
@@ -43,9 +41,9 @@ def update_repository_list(args, suffix="native", postmarketos_mirror=True,
         pmb.helpers.run.root(args, ["mkdir", "-p", os.path.dirname(path)])
 
     # Up to date: Save cache, return
-    lines_new = pmb.helpers.repo.urls(args, postmarketos_mirror=postmarketos_mirror)
+    lines_new = pmb.helpers.repo.urls(args)
     if lines_old == lines_new:
-        pmb.helpers.other.cache["apk_repository_list_updated"].append(suffix)
+        args.cache["apk_repository_list_updated"].append(suffix)
         return
 
     # Check phase: raise error when still outdated
@@ -59,7 +57,7 @@ def update_repository_list(args, suffix="native", postmarketos_mirror=True,
     for line in lines_new:
         pmb.helpers.run.root(args, ["sh", "-c", "echo "
                                     f"{shlex.quote(line)} >> {path}"])
-    update_repository_list(args, suffix, postmarketos_mirror, True)
+    update_repository_list(args, suffix, True)
 
 
 def check_min_version(args, suffix="native"):
@@ -69,7 +67,7 @@ def check_min_version(args, suffix="native"):
     """
 
     # Skip if we already did this
-    if suffix in pmb.helpers.other.cache["apk_min_version_checked"]:
+    if suffix in args.cache["apk_min_version_checked"]:
         return
 
     # Skip if apk is not installed yet
@@ -86,118 +84,154 @@ def check_min_version(args, suffix="native"):
         " 'pmbootstrap zap -hc'")
 
     # Mark this suffix as checked
-    pmb.helpers.other.cache["apk_min_version_checked"].append(suffix)
+    args.cache["apk_min_version_checked"].append(suffix)
 
 
-def install_build(args, package, arch):
+def install_is_necessary(args, build, arch, package, packages_installed):
     """
-    Build an outdated package unless pmbootstrap was invoked with
+    This function optionally builds an out of date package, and checks if the
-    "pmbootstrap install" and the option to build packages during pmb install
+    version installed inside a chroot is up to date.
-    is disabled.
+    :param build: Set to true to build the package, if the binary packages are
-
+                  out of date, and it is in the aports folder.
-    :param package: name of the package to build
+    :param packages_installed: Return value from installed().
-    :param arch: architecture of the package to build
+    :returns: True if the package needs to be installed/updated,
+              False otherwise.
     """
-    # User may have disabled building packages during "pmbootstrap install"
+    # User may have disabled buiding packages during "pmbootstrap install"
+    build_disabled = False
     if args.action == "install" and not args.build_pkgs_on_install:
-        if not pmb.parse.apkindex.package(args, package, arch, False):
+        build_disabled = True
+
+    # Build package
+    if build and not build_disabled:
+        pmb.build.package(args, package, arch)
+
+    # No further checks when not installed
+    if package not in packages_installed:
+        return True
+
+    # Make sure, that we really have a binary package
+    data_repo = pmb.parse.apkindex.package(args, package, arch, False)
+    if not data_repo:
+        if build_disabled:
             raise RuntimeError(f"{package}: no binary package found for"
                                f" {arch}, and compiling packages during"
                                " 'pmbootstrap install' has been disabled."
                                " Consider changing this option in"
                                " 'pmbootstrap init'.")
-        # Use the existing binary package
+        logging.warning("WARNING: Internal error in pmbootstrap,"
-        return
+                        f" package '{package}' for {arch}"
+                        " has not been built yet, but it should have"
+                        " been. Rebuilding it with force. Please "
+                        " report this, if there is no ticket about this"
+                        " yet!")
+        pmb.build.package(args, package, arch, True)
+        return install_is_necessary(args, build, arch, package,
+                                    packages_installed)
 
-    # Build the package if it's in pmaports and there is no binary package
+    # Compare the installed version vs. the version in the repos
-    # with the same pkgver and pkgrel. This check is done in
+    data_installed = packages_installed[package]
-    # pmb.build.is_necessary, which gets called in pmb.build.package.
+    compare = pmb.parse.version.compare(data_installed["version"],
-    return pmb.build.package(args, package, arch)
+                                        data_repo["version"])
+    # a) Installed newer (should not happen normally)
+    if compare == 1:
+        logging.info(f"WARNING: {arch} package '{package}'"
+                     f" installed version {data_installed['version']}"
+                     " is newer, than the version in the repositories:"
+                     f" {data_repo['version']}"
+                     " See also: <https://postmarketos.org/warning-repo>")
+        return False
+
+    # b) Repo newer
+    elif compare == -1:
+        return True
+
+    # c) Same version, look at last modified
+    elif compare == 0:
+        time_installed = float(data_installed["timestamp"])
+        time_repo = float(data_repo["timestamp"])
+        return time_repo > time_installed
 
 
-def packages_split_to_add_del(packages):
+def replace_aports_packages_with_path(args, packages, suffix, arch):
     """
-    Sort packages into "to_add" and "to_del" lists depending on their pkgname
+    apk will only re-install packages with the same pkgname,
-    starting with an exclamation mark.
+    pkgver and pkgrel, when you give it the absolute path to the package.
-
+    This function replaces all packages, that were built locally,
-    :param packages: list of pkgnames
+    with the absolute path to the package.
-    :returns: (to_add, to_del) - tuple of lists of pkgnames, e.g.
-              (["hello-world", ...], ["some-conflict-pkg", ...])
     """
-    to_add = []
-    to_del = []
-
-    for package in packages:
-        if package.startswith("!"):
-            to_del.append(package.lstrip("!"))
-        else:
-            to_add.append(package)
-
-    return (to_add, to_del)
-
-
-def packages_get_locally_built_apks(args, packages, arch):
-    """
-    Iterate over packages and if existing, get paths to locally built packages.
-    This is used to force apk to upgrade packages to newer local versions, even
-    if the pkgver and pkgrel did not change.
-
-    :param packages: list of pkgnames
-    :param arch: architecture that the locally built packages should have
-    :returns: list of apk file paths that are valid inside the chroots, e.g.
-              ["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...]
-    """
-    channel = pmb.config.pmaports.read_config(args)["channel"]
     ret = []
-
     for package in packages:
-        data_repo = pmb.parse.apkindex.package(args, package, arch, False)
+        aport = pmb.helpers.pmaports.find(args, package, False)
-        if not data_repo:
+        if aport:
-            continue
+            data_repo = pmb.parse.apkindex.package(args, package, arch, False)
-
+            if not data_repo:
-        apk_file = f"{package}-{data_repo['version']}.apk"
+                raise RuntimeError(f"{package}: could not find binary"
-        if not os.path.exists(f"{args.work}/packages/{channel}/{arch}/{apk_file}"):
+                                   " package, although it should exist for"
-            continue
+                                   " sure at this point in the code."
-
+                                   " Probably an APKBUILD subpackage parsing"
-        ret.append(f"/mnt/pmbootstrap/packages/{arch}/{apk_file}")
+                                   " bug. Related: https://gitlab.com/"
-
+                                   "postmarketOS/build.postmarketos.org/"
+                                   "issues/61")
+            apk_path = (f"/mnt/pmbootstrap-packages/{arch}/"
+                        f"{package}-{data_repo['version']}.apk")
+            if os.path.exists(f"{args.work}/chroot_{suffix}{apk_path}"):
+                package = apk_path
+        ret.append(package)
     return ret
 
 
-def install_run_apk(args, to_add, to_add_local, to_del, suffix):
+def install(args, packages, suffix="native", build=True):
     """
-    Run apk to add packages, and ensure only the desired packages get
+    :param build: automatically build the package, when it does not exist yet
-    explicitly marked as installed.
+                  or needs to be updated, and it is inside the pm-aports
+                  folder. Checking this is expensive - if you know, that all
+                  packages are provides by upstream repos, set this to False!
+    """
+    # Initialize chroot
+    check_min_version(args, suffix)
+    pmb.chroot.init(args, suffix)
+
+    # Add depends to packages
+    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
+    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
+
+    # Filter outdated packages (build them if required)
+    packages_installed = installed(args, suffix)
+    packages_todo = []
+    for package in packages_with_depends:
+        if install_is_necessary(
+                args, build, arch, package, packages_installed):
+            packages_todo.append(package)
+    if not len(packages_todo):
+        return
 
-    :param to_add: list of pkgnames to install, without their dependencies
-    :param to_add_local: return of packages_get_locally_built_apks()
-    :param to_del: list of pkgnames to be deleted, this should be set to
-                   conflicting dependencies in any of the packages to be
-                   installed or their dependencies (e.g. ["unl0kr"])
-    :param suffix: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64"
-    """
     # Sanitize packages: don't allow '--allow-untrusted' and other options
     # to be passed to apk!
-    for package in to_add + to_add_local + to_del:
+    for package in packages_todo:
         if package.startswith("-"):
             raise ValueError(f"Invalid package name: {package}")
 
-    commands = [["add"] + to_add]
+    # Readable install message without dependencies
+    message = f"({suffix}) install"
+    for pkgname in packages:
+        if pkgname not in packages_installed:
+            message += f" {pkgname}"
+    logging.info(message)
+
+    # Local packages: Using the path instead of pkgname makes apk update
+    # packages of the same version if the build date is different
+    packages_todo = replace_aports_packages_with_path(args, packages_todo,
+                                                      suffix, arch)
 
     # Use a virtual package to mark only the explicitly requested packages as
-    # explicitly installed, not the ones in to_add_local
+    # explicitly installed, not their dependencies or specific paths (#1212)
-    if to_add_local:
+    commands = [["add"] + packages]
-        commands += [["add", "-u", "--virtual", ".pmbootstrap"] + to_add_local,
+    if packages != packages_todo:
-                     ["del", ".pmbootstrap"]]
+        commands = [["add", "-u", "--virtual", ".pmbootstrap"] + packages_todo,
-
+                    ["add"] + packages,
-    if to_del:
+                    ["del", ".pmbootstrap"]]
-        commands += [["del"] + to_del]
-
     for (i, command) in enumerate(commands):
-        # --no-interactive is a parameter to `add`, so it must be appended or apk
-        # gets confused
-        command += ["--no-interactive"]
-
         if args.offline:
             command = ["--no-network"] + command
         if i == 0:
@@ -211,44 +245,6 @@ def install_run_apk(args, to_add, to_add_local, to_del, suffix):
                             suffix=suffix)
 
 
-def install(args, packages, suffix="native", build=True):
-    """
-    Install packages from pmbootstrap's local package index or the pmOS/Alpine
-    binary package mirrors. Iterate over all dependencies recursively, and
-    build missing packages as necessary.
-
-    :param packages: list of pkgnames to be installed
-    :param suffix: the chroot suffix, e.g. "native" or "rootfs_qemu-amd64"
-    :param build: automatically build the package, when it does not exist yet
-                  or needs to be updated, and it is inside pmaports. For the
-                  special case that all packages are expected to be in Alpine's
-                  repositories, set this to False for performance optimization.
-    """
-    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
-
-    if not packages:
-        logging.verbose("pmb.chroot.apk.install called with empty packages list,"
-                        " ignoring")
-        return
-
-    # Initialize chroot
-    check_min_version(args, suffix)
-    pmb.chroot.init(args, suffix)
-
-    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
-    to_add, to_del = packages_split_to_add_del(packages_with_depends)
-
-    if build:
-        for package in to_add:
-            install_build(args, package, arch)
-
-    to_add_local = packages_get_locally_built_apks(args, to_add, arch)
-    to_add_no_deps, _ = packages_split_to_add_del(packages)
-
-    logging.info(f"({suffix}) install {' '.join(to_add_no_deps)}")
-    install_run_apk(args, to_add_no_deps, to_add_local, to_del, suffix)
-
-
 def installed(args, suffix="native"):
     """
     Read the list of installed packages (which has almost the same format, as
@@ -256,15 +252,13 @@ def installed(args, suffix="native"):
 
     :returns: a dictionary with the following structure:
               { "postmarketos-mkinitfs":
-              {
+                {
-              "pkgname": "postmarketos-mkinitfs"
+                  "pkgname": "postmarketos-mkinitfs"
-              "version": "0.0.4-r10",
+                  "version": "0.0.4-r10",
-              "depends": ["busybox-extras", "lddtree", ...],
+                  "depends": ["busybox-extras", "lddtree", ...],
-              "provides": ["mkinitfs=0.0.1"]
+                  "provides": ["mkinitfs=0.0.1"]
-              }, ...
+                }, ...
-
               }
-
     """
     path = f"{args.work}/chroot_{suffix}/lib/apk/db/installed"
-    return pmb.parse.apkindex.parse(path, False)
+    return pmb.parse.apkindex.parse(args, path, False)

pmb/chroot/apk_static.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -18,7 +18,7 @@ import pmb.parse.version
 
 def read_signature_info(tar):
     """
-    Find various information about the signature that was used to sign
+    Find various information about the signature, that was used to sign
     /sbin/apk.static inside the archive (not to be confused with the normal apk
     archive signature!)
 
@@ -33,7 +33,7 @@ def read_signature_info(tar):
             break
     if not sigfilename:
         raise RuntimeError("Could not find signature filename in apk."
-                           " This means that your apk file is damaged."
+                           " This means, that your apk file is damaged."
                            " Delete it and try again."
                            " If the problem persists, fill out a bug report.")
     sigkey = sigfilename[len(prefix):]
@@ -114,7 +114,7 @@ def extract(args, version, apk_path):
     os.unlink(files["sig"]["temp_path"])
     temp_path = files["apk"]["temp_path"]
 
-    # Verify the version that the extracted binary reports
+    # Verify the version, that the extracted binary reports
     logging.debug("Verify the version reported by the apk.static binary"
                   f" (must match the package version {version})")
     os.chmod(temp_path, os.stat(temp_path).st_mode | stat.S_IEXEC)
@@ -141,7 +141,7 @@ def download(args, file):
     """
     channel_cfg = pmb.config.pmaports.read_config_channel(args)
     mirrordir = channel_cfg["mirrordir_alpine"]
-    base_url = f"{args.mirror_alpine}{mirrordir}/main/{pmb.config.arch_native}"
+    base_url = f"{args.mirror_alpine}{mirrordir}/main/{args.arch_native}"
     return pmb.helpers.http.download(args, f"{base_url}/{file}", file)
 
 
@@ -166,10 +166,6 @@ def init(args):
 
 
 def run(args, parameters):
-    # --no-interactive is a parameter to `add`, so it must be appended or apk
-    # gets confused
-    parameters += ["--no-interactive"]
-
     if args.offline:
         parameters = ["--no-network"] + parameters
     pmb.helpers.apk.apk_with_progress(

pmb/chroot/binfmt.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
@@ -33,7 +33,7 @@ def register(args, arch):
     if is_registered(arch_qemu):
         return
 
-    info = pmb.parse.binfmt_info(arch_qemu)
+    info = pmb.parse.binfmt_info(args, arch_qemu)
 
     # Build registration string
     # https://en.wikipedia.org/wiki/Binfmt_misc

pmb/chroot/distccd.py

@@ -0,0 +1,250 @@
+# Copyright 2021 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+import errno
+import json
+import logging
+import os
+import pmb.chroot
+import pmb.config
+import pmb.chroot.apk
+
+""" Packages for foreign architectures (e.g. armhf) get built in chroots
+    running with QEMU. While this works, it is painfully slow. So we speed it
+    up by using distcc to let cross compilers running in the native chroots do
+    the heavy lifting.
+
+    This file sets up an SSH server in the native chroot, which will then be
+    used by the foreign arch chroot to communicate with the distcc daemon. We
+    make sure that only the foreign arch chroot can connect to the sshd by only
+    listening on localhost, as well as generating dedicated ssh keys.
+
+    Using the SSH server instead of running distccd directly is a security
+    measure. Distccd does not authenticate its clients and would therefore
+    allow any process of the host system (not related to pmbootstrap) to
+    execute compilers in the native chroot. By modifying the compiler's options
+    or sending malicious data to the compiler, it is likely that the process
+    can gain remote code execution [1]. That way, a compromised, but sandboxed
+    process could gain privilege escalation.
+
+    [1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
+"""
+
+
+def init_server(args):
+    """
+    Install dependencies and generate keys for the server.
+    """
+    # Install dependencies
+    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
+                                  "openssh-server"])
+
+    # Config folder (nothing to do if existing)
+    dir = "/home/pmos/.distcc-sshd"
+    dir_outside = args.work + "/chroot_native" + dir
+    if os.path.exists(dir_outside):
+        return
+
+    # Generate keys
+    logging.info("(native) generate distcc-sshd server keys")
+    pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
+    pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
+
+
+def init_client(args, suffix):
+    """
+    Install dependencies and generate keys for the client.
+    """
+    # Install dependencies
+    pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
+                                  "openssh-client"], suffix)
+
+    # Public key path (nothing to do if existing)
+    pub = "/home/pmos/id_ed25519.pub"
+    pub_outside = args.work + "/chroot_" + suffix + pub
+    if os.path.exists(pub_outside):
+        return
+
+    # Generate keys
+    logging.info("(" + suffix + ") generate distcc-sshd client keys")
+    pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
+                           "-f", "/home/pmos/.ssh/id_ed25519"], suffix)
+    pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
+                    suffix)
+
+
+def configure_authorized_keys(args, suffix):
+    """
+    Exclusively allow one foreign arch chroot to access the sshd.
+    """
+    auth = "/home/pmos/.distcc-sshd/authorized_keys"
+    auth_outside = args.work + "/chroot_native/" + auth
+    pub = "/home/pmos/id_ed25519.pub"
+    pub_outside = args.work + "/chroot_" + suffix + pub
+    pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
+
+
+def configure_cmdlist(args, arch):
+    """
+    Create a whitelist of all the cross compiler wrappers.
+
+    Distcc 3.3 and above requires such a whitelist, or else it will only run
+    with the --make-me-a-botnet parameter (even in ssh mode).
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
+        for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
+            cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
+            handle.write(cmd_full + "\n")
+    pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
+    pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
+
+
+def configure_distccd_wrapper(args):
+    """
+    Wrap distccd in a shell script, so we can pass the compiler whitelist and
+    set the verbose flag (when pmbootstrap is running with --verbose).
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
+        handle.write("#!/bin/sh\n"
+                     "export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
+                     "distccd --log-file /home/pmos/distccd.log --nice 19")
+        if args.verbose:
+            handle.write(" --verbose")
+        handle.write(" \"$@\"\n")
+    pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
+    pmb.chroot.user(args, ["cat", dir + "/distccd"])
+    pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
+
+
+def configure_sshd(args):
+    """
+    Configure the SSH daemon in the native chroot.
+    """
+    dir = "/home/pmos/.distcc-sshd"
+    config = """AllowAgentForwarding no
+                AllowTcpForwarding no
+                AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
+                HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
+                ListenAddress 127.0.0.1
+                PasswordAuthentication no
+                PidFile /home/pmos/.distcc-sshd/sshd.pid
+                Port """ + args.port_distccd + """
+                X11Forwarding no"""
+
+    with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
+        for line in config.split("\n"):
+            handle.write(line.lstrip() + "\n")
+    pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
+    pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
+
+
+def get_running_pid(args):
+    """
+    :returns: the running distcc-sshd's pid as integer or None
+    """
+    # PID file must exist
+    pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
+    pidfile_outside = args.work + "/chroot_native" + pidfile
+    if not os.path.exists(pidfile_outside):
+        return None
+
+    # Verify, if it still exists by sending a kill signal
+    with open(pidfile_outside, "r") as handle:
+        pid = int(handle.read()[:-1])
+    try:
+        os.kill(pid, 0)
+    except OSError as err:
+        if err.errno == errno.ESRCH:  # no such process
+            pmb.helpers.run.root(args, ["rm", pidfile_outside])
+            return None
+    return pid
+
+
+def get_running_parameters(args):
+    """
+    Get the parameters of the currently running distcc-sshd instance.
+
+    :returns: a dictionary in the form of
+              {"arch": "armhf", "port": 1234, "verbose": False}
+              If the information can not be read, "arch" is set to "unknown"
+    """
+    # Return defaults
+    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
+    if not os.path.exists(path):
+        return {"arch": "unknown", "port": 0, "verbose": False}
+
+    # Parse the file as JSON
+    with open(path, "r") as handle:
+        return json.loads(handle.read())
+
+
+def set_running_parameters(args, arch):
+    """
+    Set the parameters of the currently running distcc-sshd instance.
+    """
+    parameters = {"arch": arch,
+                  "port": args.port_distccd,
+                  "verbose": args.verbose}
+
+    path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
+    with open(path, "w") as handle:
+        json.dump(parameters, handle)
+
+
+def is_running_with_same_parameters(args, arch):
+    """
+    Check whether we can use the already running distcc-sshd instance with our
+    current set of parameters. In case we can use it directly, we save some
+    time, otherwise we need to stop it, configure it again, and start it once
+    more.
+    """
+    if not get_running_pid(args):
+        return False
+
+    parameters = get_running_parameters(args)
+    return (parameters["arch"] == arch and
+            parameters["port"] == args.port_distccd and
+            parameters["verbose"] == args.verbose)
+
+
+def stop(args):
+    """
+    Kill the sshd process (by using its pid).
+    """
+    pid = get_running_pid(args)
+    if not pid:
+        return
+
+    parameters = get_running_parameters(args)
+    logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
+    pmb.chroot.user(args, ["kill", str(pid)])
+
+
+def start(args, arch):
+    """
+    Set up a new distcc-sshd instance or use an already running one.
+    """
+    if is_running_with_same_parameters(args, arch):
+        return
+    stop(args)
+
+    # Initialize server and client
+    suffix = "buildroot_" + arch
+    init_server(args)
+    init_client(args, suffix)
+
+    logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
+                 args.port_distccd)
+
+    # Configure server parameters (arch, port, verbose)
+    configure_authorized_keys(args, suffix)
+    configure_distccd_wrapper(args)
+    configure_cmdlist(args, arch)
+    configure_sshd(args)
+
+    # Run
+    dir = "/home/pmos/.distcc-sshd"
+    pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
+                           "-E", dir + "/log.txt"])
+    set_running_parameters(args, arch)

pmb/chroot/init.py

@@ -1,10 +1,9 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import enum
-import filecmp
-import glob
 import logging
 import os
+import glob
+import filecmp
 
 import pmb.chroot
 import pmb.chroot.apk_static
@@ -14,17 +13,6 @@ import pmb.helpers.repo
 import pmb.helpers.run
 import pmb.parse.arch
 
-cache_chroot_is_outdated = []
-
-class UsrMerge(enum.Enum):
-    """
-    Merge /usr while initializing chroot.
-    https://systemd.io/THE_CASE_FOR_THE_USR_MERGE/
-    """
-    AUTO = 0
-    ON = 1
-    OFF = 2
-
 
 def copy_resolv_conf(args, suffix="native"):
     """
@@ -55,7 +43,7 @@ def mark_in_chroot(args, suffix="native"):
 
 def setup_qemu_emulation(args, suffix):
     arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
-    if not pmb.parse.arch.cpu_emulation_required(arch):
+    if not pmb.parse.arch.cpu_emulation_required(args, arch):
         return
 
     chroot = f"{args.work}/chroot_{suffix}"
@@ -69,58 +57,7 @@ def setup_qemu_emulation(args, suffix):
                                 create_folders=True)
 
 
-def init_keys(args):
+def init(args, suffix="native"):
-    """
-    All Alpine and postmarketOS repository keys are shipped with pmbootstrap.
-    Copy them into $WORK/config_apk_keys, which gets mounted inside the various
-    chroots as /etc/apk/keys.
-
-    This is done before installing any package, so apk can verify APKINDEX
-    files of binary repositories even though alpine-keys/postmarketos-keys are
-    not installed yet.
-    """
-    for key in glob.glob(f"{pmb.config.apk_keys_path}/*.pub"):
-        target = f"{args.work}/config_apk_keys/{os.path.basename(key)}"
-        if not os.path.exists(target):
-            # Copy as root, so the resulting files in chroots are owned by root
-            pmb.helpers.run.root(args, ["cp", key, target])
-
-
-def init_usr_merge(args, suffix):
-    logging.info(f"({suffix}) merge /usr")
-    script = f"{pmb.config.pmb_src}/pmb/data/merge-usr.sh"
-    pmb.helpers.run.root(args, ["sh", "-e", script, "CALLED_FROM_PMB",
-                                f"{args.work}/chroot_{suffix}"])
-
-
-def warn_if_chroot_is_outdated(args, suffix):
-    global cache_chroot_is_outdated
-
-    # Only check / display the warning once per session
-    if suffix in cache_chroot_is_outdated:
-        return
-
-    if pmb.config.workdir.chroots_outdated(args, suffix):
-        days_warn = int(pmb.config.chroot_outdated / 3600 / 24)
-        logging.warning(f"WARNING: Your {suffix} chroot is older than"
-                        f" {days_warn} days. Consider running"
-                        " 'pmbootstrap zap'.")
-
-    cache_chroot_is_outdated += [suffix]
-
-
-def init(args, suffix="native", usr_merge=UsrMerge.AUTO,
-         postmarketos_mirror=True):
-    """
-    Initialize a chroot by copying the resolv.conf and updating
-    /etc/apk/repositories. If /bin/sh is missing, create the chroot from
-    scratch.
-
-    :param usr_merge: set to ON to force having a merged /usr. With AUTO it is
-                      only done if the user chose to install systemd in
-                      pmbootstrap init.
-    :param postmarketos_mirror: add postmarketos mirror URLs
-    """
     # When already initialized: just prepare the chroot
     chroot = f"{args.work}/chroot_{suffix}"
     arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
@@ -131,8 +68,7 @@ def init(args, suffix="native", usr_merge=UsrMerge.AUTO,
     if os.path.islink(f"{chroot}/bin/sh"):
         pmb.config.workdir.chroot_check_channel(args, suffix)
         copy_resolv_conf(args, suffix)
-        pmb.chroot.apk.update_repository_list(args, suffix, postmarketos_mirror)
+        pmb.chroot.apk.update_repository_list(args, suffix)
-        warn_if_chroot_is_outdated(args, suffix)
         return
 
     # Require apk-tools-static
@@ -146,9 +82,11 @@ def init(args, suffix="native", usr_merge=UsrMerge.AUTO,
                                 f"{chroot}/etc/apk/cache"])
 
     # Initialize /etc/apk/keys/, resolv.conf, repositories
-    init_keys(args)
+    for key in glob.glob(f"{pmb.config.apk_keys_path}/*.pub"):
+        pmb.helpers.run.root(args, ["cp", key, f"{args.work}"
+                                               "/config_apk_keys/"])
     copy_resolv_conf(args, suffix)
-    pmb.chroot.apk.update_repository_list(args, suffix, postmarketos_mirror)
+    pmb.chroot.apk.update_repository_list(args, suffix)
 
     pmb.config.workdir.chroot_save_init(args, suffix)
 
@@ -157,7 +95,7 @@ def init(args, suffix="native", usr_merge=UsrMerge.AUTO,
     pmb.chroot.apk_static.run(args, ["--root", chroot,
                                      "--cache-dir", apk_cache,
                                      "--initdb", "--arch", arch,
-                                     "add", "alpine-base"])
+                                     "add", "alpine-base", "postmarketos-keys"])
 
     # Building chroots: create "pmos" user, add symlinks to /home/pmos
     if not suffix.startswith("rootfs_"):
@@ -174,13 +112,3 @@ def init(args, suffix="native", usr_merge=UsrMerge.AUTO,
                 pmb.chroot.root(args, ["mkdir", "-p", target], suffix)
             pmb.chroot.user(args, ["ln", "-s", target, link_name], suffix)
             pmb.chroot.root(args, ["chown", "pmos:pmos", target], suffix)
-
-    # Merge /usr
-    if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(args):
-        usr_merge = UsrMerge.ON
-    if usr_merge is UsrMerge.ON:
-        init_usr_merge(args, suffix)
-
-    # Upgrade packages in the chroot, in case alpine-base, apk, etc. have been
-    # built from source with pmbootstrap
-    pmb.chroot.root(args, ["apk", "--no-network", "upgrade", "-a"], suffix)

pmb/chroot/initfs.py

@@ -1,11 +1,10 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging
 import pmb.chroot.initfs_hooks
 import pmb.chroot.other
 import pmb.chroot.apk
-import pmb.config.pmaports
 import pmb.helpers.cli
 
 
@@ -36,16 +35,9 @@ def extract(args, flavor, suffix, extra=False):
     """
     # Extraction folder
     inside = "/tmp/initfs-extracted"
-
-    pmaports_cfg = pmb.config.pmaports.read_config(args)
-    if pmaports_cfg.get("supported_mkinitfs_without_flavors", False):
-        initfs_file = "/boot/initramfs"
-    else:
-        initfs_file = f"/boot/initramfs-${flavor}"
     if extra:
         inside = "/tmp/initfs-extra-extracted"
-        initfs_file += "-extra"
+        flavor += "-extra"
-
     outside = f"{args.work}/chroot_{suffix}{inside}"
     if os.path.exists(outside):
         if not pmb.helpers.cli.confirm(args, f"Extraction folder {outside}"
@@ -63,7 +55,7 @@ def extract(args, flavor, suffix, extra=False):
 
     # Extract
     commands = [["mkdir", "-p", inside],
-                ["cp", initfs_file, f"{inside}/_initfs.gz"],
+                ["cp", f"/boot/initramfs-{flavor}", f"{inside}/_initfs.gz"],
                 ["gzip", "-d", f"{inside}/_initfs.gz"],
                 ["cat", "/tmp/_extract.sh"],  # for the log
                 ["sh", "/tmp/_extract.sh"],

pmb/chroot/initfs_hooks.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob
@@ -55,6 +55,6 @@ def delete(args, hook, suffix):
 
 def update(args, suffix):
     """
-    Rebuild and update all hooks that are out of date
+    Rebuild and update all hooks, that are out of date
     """
     pmb.chroot.apk.install(args, list_chroot(args, suffix, False), suffix)

pmb/chroot/mount.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -72,9 +72,6 @@ def mount_dev_tmpfs(args, suffix="native"):
                                 "tmpfs", dev + "/shm"])
     create_device_nodes(args, suffix)
 
-    # Setup /dev/fd as a symlink
-    pmb.helpers.run.root(args, ["ln", "-sf", "/proc/self/fd", f"{dev}/"])
-
 
 def mount(args, suffix="native"):
     # Mount tmpfs as the chroot's /dev
@@ -106,19 +103,3 @@ def mount_native_into_foreign(args, suffix):
     if not os.path.lexists(musl_link):
         pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,
                                     musl_link])
-        pmb.helpers.run.root(args, ["ln", "-sf", "/native/bin/busybox", "/usr/local/bin/gzip"])
-
-def remove_mnt_pmbootstrap(args, suffix):
-    """ Safely remove /mnt/pmbootstrap directories from the chroot, without
-        running rm -r as root and potentially removing data inside the
-        mountpoint in case it was still mounted (bug in pmbootstrap, or user
-        ran pmbootstrap 2x in parallel). This is similar to running 'rm -r -d',
-        but we don't assume that the host's rm has the -d flag (busybox does
-        not). """
-    mnt_dir = f"{args.work}/chroot_{suffix}/mnt/pmbootstrap"
-
-    if not os.path.exists(mnt_dir):
-        return
-
-    for path in glob.glob(f"{mnt_dir}/*") + [mnt_dir]:
-        pmb.helpers.run.root(args, ["rmdir", path])

pmb/chroot/other.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import glob
@@ -33,7 +33,7 @@ def kernel_flavor_installed(args, suffix, autoinstall=True):
 
 def tempfolder(args, path, suffix="native"):
     """
-    Create a temporary folder inside the chroot that belongs to "user".
+    Create a temporary folder inside the chroot, that belongs to "user".
     The folder gets deleted, if it already exists.
 
     :param path: of the temporary folder inside the chroot

pmb/chroot/root.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import shutil
@@ -19,7 +19,7 @@ def executables_absolute_path():
         path = shutil.which(binary, path=pmb.config.chroot_host_path)
         if not path:
             raise RuntimeError(f"Could not find the '{binary}'"
-                               " executable. Make sure that it is in"
+                               " executable. Make sure, that it is in"
                                " your current user's PATH.")
         ret[binary] = path
     return ret
@@ -27,17 +27,13 @@ def executables_absolute_path():
 
 def root(args, cmd, suffix="native", working_dir="/", output="log",
          output_return=False, check=None, env={}, auto_init=True,
-         disable_timeout=False, add_proxy_env_vars=True):
+         disable_timeout=False):
     """
     Run a command inside a chroot as root.
 
     :param env: dict of environment variables to be passed to the command, e.g.
                 {"JOBS": "5"}
     :param auto_init: automatically initialize the chroot
-    :param add_proxy_env_vars: if True, preserve HTTP_PROXY etc. vars from host
-                               environment. pmb.chroot.user sets this to False
-                               when calling pmb.chroot.root, because it already
-                               makes the variables part of the cmd argument.
 
     See pmb.helpers.run_core.core() for a detailed description of all other
     arguments and the return value.
@@ -59,17 +55,14 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
 
     # Merge env with defaults into env_all
     env_all = {"CHARSET": "UTF-8",
+               "LANG": "UTF-8",
                "HISTFILE": "~/.ash_history",
                "HOME": "/root",
-               "LANG": "UTF-8",
                "PATH": pmb.config.chroot_path,
-               "PYTHONUNBUFFERED": "1",
                "SHELL": "/bin/ash",
                "TERM": "xterm"}
     for key, value in env.items():
         env_all[key] = value
-    if add_proxy_env_vars:
-        pmb.helpers.run_core.add_proxy_env_vars(env_all)
 
     # Build the command in steps and run it, e.g.:
     # cmd: ["echo", "test"]
@@ -77,11 +70,9 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
     # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
     executables = executables_absolute_path()
     cmd_chroot = [executables["chroot"], chroot, "/bin/sh", "-c",
-                  pmb.helpers.run_core.flat_cmd(cmd, working_dir)]
+                  pmb.helpers.run.flat_cmd(cmd, working_dir)]
-    cmd_sudo = pmb.config.sudo([
+    cmd_sudo = ["sudo", "env", "-i", executables["sh"], "-c",
-        "env", "-i", executables["sh"], "-c",
+                pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)]
-        pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)]
-    )
     return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output,
                                      output_return, check, True,
                                      disable_timeout)

pmb/chroot/shutdown.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -7,6 +7,7 @@ import socket
 from contextlib import closing
 
 import pmb.chroot
+import pmb.chroot.distccd
 import pmb.helpers.mount
 import pmb.install.losetup
 import pmb.parse.arch
@@ -22,17 +23,6 @@ def kill_adb(args):
             pmb.chroot.root(args, ["adb", "-P", str(port), "kill-server"])
 
 
-def kill_sccache(args):
-    """
-    Kill sccache daemon if it's running. Unlike ccache it automatically spawns
-    a daemon when you call it and exits after some time of inactivity.
-    """
-    port = 4226
-    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
-        if sock.connect_ex(("127.0.0.1", port)) == 0:
-            pmb.chroot.root(args, ["sccache", "--stop-server"])
-
-
 def shutdown_cryptsetup_device(args, name):
     """
     :param name: cryptsetup device name, usually "pm_crypt" in pmbootstrap
@@ -59,9 +49,10 @@ def shutdown_cryptsetup_device(args, name):
 
 
 def shutdown(args, only_install_related=False):
-    # Stop daemons
+    pmb.chroot.distccd.stop(args)
+
+    # Stop adb server
     kill_adb(args)
-    kill_sccache(args)
 
     # Umount installation-related paths (order is important!)
     pmb.helpers.mount.umount_all(args, args.work +
@@ -99,6 +90,6 @@ def shutdown(args, only_install_related=False):
 
         # Clean up the rest
         for arch in pmb.config.build_device_architectures:
-            if pmb.parse.arch.cpu_emulation_required(arch):
+            if pmb.parse.arch.cpu_emulation_required(args, arch):
                 pmb.chroot.binfmt.unregister(args, arch)
         logging.debug("Shutdown complete")

pmb/chroot/user.py

@@ -1,8 +1,7 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.chroot.root
 import pmb.helpers.run
-import pmb.helpers.run_core
 
 
 def user(args, cmd, suffix="native", working_dir="/", output="log",
@@ -19,17 +18,13 @@ def user(args, cmd, suffix="native", working_dir="/", output="log",
     See pmb.helpers.run_core.core() for a detailed description of all other
     arguments and the return value.
     """
-    env = env.copy()
-    pmb.helpers.run_core.add_proxy_env_vars(env)
-
     if "HOME" not in env:
         env["HOME"] = "/home/pmos"
 
-    flat_cmd = pmb.helpers.run_core.flat_cmd(cmd, env=env)
+    flat_cmd = pmb.helpers.run.flat_cmd(cmd, env=env)
     cmd = ["busybox", "su", "pmos", "-c", flat_cmd]
     return pmb.chroot.root(args, cmd, suffix, working_dir, output,
-                           output_return, check, {}, auto_init,
+                           output_return, check, {}, auto_init)
-                           add_proxy_env_vars=False)
 
 
 def exists(args, username, suffix="native"):

pmb/chroot/zap.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import glob
 import logging
@@ -15,21 +15,20 @@ import pmb.parse.apkindex
 
 def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False,
-        rust=False, netboot=False):
+        rust=False):
     """
-    Shutdown everything inside the chroots (e.g. adb), umount
+    Shutdown everything inside the chroots (e.g. distccd, adb), umount
     everything and then safely remove folders from the work-directory.
 
     :param dry: Only show what would be deleted, do not delete for real
     :param pkgs_local: Remove *all* self-compiled packages (!)
     :param http: Clear the http cache (used e.g. for the initial apk download)
-    :param pkgs_local_mismatch: Remove the packages that have
+    :param pkgs_local_mismatch: Remove the packages, that have
         a different version compared to what is in the aports folder.
     :param pkgs_online_mismatch: Clean out outdated binary packages
         downloaded from mirrors (e.g. from Alpine)
     :param distfiles: Clear the downloaded files cache
     :param rust: Remove rust related caches
-    :param netboot: Remove images for netboot
 
     NOTE: This function gets called in pmb/config/init.py, with only args.work
     and args.device set!
@@ -66,8 +65,6 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
         patterns += ["cache_distfiles"]
     if rust:
         patterns += ["cache_rust"]
-    if netboot:
-        patterns += ["images_netboot"]
 
     # Delete everything matching the patterns
     for pattern in patterns:
@@ -84,7 +81,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
     pmb.config.workdir.clean(args)
 
     # Chroots were zapped, so no repo lists exist anymore
-    pmb.helpers.other.cache["apk_repository_list_updated"].clear()
+    args.cache["apk_repository_list_updated"].clear()
 
     # Print amount of cleaned up space
     if dry:
@@ -109,7 +106,7 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
     pattern = f"{args.work}/packages/{channel}/*/APKINDEX.tar.gz"
     for apkindex_path in glob.glob(pattern):
         # Delete packages without same version in aports
-        blocks = pmb.parse.apkindex.parse_blocks(apkindex_path)
+        blocks = pmb.parse.apkindex.parse_blocks(args, apkindex_path)
         for block in blocks:
             pkgname = block["pkgname"]
             origin = block["origin"]
@@ -135,7 +132,7 @@ def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
                 continue
 
             # Clear out any binary apks that do not match what is in aports
-            apkbuild = pmb.parse.apkbuild(f"{aport_path}/APKBUILD")
+            apkbuild = pmb.parse.apkbuild(args, f"{aport_path}/APKBUILD")
             version_aport = f"{apkbuild['pkgver']}-r{apkbuild['pkgrel']}"
             if version != version_aport:
                 logging.info(f"% rm {apk_path_short}"
@@ -161,9 +158,7 @@ def zap_pkgs_online_mismatch(args, confirm=True, dry=False):
     # Iterate over existing apk caches
     for path in paths:
         arch = os.path.basename(path).split("_", 2)[2]
-        suffix = f"buildroot_{arch}"
+        suffix = "native" if arch == args.arch_native else f"buildroot_{arch}"
-        if arch == pmb.config.arch_native:
-            suffix = "native"
 
         # Clean the cache with apk
         logging.info(f"({suffix}) apk -v cache clean")

pmb/ci/__init__.py

@@ -1,184 +0,0 @@
-# Copyright 2023 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-import collections
-import glob
-import logging
-import os
-import shlex
-import pmb.chroot
-import pmb.helpers.cli
-
-
-def get_ci_scripts(topdir):
-    """ Find 'pmbootstrap ci'-compatible scripts inside a git repository, and
-        parse their metadata (description, options). The reference is at:
-        https://postmarketos.org/pmb-ci
-
-        :param topdir: top directory of the git repository, get it with: pmb.helpers.git.get_topdir()
-
-        :returns: a dict of CI scripts found in the git repository, e.g. 
-	  {"ruff": {"description": "lint all python scripts", "options": []}, ...} 
-	"""
-    ret = {}
-    for script in glob.glob(f"{topdir}/.ci/*.sh"):
-        is_pmb_ci_script = False
-        description = ""
-        options = []
-
-        with open(script) as handle:
-            for line in handle:
-                if line.startswith("# https://postmarketos.org/pmb-ci"):
-                    is_pmb_ci_script = True
-                elif line.startswith("# Description: "):
-                    description = line.split(": ", 1)[1].rstrip()
-                elif line.startswith("# Options: "):
-                    options = line.split(": ", 1)[1].rstrip().split(" ")
-                elif not line.startswith("#"):
-                    # Stop parsing after the block of comments on top
-                    break
-
-        if not is_pmb_ci_script:
-            continue
-
-        if not description:
-            logging.error(f"ERROR: {script}: missing '# Description: …' line")
-            exit(1)
-
-        for option in options:
-            if option not in pmb.config.ci_valid_options:
-                raise RuntimeError(f"{script}: unsupported option '{option}'."
-                                   " Typo in script or pmbootstrap too old?")
-
-        short_name = os.path.basename(script).split(".", -1)[0]
-        ret[short_name] = {"description": description,
-                           "options": options}
-    return ret
-
-
-def sort_scripts_by_speed(scripts):
-    """ Order the scripts, so fast scripts run before slow scripts. Whether a
-        script is fast or not is determined by the '# Options: slow' comment in
-        the file.
-        
-	:param scripts: return of get_ci_scripts()
-        
-	:returns: same format as get_ci_scripts(), but as ordered dict with
-          fast scripts before slow scripts 
-	
-	"""
-    ret = collections.OrderedDict()
-
-    # Fast scripts first
-    for script_name, script in scripts.items():
-        if "slow" in script["options"]:
-            continue
-        ret[script_name] = script
-
-    # Then slow scripts
-    for script_name, script in scripts.items():
-        if "slow" not in script["options"]:
-            continue
-        ret[script_name] = script
-    return ret
-
-
-def ask_which_scripts_to_run(scripts_available):
-    """ Display an interactive prompt about which of the scripts the user
-        wishes to run, or all of them.
-        
-	:param scripts_available: same format as get_ci_scripts()
-        
-	:returns: either full scripts_available (all selected), or a subset 
-	
-	"""
-    count = len(scripts_available.items())
-    choices = ["all"]
-
-    logging.info(f"Available CI scripts ({count}):")
-    for script_name, script in scripts_available.items():
-        extra = ""
-        if "slow" in script["options"]:
-            extra += " (slow)"
-        logging.info(f"* {script_name}: {script['description']}{extra}")
-        choices += [script_name]
-
-    selection = pmb.helpers.cli.ask("Which script?", None, "all",
-                                    complete=choices)
-    if selection == "all":
-        return scripts_available
-
-    ret = {}
-    ret[selection] = scripts_available[selection]
-    return ret
-
-
-def copy_git_repo_to_chroot(args, topdir):
-    """ Create a tarball of the git repo (including unstaged changes and new
-        files) and extract it in chroot_native.
-        
-	:param topdir: top directory of the git repository, get it with:
-          pmb.helpers.git.get_topdir() 
-
-	"""
-    pmb.chroot.init(args)
-    tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz"
-    files = pmb.helpers.git.get_files(args, topdir)
-
-    with open(f"{tarball_path}.files", "w") as handle:
-        for file in files:
-            handle.write(file)
-            handle.write("\n")
-
-    pmb.helpers.run.user(args, ["tar", "-cf", tarball_path, "-T",
-                                f"{tarball_path}.files"], topdir)
-
-    ci_dir = "/home/pmos/ci"
-    pmb.chroot.user(args, ["rm", "-rf", ci_dir])
-    pmb.chroot.user(args, ["mkdir", ci_dir])
-    pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"],
-                    working_dir=ci_dir)
-
-
-def run_scripts(args, topdir, scripts):
-    """ Run one of the given scripts after another, either natively or in a
-        chroot. Display a progress message and stop on error (without printing
-        a python stack trace).
-        
-	:param topdir: top directory of the git repository, get it with:
-          pmb.helpers.git.get_topdir()
-        
-	:param scripts: return of get_ci_scripts() 
-
-	"""
-    steps = len(scripts)
-    step = 0
-    repo_copied = False
-
-    for script_name, script in scripts.items():
-        step += 1
-
-        where = "pmbootstrap chroot"
-        if "native" in script["options"]:
-            where = "native"
-
-        script_path = f".ci/{script_name}.sh"
-        logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}"
-                     f" [{where}] ***")
-
-        if "native" in script["options"]:
-            rc = pmb.helpers.run.user(args, [script_path], topdir,
-                                      output="tui")
-            continue
-        else:
-            # Run inside pmbootstrap chroot
-            if not repo_copied:
-                copy_git_repo_to_chroot(args, topdir)
-                repo_copied = True
-
-            env = {"TESTUSER": "pmos"}
-            rc = pmb.chroot.root(args, [script_path], check=False, env=env,
-                                 working_dir="/home/pmos/ci",
-                                 output="tui")
-        if rc:
-            logging.error(f"ERROR: CI script failed: {script_name}")
-            exit(1)

pmb/config/init.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import glob
@@ -11,11 +11,9 @@ import pmb.config
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.devices
-import pmb.helpers.git
 import pmb.helpers.http
 import pmb.helpers.logging
 import pmb.helpers.other
-import pmb.helpers.pmaports
 import pmb.helpers.run
 import pmb.helpers.ui
 import pmb.chroot.zap
@@ -34,29 +32,14 @@ def require_programs():
                            f" {', '.join(missing)}")
 
 
-def ask_for_username(args):
-    """Ask for a reasonable username for the non-root user.
-
-    :returns: the username
-    """
-    while True:
-        ret = pmb.helpers.cli.ask("Username", None, args.user, False,
-                                  "[a-z_][a-z0-9_-]*")
-        if ret == "root":
-            logging.fatal("ERROR: don't put \"root\" here. This is about"
-                          " creating an additional non-root user. Don't worry,"
-                          " the root user will also be created ;)")
-            continue
-        return ret
-
-
 def ask_for_work_path(args):
-    """Ask for the work path, until we can create it (when it does not exist) and write into it.
+    """
-
+    Ask for the work path, until we can create it (when it does not exist) and
+    write into it.
     :returns: (path, exists)
-        * path: is the full path, with expanded ~ sign
+              * path: is the full path, with expanded ~ sign
-        * exists: is False when the folder did not exist before we tested whether we can create it
+              * exists: is False when the folder did not exist before we tested
-
+                        whether we can create it
     """
     logging.info("Location of the 'work' path. Multiple chroots"
                  " (native, device arch, device rootfs) will be created"
@@ -64,7 +47,7 @@ def ask_for_work_path(args):
     while True:
         try:
             work = os.path.expanduser(pmb.helpers.cli.ask(
-                "Work path", None, args.work, False))
+                args, "Work path", None, args.work, False))
             work = os.path.realpath(work)
             exists = os.path.exists(work)
 
@@ -80,12 +63,10 @@ def ask_for_work_path(args):
             if not exists:
                 os.makedirs(work, 0o700, True)
 
-            # If the version file doesn't exists yet because we either just
+            if not os.listdir(work):
-            # created the work directory or the user has deleted it for
+                # Directory is empty, either because we just created it or
-            # whatever reason then we need to write initialize it.
+                # because user created it before running pmbootstrap init
-            work_version_file = f"{work}/version"
+                with open(f"{work}/version", "w") as handle:
-            if not os.path.isfile(work_version_file):
-                with open(work_version_file, "w") as handle:
                     handle.write(f"{pmb.config.work_version}\n")
 
             # Create cache_git dir, so it is owned by the host system's user
@@ -98,22 +79,17 @@ def ask_for_work_path(args):
 
 
 def ask_for_channel(args):
-    """Ask for the postmarketOS release channel.
+    """ Ask for the postmarketOS release channel. The channel dictates, which
-    The channel dictates, which pmaports branch pmbootstrap will check out,
+        pmaports branch pmbootstrap will check out, and which repository URLs
-    and which repository URLs will be used when initializing chroots.
+        will be used when initializing chroots.
-
+        :returns: channel name (e.g. "edge", "v21.03") """
-    :returns: channel name (e.g. "edge", "v21.03")
-    """
     channels_cfg = pmb.helpers.git.parse_channels_cfg(args)
     count = len(channels_cfg["channels"])
 
     # List channels
     logging.info("Choose the postmarketOS release channel.")
     logging.info(f"Available ({count}):")
-    # Only show the first 3 releases. This includes edge, the latest supported
+    for channel, channel_data in channels_cfg["channels"].items():
-    # release plus one. Should be a good solution until new needs arrive when
-    # we might want to have a custom channels.cfg attribute.
-    for channel, channel_data in list(channels_cfg["channels"].items())[:3]:
         logging.info(f"* {channel}: {channel_data['description']}")
 
     # Default for first run: "recommended" from channels.cfg
@@ -127,7 +103,7 @@ def ask_for_channel(args):
 
     # Ask until user gives valid channel
     while True:
-        ret = pmb.helpers.cli.ask("Channel", None, default,
+        ret = pmb.helpers.cli.ask(args, "Channel", None, default,
                                   complete=choices)
         if ret in choices:
             return ret
@@ -135,7 +111,8 @@ def ask_for_channel(args):
                       " from the list above.")
 
 
-def ask_for_ui(args, info):
+def ask_for_ui(args, device):
+    info = pmb.parse.deviceinfo(args, device)
     ui_list = pmb.helpers.ui.list(args, info["arch"])
     hidden_ui_count = 0
     device_is_accelerated = info.get("gpu_accelerated") == "true"
@@ -149,22 +126,19 @@ def ask_for_ui(args, info):
                 ui_list.pop(i)
                 hidden_ui_count += 1
 
-    # Get default
-    default = args.ui
-    if default not in dict(ui_list).keys():
-        default = pmb.config.defaults["ui"]
-
     logging.info(f"Available user interfaces ({len(ui_list) - 1}): ")
     ui_completion_list = []
     for ui in ui_list:
         logging.info(f"* {ui[0]}: {ui[1]}")
         ui_completion_list.append(ui[0])
     if hidden_ui_count > 0:
-        logging.info(f"NOTE: {hidden_ui_count} UIs are hidden because"
+        logging.info(f"NOTE: {hidden_ui_count} user interfaces are not"
-                     " \"deviceinfo_gpu_accelerated\" is not set (see"
+                     " available. If device supports GPU acceleration,"
-                     " https://postmarketos.org/deviceinfo).")
+                     " set \"deviceinfo_gpu_accelerated\" to make UIs"
+                     " available. See: <https://wiki.postmarketos.org/wiki/"
+                     "Deviceinfo_reference")
     while True:
-        ret = pmb.helpers.cli.ask("User interface", None, default, True,
+        ret = pmb.helpers.cli.ask(args, "User interface", None, args.ui, True,
                                   complete=ui_completion_list)
         if ret in dict(ui_list).keys():
             return ret
@@ -189,30 +163,8 @@ def ask_for_ui_extras(args, ui):
                                    default=args.ui_extras)
 
 
-def ask_for_systemd(args, ui):
+def ask_for_keymaps(args, device):
-    if "systemd" not in pmb.config.pmaports.read_config_repos(args):
+    info = pmb.parse.deviceinfo(args, device)
-        return args.systemd
-
-    if pmb.helpers.ui.check_option(args, ui, "pmb:systemd-never"):
-        logging.info("Based on your UI selection, OpenRC will be used as init"
-                     " system. This UI does not support systemd.")
-        return args.systemd
-
-    default_is_systemd = pmb.helpers.ui.check_option(args, ui, "pmb:systemd")
-    not_str = " " if default_is_systemd else " not "
-    logging.info("Based on your UI selection, 'default' will result"
-                 f" in{not_str}installing systemd.")
-
-    choices = pmb.config.allowed_values["systemd"]
-    answer = pmb.helpers.cli.ask("Install systemd?",
-                                 choices,
-                                 args.systemd,
-                                 validation_regex=f"^({'|'.join(choices)})$",
-                                 complete=choices)
-    return answer
-
-
-def ask_for_keymaps(args, info):
     if "keymaps" not in info or info["keymaps"].strip() == "":
         return ""
     options = info["keymaps"].split(' ')
@@ -222,7 +174,7 @@ def ask_for_keymaps(args, info):
         args.keymap = options[0]
 
     while True:
-        ret = pmb.helpers.cli.ask("Keymap", None, args.keymap,
+        ret = pmb.helpers.cli.ask(args, "Keymap", None, args.keymap,
                                   True, complete=options)
         if ret in options:
             return ret
@@ -256,86 +208,13 @@ def ask_for_timezone(args):
     return "GMT"
 
 
-def ask_for_provider_select(args, apkbuild, providers_cfg):
-    """Ask for selectable providers that are specified using "_pmb_select" in a APKBUILD.
-
-    :param apkbuild: the APKBUILD with the _pmb_select
-    :param providers_cfg: the configuration section with previously selected
-                          providers. Updated with new providers after selection
-    """
-    for select in apkbuild["_pmb_select"]:
-        providers = pmb.helpers.pmaports.find_providers(args, select)
-        logging.info(f"Available providers for {select} ({len(providers)}):")
-
-        has_default = False
-        providers_short = {}
-        last_selected = providers_cfg.get(select, 'default')
-
-        for pkgname, pkg in providers:
-            # Strip provider prefix if possible
-            short = pkgname
-            if short.startswith(f'{select}-'):
-                short = short[len(f"{select}-"):]
-
-            # Allow selecting the package using both short and long name
-            providers_short[pkgname] = pkgname
-            providers_short[short] = pkgname
-
-            if pkgname == last_selected:
-                last_selected = short
-
-            if not has_default and pkg.get('provider_priority', 0) != 0:
-                # Display as default provider
-                styles = pmb.config.styles
-                logging.info(f"* {short}: {pkg['pkgdesc']} "
-                             f"{styles['BOLD']}(default){styles['END']}")
-                has_default = True
-            else:
-                logging.info(f"* {short}: {pkg['pkgdesc']}")
-
-        while True:
-            ret = pmb.helpers.cli.ask("Provider", None, last_selected, True,
-                                      complete=providers_short.keys())
-
-            if has_default and ret == 'default':
-                # Selecting default means to not select any provider explicitly
-                # In other words, apk chooses it automatically based on
-                # "provider_priority"
-                if select in providers_cfg:
-                    del providers_cfg[select]
-                break
-            if ret in providers_short:
-                providers_cfg[select] = providers_short[ret]
-                break
-            logging.fatal("ERROR: Invalid provider specified, please type in"
-                          " one from the list above.")
-
-
-def ask_for_provider_select_pkg(args, pkgname, providers_cfg):
-    """Look up the APKBUILD for the specified pkgname and ask for selectable
-    providers that are specified using "_pmb_select".
-
-    :param pkgname: name of the package to search APKBUILD for
-    :param providers_cfg: the configuration section with previously selected
-                          providers. Updated with new providers after selection
-    """
-    apkbuild = pmb.helpers.pmaports.get(args, pkgname,
-                                        subpackages=False, must_exist=False)
-    if not apkbuild:
-        return
-
-    ask_for_provider_select(args, apkbuild, providers_cfg)
-
-
 def ask_for_device_kernel(args, device):
-    """Ask for the kernel that should be used with the device.
+    """
+    Ask for the kernel that should be used with the device.
 
     :param device: code name, e.g. "lg-mako"
-
     :returns: None if the kernel is hardcoded in depends without subpackages
-
     :returns: kernel type ("downstream", "stable", "mainline", ...)
-
     """
     # Get kernels
     kernels = pmb.parse._apkbuild.kernels(args, device)
@@ -362,7 +241,7 @@ def ask_for_device_kernel(args, device):
     for type in sorted(kernels.keys()):
         logging.info(f"* {type}: {kernels[type]}")
     while True:
-        ret = pmb.helpers.cli.ask("Kernel", None, default, True,
+        ret = pmb.helpers.cli.ask(args, "Kernel", None, default, True,
                                   complete=kernels)
         if ret in kernels.keys():
             return ret
@@ -371,15 +250,54 @@ def ask_for_device_kernel(args, device):
     return ret
 
 
-def ask_for_device(args):
+def ask_for_device_nonfree(args, device):
     """
-    Prompt for the device vendor, model, and kernel.
+    Ask the user about enabling proprietary firmware (e.g. Wifi) and userland
+    (e.g. GPU drivers). All proprietary components are in subpackages
+    $pkgname-nonfree-firmware and $pkgname-nonfree-userland, and we show the
+    description of these subpackages (so they can indicate which peripherals
+    are affected).
 
-    :returns: Tuple consisting of: (device, device_exists, kernel)
+    :returns: answers as dict, e.g. {"firmware": True, "userland": False}
-        * device: "<vendor>-<codename>" string for device
-        * device_exists: bool indicating if device port exists in repo
-        * kernel: type of kernel (downstream, etc)
     """
+    # Parse existing APKBUILD or return defaults (when called from test case)
+    apkbuild_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
+    ret = {"firmware": args.nonfree_firmware,
+           "userland": args.nonfree_userland}
+    if not apkbuild_path:
+        return ret
+    apkbuild = pmb.parse.apkbuild(args, apkbuild_path)
+
+    # Only run when there is a "nonfree" subpackage
+    nonfree_found = False
+    for subpackage in apkbuild["subpackages"].keys():
+        if subpackage.startswith(f"device-{device}-nonfree"):
+            nonfree_found = True
+    if not nonfree_found:
+        return ret
+
+    # Short explanation
+    logging.info("This device has proprietary components, which trade some of"
+                 " your freedom with making more peripherals work.")
+    logging.info("We would like to offer full functionality without hurting"
+                 " your freedom, but this is currently not possible for your"
+                 " device.")
+
+    # Ask for firmware and userland individually
+    for type in ["firmware", "userland"]:
+        subpkgname = f"device-{device}-nonfree-{type}"
+        subpkg = apkbuild["subpackages"].get(subpkgname, {})
+        if subpkg is None:
+            raise RuntimeError("Cannot find subpackage function for "
+                               f"{subpkgname}")
+        if subpkg:
+            logging.info(f"{subpkgname}: {subpkg['pkgdesc']}")
+            ret[type] = pmb.helpers.cli.confirm(args, "Enable this package?",
+                                                default=ret[type])
+    return ret
+
+
+def ask_for_device(args):
     vendors = sorted(pmb.helpers.devices.list_vendors(args))
     logging.info("Choose your target device vendor (either an "
                  "existing one, or a new one for porting).")
@@ -392,7 +310,7 @@ def ask_for_device(args):
         current_codename = args.device.split("-", 1)[1]
 
     while True:
-        vendor = pmb.helpers.cli.ask("Vendor", None, current_vendor,
+        vendor = pmb.helpers.cli.ask(args, "Vendor", None, current_vendor,
                                      False, r"[a-z0-9]+", vendors)
 
         new_vendor = vendor not in vendors
@@ -404,9 +322,9 @@ def ask_for_device(args):
             if not pmb.helpers.cli.confirm(args, default=True):
                 continue
         else:
-            # Archived devices can be selected, but are not displayed
+            # Unmaintained devices can be selected, but are not displayed
             devices = sorted(pmb.helpers.devices.list_codenames(
-                args, vendor, archived=False))
+                args, vendor, unmaintained=False))
             # Remove "vendor-" prefixes from device list
             codenames = [x.split('-', 1)[1] for x in devices]
             logging.info(f"Available codenames ({len(codenames)}): " +
@@ -414,7 +332,7 @@ def ask_for_device(args):
 
         if current_vendor != vendor:
             current_codename = ''
-        codename = pmb.helpers.cli.ask("Device codename", None,
+        codename = pmb.helpers.cli.ask(args, "Device codename", None,
                                        current_codename, False, r"[a-z0-9]+",
                                        codenames)
 
@@ -437,16 +355,17 @@ def ask_for_device(args):
             logging.info("Generating new aports for: {}...".format(device))
             pmb.aportgen.generate(args, f"device-{device}")
             pmb.aportgen.generate(args, f"linux-{device}")
-        elif "/archived/" in device_path:
+        elif "/unmaintained/" in device_path:
             apkbuild = f"{device_path[:-len('deviceinfo')]}APKBUILD"
-            archived = pmb.parse._apkbuild.archived(apkbuild)
+            unmaintained = pmb.parse._apkbuild.unmaintained(apkbuild)
-            logging.info(f"WARNING: {device} is archived: {archived}")
+            logging.info(f"WARNING: {device} is unmaintained: {unmaintained}")
             if not pmb.helpers.cli.confirm(args):
                 continue
         break
 
     kernel = ask_for_device_kernel(args, device)
-    return (device, device_exists, kernel)
+    nonfree = ask_for_device_nonfree(args, device)
+    return (device, device_exists, kernel, nonfree)
 
 
 def ask_for_additional_options(args, cfg):
@@ -471,20 +390,20 @@ def ask_for_additional_options(args, cfg):
                  " enough to fit the rootfs (pmbootstrap#1904)."
                  " How much extra free space do you want to add to the image"
                  " (in MB)?")
-    answer = pmb.helpers.cli.ask("Extra space size", None,
+    answer = pmb.helpers.cli.ask(args, "Extra space size", None,
                                  args.extra_space, validation_regex="^[0-9]+$")
     cfg["pmbootstrap"]["extra_space"] = answer
 
     # Boot size
     logging.info("What should be the boot partition size (in MB)?")
-    answer = pmb.helpers.cli.ask("Boot size", None, args.boot_size,
+    answer = pmb.helpers.cli.ask(args, "Boot size", None, args.boot_size,
                                  validation_regex="^[1-9][0-9]*$")
     cfg["pmbootstrap"]["boot_size"] = answer
 
     # Parallel job count
     logging.info("How many jobs should run parallel on this machine, when"
                  " compiling?")
-    answer = pmb.helpers.cli.ask("Jobs", None, args.jobs,
+    answer = pmb.helpers.cli.ask(args, "Jobs", None, args.jobs,
                                  validation_regex="^[1-9][0-9]*$")
     cfg["pmbootstrap"]["jobs"] = answer
 
@@ -495,7 +414,7 @@ def ask_for_additional_options(args, cfg):
                  " current usage with 'pmbootstrap stats'. Answer with 0 for"
                  " infinite.")
     regex = "0|[0-9]+(k|M|G|T|Ki|Mi|Gi|Ti)"
-    answer = pmb.helpers.cli.ask("Ccache size", None, args.ccache_size,
+    answer = pmb.helpers.cli.ask(args, "Ccache size", None, args.ccache_size,
                                  lowercase_answer=False,
                                  validation_regex=regex)
     cfg["pmbootstrap"]["ccache_size"] = answer
@@ -564,7 +483,7 @@ def ask_for_mirror(args):
     mirrors_list = []
     # require one valid mirror index selected by user
     while len(mirrors_list) != 1:
-        answer = pmb.helpers.cli.ask("Select a mirror", None,
+        answer = pmb.helpers.cli.ask(args, "Select a mirror", None,
                                      ",".join(mirror_indexes),
                                      validation_regex=regex)
         mirrors_list = []
@@ -580,7 +499,8 @@ def ask_for_mirror(args):
 
 def ask_for_hostname(args, device):
     while True:
-        ret = pmb.helpers.cli.ask("Device hostname (short form, e.g. 'foo')",
+        ret = pmb.helpers.cli.ask(args,
+                                  "Device hostname (short form, e.g. 'foo')",
                                   None, (args.hostname or device), True)
         if not pmb.helpers.other.validate_hostname(ret):
             continue
@@ -608,36 +528,15 @@ def ask_build_pkgs_on_install(args):
                                    default=args.build_pkgs_on_install)
 
 
-def get_locales():
-    ret = []
-    list_path = f"{pmb.config.pmb_src}/pmb/data/locales"
-    with open(list_path, "r") as handle:
-        for line in handle:
-            ret += [line.rstrip()]
-    return ret
-
-
 def ask_for_locale(args):
-    locales = get_locales()
+    locales = pmb.config.locales
-    logging.info("Choose your preferred locale, like e.g. en_US. Only UTF-8"
+    logging.info(f"Available locales ({len(locales)}): {', '.join(locales)}")
-                 " is supported, it gets appended automatically. Use"
+    return pmb.helpers.cli.ask(args, "Choose default locale for installation",
-                 " tab-completion if needed.")
+                               choices=None,
-
+                               default=args.locale,
-    while True:
+                               lowercase_answer=False,
-        ret = pmb.helpers.cli.ask("Locale",
+                               validation_regex="|".join(locales),
-                                  choices=None,
+                               complete=locales)
-                                  default=args.locale.replace(".UTF-8", ""),
-                                  lowercase_answer=False,
-                                  complete=locales)
-        ret = ret.replace(".UTF-8", "")
-        if ret not in locales:
-            logging.info("WARNING: this locale is not in the list of known"
-                         " valid locales.")
-            if pmb.helpers.cli.ask() != "y":
-                # Ask again
-                continue
-
-        return f"{ret}.UTF-8"
 
 
 def frontend(args):
@@ -663,51 +562,32 @@ def frontend(args):
     pmb.config.pmaports.switch_to_channel_branch(args, channel)
     cfg["pmbootstrap"]["is_default_channel"] = "False"
 
-    # Copy the git hooks if master was checked out. (Don't symlink them and
-    # only do it on master, so the git hooks don't change unexpectedly when
-    # having a random branch checked out.)
-    branch_current = pmb.helpers.git.rev_parse(args, args.aports,
-                                               extra_args=["--abbrev-ref"])
-    if branch_current == "master":
-        logging.info("NOTE: pmaports is on master branch, copying git hooks.")
-        pmb.config.pmaports.install_githooks(args)
-
     # Device
-    device, device_exists, kernel = ask_for_device(args)
+    device, device_exists, kernel, nonfree = ask_for_device(args)
     cfg["pmbootstrap"]["device"] = device
     cfg["pmbootstrap"]["kernel"] = kernel
-
+    cfg["pmbootstrap"]["nonfree_firmware"] = str(nonfree["firmware"])
-    info = pmb.parse.deviceinfo(args, device)
+    cfg["pmbootstrap"]["nonfree_userland"] = str(nonfree["userland"])
-    apkbuild_path = pmb.helpers.devices.find_path(args, device, 'APKBUILD')
-    if apkbuild_path:
-        apkbuild = pmb.parse.apkbuild(apkbuild_path)
-        ask_for_provider_select(args, apkbuild, cfg["providers"])
 
     # Device keymap
     if device_exists:
-        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, info)
+        cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, device)
-
-    cfg["pmbootstrap"]["user"] = ask_for_username(args)
-    ask_for_provider_select_pkg(args, "postmarketos-base", cfg["providers"])
-    ask_for_provider_select_pkg(args, "postmarketos-base-ui", cfg["providers"])
 
+    # Username
+    cfg["pmbootstrap"]["user"] = pmb.helpers.cli.ask(args, "Username", None,
+                                                     args.user, False,
+                                                     "[a-z_][a-z0-9_-]*")
     # UI and various build options
-    ui = ask_for_ui(args, info)
+    ui = ask_for_ui(args, device)
     cfg["pmbootstrap"]["ui"] = ui
     cfg["pmbootstrap"]["ui_extras"] = str(ask_for_ui_extras(args, ui))
-
-    # systemd
-    cfg["pmbootstrap"]["systemd"] = ask_for_systemd(args, ui)
-
-    ask_for_provider_select_pkg(args, f"postmarketos-ui-{ui}",
-                                cfg["providers"])
     ask_for_additional_options(args, cfg)
 
     # Extra packages to be installed to rootfs
     logging.info("Additional packages that will be installed to rootfs."
                  " Specify them in a comma separated list (e.g.: vim,file)"
                  " or \"none\"")
-    extra = pmb.helpers.cli.ask("Extra packages", None,
+    extra = pmb.helpers.cli.ask(args, "Extra packages", None,
                                 args.extra_packages,
                                 validation_regex=r"^([-.+\w]+)(,[-.+\w]+)*$")
     cfg["pmbootstrap"]["extra_packages"] = extra
@@ -740,9 +620,13 @@ def frontend(args):
             pmb.helpers.cli.confirm(
                 args, "Zap existing chroots to apply configuration?",
                 default=True)):
-        setattr(args, "deviceinfo", info)
+        setattr(args, "deviceinfo", pmb.parse.deviceinfo(args, device=device))
 
         # Do not zap any existing packages or cache_http directories
         pmb.chroot.zap(args, confirm=False)
 
+    logging.info("WARNING: The chroots and git repositories in the work dir do"
+                 " not get updated automatically.")
+    logging.info("Run 'pmbootstrap status' once a day before working with"
+                 " pmbootstrap to make sure that everything is up-to-date.")
     logging.info("DONE!")

pmb/config/load.py

@@ -1,32 +1,11 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import configparser
 import os
-import sys
 import pmb.config
 
 
-def sanity_check(args, cfg, key, allowed, print_path):
-    value = cfg["pmbootstrap"][key]
-
-    if value in allowed:
-        return
-
-    logging.error(f"pmbootstrap.cfg: invalid value for {key}: '{value}'")
-    logging.error(f"Allowed: {', '.join(allowed)}")
-
-    if print_path:
-        logging.error(f"Fix it here and try again: {args.config}")
-
-    sys.exit(1)
-
-
-def sanity_checks(args, cfg, print_path=True):
-    for key, allowed in pmb.config.allowed_values.items():
-        sanity_check(args, cfg, key, allowed, print_path)
-
-
 def load(args):
     cfg = configparser.ConfigParser()
     if os.path.isfile(args.config):
@@ -34,8 +13,6 @@ def load(args):
 
     if "pmbootstrap" not in cfg:
         cfg["pmbootstrap"] = {}
-    if "providers" not in cfg:
-        cfg["providers"] = {}
 
     for key in pmb.config.defaults:
         if key in pmb.config.config_keys and key not in cfg["pmbootstrap"]:
@@ -52,6 +29,4 @@ def load(args):
                           f" {cfg['pmbootstrap'][key]}")
             del cfg["pmbootstrap"][key]
 
-    sanity_checks(args, cfg)
-
     return cfg

pmb/config/merge_with_args.py

@@ -1,10 +1,11 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config
 
 
 def merge_with_args(args):
-    """We have the internal config (pmb/config/__init__.py) and the user config
+    """
+    We have the internal config (pmb/config/__init__.py) and the user config
     (usually ~/.config/pmbootstrap.cfg, can be changed with the '-c'
     parameter).
 
@@ -28,7 +29,6 @@ def merge_with_args(args):
                 if isinstance(default, bool):
                     value = (value.lower() == "true")
             setattr(args, key, value)
-    setattr(args, 'selected_providers', cfg['providers'])
 
     # Use defaults from pmb.config.defaults
     for key, value in pmb.config.defaults.items():

pmb/config/other.py

@@ -1,30 +0,0 @@
-# Copyright 2024 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-import pmb.helpers.ui
-import pmb.config.pmaports
-
-
-def is_systemd_selected(args):
-    if "systemd" not in pmb.config.pmaports.read_config_repos(args):
-        return False
-    if pmb.helpers.ui.check_option(args, args.ui, "pmb:systemd-never"):
-        return False
-    if args.systemd == "always":
-        return True
-    if args.systemd == "never":
-        return False
-    return pmb.helpers.ui.check_option(args, args.ui, "pmb:systemd")
-
-
-def systemd_selected_str(args):
-    if "systemd" not in pmb.config.pmaports.read_config_repos(args):
-        return "no", "not supported by pmaports branch"
-    if pmb.helpers.ui.check_option(args, args.ui, "pmb:systemd-never"):
-        return "no", "not supported by selected UI"
-    if args.systemd == "always":
-        return "yes", "'always' selected in 'pmbootstrap init'"
-    if args.systemd == "never":
-        return "no", "'never' selected in 'pmbootstrap init'"
-    if pmb.helpers.ui.check_option(args, args.ui, "pmb:systemd"):
-        return "yes", "default for selected UI"
-    return "no", "default for selected UI"

pmb/config/pmaports.py

@@ -1,9 +1,8 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 import logging
 import os
-import sys
 
 import pmb.config
 import pmb.helpers.git
@@ -59,7 +58,7 @@ def check_version_pmaports(real):
 
 def check_version_pmbootstrap(min):
     # Compare versions
-    real = pmb.__version__
+    real = pmb.config.version
     if pmb.parse.version.compare(real, min) >= 0:
         return
 
@@ -81,40 +80,18 @@ def check_version_pmbootstrap(min):
                        " of pmbootstrap from git.")
 
 
-def read_config_repos(args):
-    """ Read the sections starting with "repo:" from pmaports.cfg. """
-    # Try cache first
-    cache_key = "pmb.config.pmaports.read_config_repos"
-    if pmb.helpers.other.cache[cache_key]:
-        return pmb.helpers.other.cache[cache_key]
-
-    cfg = configparser.ConfigParser()
-    cfg.read(f"{args.aports}/pmaports.cfg")
-
-    ret = {}
-    for section in cfg.keys():
-        if not section.startswith("repo:"):
-            continue
-        repo = section.split("repo:", 1)[1]
-        ret[repo] = cfg[section]
-
-    # Cache and return
-    pmb.helpers.other.cache[cache_key] = ret
-    return ret
-
-
 def read_config(args):
-    """Read and verify pmaports.cfg."""
+    """ Read and verify pmaports.cfg. """
     # Try cache first
     cache_key = "pmb.config.pmaports.read_config"
-    if pmb.helpers.other.cache[cache_key]:
+    if args.cache[cache_key]:
-        return pmb.helpers.other.cache[cache_key]
+        return args.cache[cache_key]
 
     # Migration message
     if not os.path.exists(args.aports):
-        logging.error(f"ERROR: pmaports dir not found: {args.aports}")
+        raise RuntimeError("We have split the aports repository from the"
-        logging.error("Did you run 'pmbootstrap init'?")
+                           " pmbootstrap repository (#383). Please run"
-        sys.exit(1)
+                           " 'pmbootstrap init' again to clone it.")
 
     # Require the config
     path_cfg = args.aports + "/pmaports.cfg"
@@ -135,21 +112,17 @@ def read_config(args):
     ret["channel"] = pmb.helpers.pmaports.get_channel_new(ret["channel"])
 
     # Cache and return
-    pmb.helpers.other.cache[cache_key] = ret
+    args.cache[cache_key] = ret
     return ret
 
 
 def read_config_channel(args):
-    """Get the properties of the currently active channel in pmaports.git.
+    """ Get the properties of the currently active channel in pmaports.git,
-
+        as specified in channels.cfg (https://postmarketos.org/channels.cfg).
-    As specified in channels.cfg (https://postmarketos.org/channels.cfg).
+        :returns: {"description: ...,
-
+                   "branch_pmaports": ...,
-    :returns: {"description: ...,
+                   "branch_aports": ...,
-               "branch_pmaports": ...,
+                   "mirrordir_alpine": ...} """
-               "branch_aports": ...,
-               "mirrordir_alpine": ...}
-
-    """
     channel = read_config(args)["channel"]
     channels_cfg = pmb.helpers.git.parse_channels_cfg(args)
 
@@ -183,12 +156,9 @@ def init(args):
 
 
 def switch_to_channel_branch(args, channel_new):
-    """Checkout the channel's branch in pmaports.git.
+    """ Checkout the channel's branch in pmaports.git.
-
+        :channel_new: channel name (e.g. "edge", "v21.03")
-    :channel_new: channel name (e.g. "edge", "v21.03")
+        :returns: True if another branch was checked out, False otherwise """
-
-    :returns: True if another branch was checked out, False otherwise
-    """
     # Check current pmaports branch channel
     channel_current = read_config(args)["channel"]
     if channel_current == channel_new:
@@ -216,22 +186,8 @@ def switch_to_channel_branch(args, channel_new):
                            f"{args.aports}")
 
     # Invalidate all caches
-    pmb.helpers.other.init_cache()
+    pmb.helpers.args.add_cache(args)
 
     # Verify pmaports.cfg on new branch
     read_config(args)
     return True
-
-
-def install_githooks(args):
-    hooks_dir = os.path.join(args.aports, ".githooks")
-    if not os.path.exists(hooks_dir):
-        logging.info("No .githooks dir found")
-        return
-    for h in os.listdir(hooks_dir):
-        src = os.path.join(hooks_dir, h)
-        # Use git default hooks dir so users can ignore our hooks
-        # if they dislike them by setting "core.hooksPath" git config
-        dst = os.path.join(args.aports, ".git", "hooks", h)
-        if pmb.helpers.run.user(args, ["cp", src, dst], check=False):
-            logging.warning(f"WARNING: Copying git hook failed: {dst}")

pmb/config/save.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 import logging

pmb/config/sudo.py

@@ -1,35 +0,0 @@
-# Copyright 2023 Anjandev Momi
-# SPDX-License-Identifier: GPL-3.0-or-later
-import os
-import shutil
-from functools import lru_cache
-from typing import Optional
-
-
-@lru_cache()
-def which_sudo() -> Optional[str]:
-    """Return a command required to run commands as root, if any.
-
-    Find whether sudo or doas is installed for commands that require root.
-    Allows user to override preferred sudo with PMB_SUDO env variable.
-    """
-    if os.getuid() == 0:
-        return None
-
-    supported_sudos = ['doas', 'sudo']
-
-    user_set_sudo = os.getenv("PMB_SUDO")
-    if user_set_sudo is not None:
-        if shutil.which(user_set_sudo) is None:
-            raise RuntimeError("PMB_SUDO environmental variable is set to"
-                               f" {user_set_sudo} but pmbootstrap cannot find"
-                               " this command on your system.")
-        return user_set_sudo
-
-    for sudo in supported_sudos:
-        if shutil.which(sudo) is not None:
-            return sudo
-
-    raise RuntimeError("Can't find sudo or doas required to run pmbootstrap."
-                       " Please install sudo, doas, or specify your own sudo"
-                       " with the PMB_SUDO environmental variable.")

pmb/config/workdir.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 """ Save, read, verify workdir state related information in $WORK/workdir.cfg,
     for example the init dates of the chroots. This is not saved in
@@ -13,7 +13,7 @@ import pmb.config.pmaports
 
 
 def chroot_save_init(args, suffix):
-    """Save the chroot initialization data in $WORK/workdir.cfg."""
+    """ Save the chroot initialization data in $WORK/workdir.cfg. """
     # Read existing cfg
     cfg = configparser.ConfigParser()
     path = args.work + "/workdir.cfg"
@@ -35,15 +35,11 @@ def chroot_save_init(args, suffix):
         cfg.write(handle)
 
 
-def chroots_outdated(args, suffix=None):
+def chroots_outdated(args):
-    """Check if init dates from workdir.cfg indicate that any chroot is
+    """ Check if init dates from workdir.cfg indicate that any chroot is
-    outdated.
+        outdated.
-
+        :returns: True if any of the chroots are outdated and should be zapped,
-    :param suffix: only check a specific chroot suffix
+                  False otherwise """
-
-    :returns: True if any of the chroots are outdated and should be zapped,
-              False otherwise
-    """
     # Skip if workdir.cfg doesn't exist
     path = args.work + "/workdir.cfg"
     if not os.path.exists(path):
@@ -56,10 +52,8 @@ def chroots_outdated(args, suffix=None):
         return False
 
     date_outdated = time.time() - pmb.config.chroot_outdated
-    for cfg_suffix in cfg[key]:
+    for suffix in cfg[key]:
-        if suffix and cfg_suffix != suffix:
+        date_init = int(cfg[key][suffix])
-            continue
-        date_init = int(cfg[key][cfg_suffix])
         if date_init <= date_outdated:
             return True
     return False
@@ -88,12 +82,10 @@ def chroot_check_channel(args, suffix):
 
 
 def clean(args):
-    """Remove obsolete data data from workdir.cfg.
+    """ Remove obsolete data data from workdir.cfg.
-
+        :returns: None if workdir does not exist,
-    :returns: None if workdir does not exist,
+                  True if config was rewritten,
-        True if config was rewritten,
+                  False if config did not change """
-        False if config did not change
-    """
     # Skip if workdir.cfg doesn't exist
     path = args.work + "/workdir.cfg"
     if not os.path.exists(path):

pmb/data/keys/alpine-devel@lists.alpinelinux.org-5e69ca50.rsa.pub

@@ -1,9 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXEJ8uVwJPODshTkf2BH
-pH5fVVDppOa974+IQJsZDmGd3Ny0dcd+WwYUhNFUW3bAfc3/egaMWCaprfaHn+oS
-4ddbOFgbX8JCHdru/QMAAU0aEWSMybfJGA569c38fNUF/puX6XK/y0lD2SS3YQ/a
-oJ5jb5eNrQGR1HHMAd0G9WC4JeZ6WkVTkrcOw55F00aUPGEjejreXBerhTyFdabo
-dSfc1TILWIYD742Lkm82UBOPsOSdSfOdsMOOkSXxhdCJuCQQ70DHkw7Epy9r+X33
-ybI4r1cARcV75OviyhD8CFhAlapLKaYnRFqFxlA515e6h8i8ih/v3MSEW17cCK0b
-QwIDAQAB
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-60ac2099.rsa.pub

@@ -1,9 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwR4uJVtJOnOFGchnMW5Y
-j5/waBdG1u5BTMlH+iQMcV5+VgWhmpZHJCBz3ocD+0IGk2I68S5TDOHec/GSC0lv
-6R9o6F7h429GmgPgVKQsc8mPTPtbjJMuLLs4xKc+viCplXc0Nc0ZoHmCH4da6fCV
-tdpHQjVe6F9zjdquZ4RjV6R6JTiN9v924dGMAkbW/xXmamtz51FzondKC52Gh8Mo
-/oA0/T0KsCMCi7tb4QNQUYrf+Xcha9uus4ww1kWNZyfXJB87a2kORLiWMfs2IBBJ
-TmZ2Fnk0JnHDb8Oknxd9PvJPT0mvyT8DA+KIAPqNvOjUXP4bnjEHJcoCP9S5HkGC
-IQIDAQAB
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-6165ee59.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAutQkua2CAig4VFSJ7v54
-ALyu/J1WB3oni7qwCZD3veURw7HxpNAj9hR+S5N/pNeZgubQvJWyaPuQDm7PTs1+
-tFGiYNfAsiibX6Rv0wci3M+z2XEVAeR9Vzg6v4qoofDyoTbovn2LztaNEjTkB+oK
-tlvpNhg1zhou0jDVYFniEXvzjckxswHVb8cT0OMTKHALyLPrPOJzVtM9C1ew2Nnc
-3848xLiApMu3NBk0JqfcS3Bo5Y2b1FRVBvdt+2gFoKZix1MnZdAEZ8xQzL/a0YS5
-Hd0wj5+EEKHfOd3A75uPa/WQmA+o0cBFfrzm69QDcSJSwGpzWrD1ScH3AK8nWvoj
-v7e9gukK/9yl1b4fQQ00vttwJPSgm9EnfPHLAtgXkRloI27H6/PuLoNvSAMQwuCD
-hQRlyGLPBETKkHeodfLoULjhDi1K2gKJTMhtbnUcAA7nEphkMhPWkBpgFdrH+5z4
-Lxy+3ek0cqcI7K68EtrffU8jtUj9LFTUC8dERaIBs7NgQ/LfDbDfGh9g6qVj1hZl
-k9aaIPTm/xsi8v3u+0qaq7KzIBc9s59JOoA8TlpOaYdVgSQhHHLBaahOuAigH+VI
-isbC9vmqsThF2QdDtQt37keuqoda2E6sL7PUvIyVXDRfwX7uMDjlzTxHTymvq2Ck
-htBqojBnThmjJQFgZXocHG8CAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-61666e3f.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlEyxkHggKCXC2Wf5Mzx4
-nZLFZvU2bgcA3exfNPO/g1YunKfQY+Jg4fr6tJUUTZ3XZUrhmLNWvpvSwDS19ZmC
-IXOu0+V94aNgnhMsk9rr59I8qcbsQGIBoHzuAl8NzZCgdbEXkiY90w1skUw8J57z
-qCsMBydAueMXuWqF5nGtYbi5vHwK42PffpiZ7G5Kjwn8nYMW5IZdL6ZnMEVJUWC9
-I4waeKg0yskczYDmZUEAtrn3laX9677ToCpiKrvmZYjlGl0BaGp3cxggP2xaDbUq
-qfFxWNgvUAb3pXD09JM6Mt6HSIJaFc9vQbrKB9KT515y763j5CC2KUsilszKi3mB
-HYe5PoebdjS7D1Oh+tRqfegU2IImzSwW3iwA7PJvefFuc/kNIijfS/gH/cAqAK6z
-bhdOtE/zc7TtqW2Wn5Y03jIZdtm12CxSxwgtCF1NPyEWyIxAQUX9ACb3M0FAZ61n
-fpPrvwTaIIxxZ01L3IzPLpbc44x/DhJIEU+iDt6IMTrHOphD9MCG4631eIdB0H1b
-6zbNX1CXTsafqHRFV9XmYYIeOMggmd90s3xIbEujA6HKNP/gwzO6CDJ+nHFDEqoF
-SkxRdTkEqjTjVKieURW7Swv7zpfu5PrsrrkyGnsRrBJJzXlm2FOOxnbI2iSL1B5F
-rO5kbUxFeZUIDq+7Yv4kLWcCAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616a9724.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnC+bR4bHf/L6QdU4puhQ
-gl1MHePszRC38bzvVFDUJsmCaMCL2suCs2A2yxAgGb9pu9AJYLAmxQC4mM3jNqhg
-/E7yuaBbek3O02zN/ctvflJ250wZCy+z0ZGIp1ak6pu1j14IwHokl9j36zNfGtfv
-ADVOcdpWITFFlPqwq1qt/H3UsKVmtiF3BNWWTeUEQwKvlU8ymxgS99yn0+4OPyNT
-L3EUeS+NQJtDS01unau0t7LnjUXn+XIneWny8bIYOQCuVR6s/gpIGuhBaUqwaJOw
-7jkJZYF2Ij7uPb4b5/R3vX2FfxxqEHqssFSg8FFUNTZz3qNZs0CRVyfA972g9WkJ
-hPfn31pQYil4QGRibCMIeU27YAEjXoqfJKEPh4UWMQsQLrEfdGfb8VgwrPbniGfU
-L3jKJR3VAafL9330iawzVQDlIlwGl6u77gEXMl9K0pfazunYhAp+BMP+9ot5ckK+
-osmrqj11qMESsAj083GeFdfV3pXEIwUytaB0AKEht9DbqUfiE/oeZ/LAXgySMtVC
-sbC4ESmgVeY2xSBIJdDyUap7FR49GGrw0W49NUv9gRgQtGGaNVQQO9oGL2PBC41P
-iWF9GLoX30HIz1P8PF/cZvicSSPkQf2Z6TV+t0ebdGNS5DjapdnCrq8m9Z0pyKsQ
-uxAL2a7zX8l5i1CZh1ycUGsCAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616abc23.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA0MfCDrhODRCIxR9Dep1s
-eXafh5CE5BrF4WbCgCsevyPIdvTeyIaW4vmO3bbG4VzhogDZju+R3IQYFuhoXP5v
-Y+zYJGnwrgz3r5wYAvPnLEs1+dtDKYOgJXQj+wLJBW1mzRDL8FoRXOe5iRmn1EFS
-wZ1DoUvyu7/J5r0itKicZp3QKED6YoilXed+1vnS4Sk0mzN4smuMR9eO1mMCqNp9
-9KTfRDHTbakIHwasECCXCp50uXdoW6ig/xUAFanpm9LtK6jctNDbXDhQmgvAaLXZ
-LvFqoaYJ/CvWkyYCgL6qxvMvVmPoRv7OPcyni4xR/WgWa0MSaEWjgPx3+yj9fiMA
-1S02pFWFDOr5OUF/O4YhFJvUCOtVsUPPfA/Lj6faL0h5QI9mQhy5Zb9TTaS9jB6p
-Lw7u0dJlrjFedk8KTJdFCcaGYHP6kNPnOxMylcB/5WcztXZVQD5WpCicGNBxCGMm
-W64SgrV7M07gQfL/32QLsdqPUf0i8hoVD8wfQ3EpbQzv6Fk1Cn90bZqZafg8XWGY
-wddhkXk7egrr23Djv37V2okjzdqoyLBYBxMz63qQzFoAVv5VoY2NDTbXYUYytOvG
-GJ1afYDRVWrExCech1mX5ZVUB1br6WM+psFLJFoBFl6mDmiYt0vMYBddKISsvwLl
-IJQkzDwtXzT2cSjoj3T5QekCAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616ac3bc.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvaaoSLab+IluixwKV5Od
-0gib2YurjPatGIbn5Ov2DLUFYiebj2oJINXJSwUOO+4WcuHFEqiL/1rya+k5hLZt
-hnPL1tn6QD4rESznvGSasRCQNT2vS/oyZbTYJRyAtFkEYLlq0t3S3xBxxHWuvIf0
-qVxVNYpQWyM3N9RIeYBR/euXKJXileSHk/uq1I5wTC0XBIHWcthczGN0m9wBEiWS
-0m3cnPk4q0Ea8mUJ91Rqob19qETz6VbSPYYpZk3qOycjKosuwcuzoMpwU8KRiMFd
-5LHtX0Hx85ghGsWDVtS0c0+aJa4lOMGvJCAOvDfqvODv7gKlCXUpgumGpLdTmaZ8
-1RwqspAe3IqBcdKTqRD4m2mSg23nVx2FAY3cjFvZQtfooT7q1ItRV5RgH6FhQSl7
-+6YIMJ1Bf8AAlLdRLpg+doOUGcEn+pkDiHFgI8ylH1LKyFKw+eXaAml/7DaWZk1d
-dqggwhXOhc/UUZFQuQQ8A8zpA13PcbC05XxN2hyP93tCEtyynMLVPtrRwDnHxFKa
-qKzs3rMDXPSXRn3ZZTdKH3069ApkEjQdpcwUh+EmJ1Ve/5cdtzT6kKWCjKBFZP/s
-91MlRrX2BTRdHaU5QJkUheUtakwxuHrdah2F94lRmsnQlpPr2YseJu6sIE+Dnx4M
-CfhdVbQL2w54R645nlnohu8CAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616adfeb.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAq0BFD1D4lIxQcsqEpQzU
-pNCYM3aP1V/fxxVdT4DWvSI53JHTwHQamKdMWtEXetWVbP5zSROniYKFXd/xrD9X
-0jiGHey3lEtylXRIPxe5s+wXoCmNLcJVnvTcDtwx/ne2NLHxp76lyc25At+6RgE6
-ADjLVuoD7M4IFDkAsd8UQ8zM0Dww9SylIk/wgV3ZkifecvgUQRagrNUdUjR56EBZ
-raQrev4hhzOgwelT0kXCu3snbUuNY/lU53CoTzfBJ5UfEJ5pMw1ij6X0r5S9IVsy
-KLWH1hiO0NzU2c8ViUYCly4Fe9xMTFc6u2dy/dxf6FwERfGzETQxqZvSfrRX+GLj
-/QZAXiPg5178hT/m0Y3z5IGenIC/80Z9NCi+byF1WuJlzKjDcF/TU72zk0+PNM/H
-Kuppf3JT4DyjiVzNC5YoWJT2QRMS9KLP5iKCSThwVceEEg5HfhQBRT9M6KIcFLSs
-mFjx9kNEEmc1E8hl5IR3+3Ry8G5/bTIIruz14jgeY9u5jhL8Vyyvo41jgt9sLHR1
-/J1TxKfkgksYev7PoX6/ZzJ1ksWKZY5NFoDXTNYUgzFUTOoEaOg3BAQKadb3Qbbq
-XIrxmPBdgrn9QI7NCgfnAY3Tb4EEjs3ON/BNyEhUENcXOH6I1NbcuBQ7g9P73kE4
-VORdoc8MdJ5eoKBpO8Ww8HECAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616ae350.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAyduVzi1mWm+lYo2Tqt/0
-XkCIWrDNP1QBMVPrE0/ZlU2bCGSoo2Z9FHQKz/mTyMRlhNqTfhJ5qU3U9XlyGOPJ
-piM+b91g26pnpXJ2Q2kOypSgOMOPA4cQ42PkHBEqhuzssfj9t7x47ppS94bboh46
-xLSDRff/NAbtwTpvhStV3URYkxFG++cKGGa5MPXBrxIp+iZf9GnuxVdST5PGiVGP
-ODL/b69sPJQNbJHVquqUTOh5Ry8uuD2WZuXfKf7/C0jC/ie9m2+0CttNu9tMciGM
-EyKG1/Xhk5iIWO43m4SrrT2WkFlcZ1z2JSf9Pjm4C2+HovYpihwwdM/OdP8Xmsnr
-DzVB4YvQiW+IHBjStHVuyiZWc+JsgEPJzisNY0Wyc/kNyNtqVKpX6dRhMLanLmy+
-f53cCSI05KPQAcGj6tdL+D60uKDkt+FsDa0BTAobZ31OsFVid0vCXtsbplNhW1IF
-HwsGXBTVcfXg44RLyL8Lk/2dQxDHNHzAUslJXzPxaHBLmt++2COa2EI1iWlvtznk
-Ok9WP8SOAIj+xdqoiHcC4j72BOVVgiITIJNHrbppZCq6qPR+fgXmXa+sDcGh30m6
-9Wpbr28kLMSHiENCWTdsFij+NQTd5S47H7XTROHnalYDuF1RpS+DpQidT5tUimaT
-JZDr++FjKrnnijbyNF8b98UCAwEAAQ==
------END PUBLIC KEY-----

pmb/data/keys/alpine-devel@lists.alpinelinux.org-616db30d.rsa.pub

@@ -1,14 +0,0 @@
------BEGIN PUBLIC KEY-----
-MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAnpUpyWDWjlUk3smlWeA0
-lIMW+oJ38t92CRLHH3IqRhyECBRW0d0aRGtq7TY8PmxjjvBZrxTNDpJT6KUk4LRm
-a6A6IuAI7QnNK8SJqM0DLzlpygd7GJf8ZL9SoHSH+gFsYF67Cpooz/YDqWrlN7Vw
-tO00s0B+eXy+PCXYU7VSfuWFGK8TGEv6HfGMALLjhqMManyvfp8hz3ubN1rK3c8C
-US/ilRh1qckdbtPvoDPhSbTDmfU1g/EfRSIEXBrIMLg9ka/XB9PvWRrekrppnQzP
-hP9YE3x/wbFc5QqQWiRCYyQl/rgIMOXvIxhkfe8H5n1Et4VAorkpEAXdsfN8KSVv
-LSMazVlLp9GYq5SUpqYX3KnxdWBgN7BJoZ4sltsTpHQ/34SXWfu3UmyUveWj7wp0
-x9hwsPirVI00EEea9AbP7NM2rAyu6ukcm4m6ATd2DZJIViq2es6m60AE6SMCmrQF
-wmk4H/kdQgeAELVfGOm2VyJ3z69fQuywz7xu27S6zTKi05Qlnohxol4wVb6OB7qG
-LPRtK9ObgzRo/OPumyXqlzAi/Yvyd1ZQk8labZps3e16bQp8+pVPiumWioMFJDWV
-GZjCmyMSU8V6MB6njbgLHoyg2LCukCAeSjbPGGGYhnKLm1AKSoJh3IpZuqcKCk5C
-8CM1S15HxV78s9dFntEqIokCAwEAAQ==
------END PUBLIC KEY-----

pmb/data/locales

@@ -1,304 +0,0 @@
-C
-a_DJ
-aa_ER
-aa_ET
-af_ZA
-agr_PE
-ak_GH
-am_ET
-an_ES
-anp_IN
-ar_AE
-ar_BH
-ar_DZ
-ar_EG
-ar_IN
-ar_IQ
-ar_JO
-ar_KW
-ar_LB
-ar_LY
-ar_MA
-ar_OM
-ar_QA
-ar_SA
-ar_SD
-ar_SS
-ar_SY
-ar_TN
-ar_YE
-as_IN
-ast_ES
-ayc_PE
-az_AZ
-az_IR
-be_BY
-bem_ZM
-ber_DZ
-ber_MA
-bg_BG
-bhb_IN
-bho_IN
-bho_NP
-bi_VU
-bn_BD
-bn_IN
-bo_CN
-bo_IN
-br_FR
-brx_IN
-bs_BA
-byn_ER
-ca_AD
-ca_ES
-ca_FR
-ca_IT
-ce_RU
-ch_DE
-chr_US
-cmn_TW
-crh_UA
-cs_CZ
-csb_PL
-cv_RU
-cy_GB
-da_DK
-de_AT
-de_BE
-de_CH
-de_DE
-de_IT
-de_LI
-de_LU
-doi_IN
-dsb_DE
-dv_MV
-dz_BT
-el_CY
-el_GR
-en_AG
-en_AU
-en_BW
-en_CA
-en_DK
-en_GB
-en_HK
-en_IE
-en_IL
-en_IN
-en_NG
-en_NZ
-en_PH
-en_SC
-en_SG
-en_US
-en_ZA
-en_ZM
-en_ZW
-eo
-es_AR
-es_BO
-es_CL
-es_CO
-es_CR
-es_CU
-es_DO
-es_EC
-es_ES
-es_GT
-es_HN
-es_MX
-es_NI
-es_PA
-es_PE
-es_PR
-es_PY
-es_SV
-es_US
-es_UY
-es_VE
-et_EE
-eu_ES
-fa_IR
-ff_SN
-fi_FI
-fil_PH
-fo_FO
-fr_BE
-fr_CA
-fr_CH
-fr_FR
-fr_LU
-fur_IT
-fy_DE
-fy_NL
-ga_IE
-gd_GB
-gez_ER
-gez_ET
-gl_ES
-gu_IN
-gv_GB
-ha_NG
-hak_TW
-he_IL
-hi_IN
-hif_FJ
-hne_IN
-hr_HR
-hsb_DE
-ht_HT
-hu_HU
-hy_AM
-ia_FR
-id_ID
-ig_NG
-ik_CA
-is_IS
-it_CH
-it_IT
-iu_CA
-ja_JP
-ka_GE
-kab_DZ
-kk_KZ
-kl_GL
-km_KH
-kn_IN
-ko_KR
-kok_IN
-ks_IN
-ku_TR
-kw_GB
-ky_KG
-lb_LU
-lg_UG
-li_BE
-li_NL
-lij_IT
-ln_CD
-lo_LA
-lt_LT
-lv_LV
-lzh_TW
-mag_IN
-mai_IN
-mai_NP
-mfe_MU
-mg_MG
-mhr_RU
-mi_NZ
-miq_NI
-mjw_IN
-mk_MK
-ml_IN
-mn_MN
-mni_IN
-mnw_MM
-mr_IN
-ms_MY
-mt_MT
-my_MM
-nan_TW
-nb_NO
-nds_DE
-nds_NL
-ne_NP
-nhn_MX
-niu_NU
-niu_NZ
-nl_AW
-nl_BE
-nl_NL
-nn_NO
-nr_ZA
-nso_ZA
-oc_FR
-om_ET
-om_KE
-or_IN
-os_RU
-pa_IN
-pa_PK
-pap_AW
-pap_CW
-pl_PL
-ps_AF
-pt_BR
-pt_PT
-quz_PE
-raj_IN
-ro_RO
-ru_RU
-ru_UA
-rw_RW
-sa_IN
-sah_RU
-sat_IN
-sc_IT
-sd_IN
-se_NO
-sgs_LT
-shn_MM
-shs_CA
-si_LK
-sid_ET
-sk_SK
-sl_SI
-sm_WS
-so_DJ
-so_ET
-so_KE
-so_SO
-sq_AL
-sq_MK
-sr_ME
-sr_RS
-ss_ZA
-st_ZA
-sv_FI
-sv_SE
-sw_KE
-sw_TZ
-szl_PL
-ta_IN
-ta_LK
-tcy_IN
-te_IN
-tg_TJ
-th_TH
-the_NP
-ti_ER
-ti_ET
-tig_ER
-tk_TM
-tl_PH
-tn_ZA
-to_TO
-tpi_PG
-tr_CY
-tr_TR
-ts_ZA
-tt_RU
-ug_CN
-uk_UA
-unm_US
-ur_IN
-ur_PK
-uz_UZ
-ve_ZA
-vi_VN
-wa_BE
-wae_CH
-wal_ET
-wo_SN
-xh_ZA
-yi_US
-yo_NG
-yue_HK
-yuw_PG
-zh_CN
-zh_HK
-zh_SG
-zh_TW
-zu_ZA

pmb/data/merge-usr.sh

@@ -1,38 +0,0 @@
-#!/bin/sh -e
-# Copyright 2024 Oliver Smith
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-if [ "$1" != "CALLED_FROM_PMB" ]; then
-	echo "ERROR: this script is only meant to be called by pmbootstrap"
-	exit 1
-fi
-
-CHROOT="$2"
-
-test -n "$CHROOT"
-test -f "$CHROOT"/in-pmbootstrap
-
-if [ -L "$CHROOT"/bin ]; then
-	echo "ERROR: chroot has merged usr already: $CHROOT"
-	exit 1
-fi
-
-# /bin -> /usr/bin
-mv "$CHROOT"/bin/* "$CHROOT"/usr/bin/
-rmdir "$CHROOT"/bin
-ln -s usr/bin "$CHROOT"/bin
-
-# /sbin -> /usr/bin
-mv "$CHROOT"/sbin/* "$CHROOT"/usr/bin/
-rmdir "$CHROOT"/sbin
-ln -s usr/bin "$CHROOT"/sbin
-
-# /lib -> /usr/lib
-mv "$CHROOT"/lib/* "$CHROOT"/usr/lib/
-rmdir "$CHROOT"/lib
-ln -s usr/lib "$CHROOT"/lib
-
-# /usr/sbin -> /usr/bin
-mv "$CHROOT"/usr/sbin/* "$CHROOT"/usr/bin/
-rmdir "$CHROOT"/usr/sbin
-ln -s bin "$CHROOT"/usr/sbin

pmb/export/__init__.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.export.frontend import frontend
 from pmb.export.odin import odin

pmb/export/frontend.py

@@ -19,7 +19,7 @@ def frontend(args):
     pattern = chroot + "/home/pmos/rootfs/" + args.device + "*.img"
     if not glob.glob(pattern):
         logging.info("NOTE: To export the rootfs image, run 'pmbootstrap"
-                     " install' first (without the 'disk' parameter).")
+                     " install' first (without the 'sdcard' parameter).")
 
     # Rebuild the initramfs, just to make sure (see #69)
     flavor = pmb.helpers.frontend._parse_flavor(args, args.autoinstall)

pmb/export/odin.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -19,12 +19,6 @@ def odin(args, flavor, folder):
     pmb.flasher.init(args)
     suffix = "rootfs_" + args.device
 
-    # Backwards compatibility with old mkinitfs (pma#660)
-    suffix_flavor = f"-{flavor}"
-    pmaports_cfg = pmb.config.pmaports.read_config(args)
-    if pmaports_cfg.get("supported_mkinitfs_without_flavors", False):
-        suffix_flavor = ""
-
     # Validate method
     method = args.deviceinfo["flash_method"]
     if not method.startswith("heimdall-"):
@@ -60,16 +54,16 @@ def odin(args, flavor, folder):
         if method == "heimdall-isorec":
             handle.write(
                 # Kernel: copy and append md5
-                f"cp /boot/vmlinuz{suffix_flavor} {odin_kernel_md5}\n"
+                f"cp /boot/vmlinuz-{flavor} {odin_kernel_md5}\n"
                 f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n"
                 # Initramfs: recompress with lzop, append md5
-                f"gunzip -c /boot/initramfs{suffix_flavor}"
+                f"gunzip -c /boot/initramfs-{flavor}"
                 f" | lzop > {odin_initfs_md5}\n"
                 f"md5sum -t {odin_initfs_md5} >> {odin_initfs_md5}\n")
         elif method == "heimdall-bootimg":
             handle.write(
                 # boot.img: copy and append md5
-                f"cp /boot/boot.img{suffix_flavor} {odin_kernel_md5}\n"
+                f"cp /boot/boot.img-{flavor} {odin_kernel_md5}\n"
                 f"md5sum -t {odin_kernel_md5} >> {odin_kernel_md5}\n")
         handle.write(
             # Create tar, remove included files and append md5

pmb/export/symlinks.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -37,7 +37,6 @@ def symlinks(args, flavor, folder):
         f"{args.device}-boot.img": "Boot partition image",
         f"{args.device}-root.img": "Root partition image",
         f"pmos-{args.device}.zip": "Android recovery flashable zip",
-        "lk2nd.img": "Secondary Android bootloader",
     }
 
     # Generate a list of patterns
@@ -54,8 +53,7 @@ def symlinks(args, flavor, folder):
                 f"{path_native}/home/pmos/rootfs/{args.device}-boot.img",
                 f"{path_native}/home/pmos/rootfs/{args.device}-root.img",
                 f"{path_buildroot}/var/lib/postmarketos-android-recovery-" +
-                f"installer/pmos-{args.device}.zip",
+                f"installer/pmos-{args.device}.zip"]
-                f"{path_boot}/lk2nd.img"]
 
     # Generate a list of files from the patterns
     files = []

pmb/flasher/__init__.py

@@ -1,7 +1,6 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pmb.flasher.init import init
-from pmb.flasher.init import install_depends
 from pmb.flasher.run import run
 from pmb.flasher.run import check_partition_blacklist
 from pmb.flasher.variables import variables

pmb/flasher/frontend.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import logging
 import os
@@ -20,7 +20,7 @@ def kernel(args):
         pmb.chroot.initfs.build(args, flavor, "rootfs_" + args.device)
 
     # Check kernel config
-    pmb.parse.kconfig.check(args, flavor, must_exist=False)
+    pmb.parse.kconfig.check(args, flavor)
 
     # Generate the paths and run the flasher
     if args.action_flasher == "boot":
@@ -35,7 +35,7 @@ def kernel(args):
                  " booted:")
     logging.info("ssh {}@{}".format(args.user, pmb.config.default_ip))
     logging.info("NOTE: If you enabled full disk encryption, you should make"
-                 " sure that Unl0kr has been properly configured for your"
+                 " sure that osk-sdl has been properly configured for your"
                  " device")
 
 
@@ -88,8 +88,11 @@ def list_devices(args):
 
 
 def sideload(args):
+    method = args.flash_method or args.deviceinfo["flash_method"]
+    cfg = pmb.config.flashers[method]
+
     # Install depends
-    pmb.flasher.install_depends(args)
+    pmb.chroot.apk.install(args, cfg["depends"])
 
     # Mount the buildroot
     suffix = "buildroot_" + args.deviceinfo["arch"]
@@ -109,63 +112,29 @@ def sideload(args):
     pmb.flasher.run(args, "sideload")
 
 
-def flash_lk2nd(args):
-    method = args.flash_method or args.deviceinfo["flash_method"]
-    if method == "fastboot":
-        # In the future this could be expanded to use "fastboot flash lk2nd $img"
-        # which reflashes/updates lk2nd from itself. For now let the user handle this
-        # manually since supporting the codepath with heimdall requires more effort.
-        pmb.flasher.init(args)
-        logging.info("(native) checking current fastboot product")
-        output = pmb.chroot.root(args, ["fastboot", "getvar", "product"],
-                                 output="interactive", output_return=True)
-        # Variable "product" is e.g. "LK2ND_MSM8974" or "lk2nd-msm8226" depending
-        # on the lk2nd version.
-        if "lk2nd" in output.lower():
-            raise RuntimeError("You are currently running lk2nd. Please reboot into the regular"
-                               " bootloader mode to re-flash lk2nd.")
-
-    # Get the lk2nd package (which is a dependency of the device package)
-    device_pkg = f"device-{args.device}"
-    apkbuild = pmb.helpers.pmaports.get(args, device_pkg)
-    lk2nd_pkg = None
-    for dep in apkbuild["depends"]:
-        if dep.startswith("lk2nd"):
-            lk2nd_pkg = dep
-            break
-
-    if not lk2nd_pkg:
-        raise RuntimeError(f"{device_pkg} does not depend on any lk2nd package")
-
-    suffix = "rootfs_" + args.device
-    pmb.chroot.apk.install(args, [lk2nd_pkg], suffix)
-
-    logging.info("(native) flash lk2nd image")
-    pmb.flasher.run(args, "flash_lk2nd")
-
-
 def frontend(args):
     action = args.action_flasher
     method = args.flash_method or args.deviceinfo["flash_method"]
 
-    if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs",
+    # Legacy alias
-                                       "flash_lk2nd"]:
+    if action == "flash_system":
+        action = "flash_rootfs"
+
+    if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs"]:
         logging.info("This device doesn't support any flash method.")
         return
 
     if action in ["boot", "flash_kernel"]:
         kernel(args)
-    elif action == "flash_rootfs":
+    if action == "flash_rootfs":
         rootfs(args)
-    elif action == "flash_vbmeta":
+    if action == "flash_vbmeta":
         flash_vbmeta(args)
-    elif action == "flash_dtbo":
+    if action == "flash_dtbo":
         flash_dtbo(args)
-    elif action == "flash_lk2nd":
+    if action == "list_flavors":
-        flash_lk2nd(args)
-    elif action == "list_flavors":
         list_flavors(args)
-    elif action == "list_devices":
+    if action == "list_devices":
         list_devices(args)
-    elif action == "sideload":
+    if action == "sideload":
         sideload(args)

pmb/flasher/init.py

@@ -1,12 +1,12 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-import pmb.chroot.apk
 import pmb.config
-import pmb.config.pmaports
+import pmb.chroot.apk
 import pmb.helpers.mount
 
 
-def install_depends(args):
+def init(args):
+    # Validate method
     if hasattr(args, 'flash_method'):
         method = args.flash_method or args.deviceinfo["flash_method"]
     else:
@@ -20,28 +20,10 @@ def install_depends(args):
                            "Make sure, it is packaged for Alpine Linux, or"
                            " package it yourself, and then add it to"
                            " pmb/config/__init__.py.")
-    depends = pmb.config.flashers[method]["depends"]
+    cfg = pmb.config.flashers[method]
 
-    # Depends for some flash methods may be different for various pmaports
+    # Install depends
-    # branches, so read them from pmaports.cfg.
+    pmb.chroot.apk.install(args, cfg["depends"])
-    if method == "fastboot":
-        pmaports_cfg = pmb.config.pmaports.read_config(args)
-        depends = pmaports_cfg.get("supported_fastboot_depends",
-                                   "android-tools,avbtool").split(",")
-    elif method == "heimdall-bootimg":
-        pmaports_cfg = pmb.config.pmaports.read_config(args)
-        depends = pmaports_cfg.get("supported_heimdall_depends",
-                                   "heimdall,avbtool").split(",")
-    elif method == "mtkclient":
-        pmaports_cfg = pmb.config.pmaports.read_config(args)
-        depends = pmaports_cfg.get("supported_mtkclient_depends",
-                                   "mtkclient,android-tools").split(",")
-
-    pmb.chroot.apk.install(args, depends)
-
-
-def init(args):
-    install_depends(args)
 
     # Mount folders from host system
     for folder in pmb.config.flash_mount_bind:

pmb/flasher/run.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.flasher
 import pmb.chroot.initfs
@@ -51,14 +51,6 @@ def run(args, action, flavor=None):
                            " <https://wiki.postmarketos.org/wiki/"
                            "Deviceinfo_reference>")
 
-    if args.no_reboot and ("flash" not in action or method != "heimdall-bootimg"):
-        raise RuntimeError("The '--no-reboot' option is only"
-                           " supported when flashing with heimall-bootimg.")
-
-    if args.resume and ("flash" not in action or method != "heimdall-bootimg"):
-        raise RuntimeError("The '--resume' option is only"
-                           " supported when flashing with heimall-bootimg.")
-
     # Run the commands of each action
     for command in cfg["actions"][action]:
         # Variable replacement
@@ -74,7 +66,5 @@ def run(args, action, flavor=None):
                     check_partition_blacklist(args, key, value)
                     command[i] = command[i].replace(key, value)
 
-        # Remove empty strings
-        command = [x for x in command if x != '']
         # Run the action
         pmb.chroot.root(args, command, output="interactive")

pmb/flasher/variables.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import pmb.config.pmaports
 
@@ -10,41 +10,20 @@ def variables(args, flavor, method):
 
     flash_pagesize = args.deviceinfo['flash_pagesize']
 
-    # TODO Remove _partition_system deviceinfo support once pmaports has been
-    # updated and minimum pmbootstrap version bumped.
-    # See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243
-
     if method.startswith("fastboot"):
         _partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\
             or "boot"
-        _partition_rootfs = args.deviceinfo["flash_fastboot_partition_rootfs"]\
+        _partition_system = args.deviceinfo["flash_fastboot_partition_system"]\
-            or args.deviceinfo["flash_fastboot_partition_system"] or "userdata"
+            or "system"
         _partition_vbmeta = args.deviceinfo["flash_fastboot_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_fastboot_partition_dtbo"]\
             or None
-    # Require that the partitions are specified in deviceinfo for now
-    elif method.startswith("rkdeveloptool"):
-        _partition_kernel = args.deviceinfo["flash_rk_partition_kernel"]\
-            or None
-        _partition_rootfs = args.deviceinfo["flash_rk_partition_rootfs"]\
-            or args.deviceinfo["flash_rk_partition_system"] or None
-        _partition_vbmeta = None
-        _partition_dtbo = None
-    elif method.startswith("mtkclient"):
-        _partition_kernel = args.deviceinfo["flash_mtkclient_partition_kernel"]\
-            or "boot"
-        _partition_rootfs = args.deviceinfo["flash_mtkclient_partition_rootfs"]\
-            or "userdata"
-        _partition_vbmeta = args.deviceinfo["flash_mtkclient_partition_vbmeta"]\
-            or None
-        _partition_dtbo = args.deviceinfo["flash_mtkclient_partition_dtbo"]\
-            or None
     else:
         _partition_kernel = args.deviceinfo["flash_heimdall_partition_kernel"]\
             or "KERNEL"
-        _partition_rootfs = args.deviceinfo["flash_heimdall_partition_rootfs"]\
+        _partition_system = args.deviceinfo["flash_heimdall_partition_system"]\
-            or args.deviceinfo["flash_heimdall_partition_system"] or "SYSTEM"
+            or "SYSTEM"
         _partition_vbmeta = args.deviceinfo["flash_heimdall_partition_vbmeta"]\
             or None
         _partition_dtbo = args.deviceinfo["flash_heimdall_partition_dtbo"]\
@@ -54,7 +33,7 @@ def variables(args, flavor, method):
         # Only one operation is done at same time so it doesn't matter
         # sharing the arg
         _partition_kernel = args.partition
-        _partition_rootfs = args.partition
+        _partition_system = args.partition
         _partition_vbmeta = args.partition
         _partition_dtbo = args.partition
 
@@ -62,14 +41,6 @@ def variables(args, flavor, method):
     if args.deviceinfo["append_dtb"] == "true":
         _dtb = "-dtb"
 
-    _no_reboot = ""
-    if getattr(args, 'no_reboot', False):
-        _no_reboot = "--no-reboot"
-
-    _resume = ""
-    if getattr(args,'resume', False):
-        _resume = "--resume"
-
     vars = {
         "$BOOT": "/mnt/rootfs_" + args.device + "/boot",
         "$DTB": _dtb,
@@ -80,7 +51,7 @@ def variables(args, flavor, method):
         "$PARTITION_KERNEL": _partition_kernel,
         "$PARTITION_INITFS": args.deviceinfo[
             "flash_heimdall_partition_initfs"] or "RECOVERY",
-        "$PARTITION_ROOTFS": _partition_rootfs,
+        "$PARTITION_SYSTEM": _partition_system,
         "$PARTITION_VBMETA": _partition_vbmeta,
         "$PARTITION_DTBO": _partition_dtbo,
         "$FLASH_PAGESIZE": flash_pagesize,
@@ -88,9 +59,7 @@ def variables(args, flavor, method):
                          "/var/lib/postmarketos-android-recovery-installer"
                          "/pmos-" + args.device + ".zip",
         "$UUU_SCRIPT": "/mnt/rootfs_" + args.deviceinfo["codename"] +
-                       "/usr/share/uuu/flash_script.lst",
+                       "/usr/share/uuu/flash_script.lst"
-        "$NO_REBOOT": _no_reboot,
-        "$RESUME": _resume
     }
 
     # Backwards compatibility with old mkinitfs (pma#660)

pmb/helpers/__init__.py

@@ -1,2 +1,2 @@
-# Copyright 2023 Oliver Smith
+# Copyright 2021 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later

pmb/helpers/apk.py

@@ -1,4 +1,4 @@
-# Copyright 2023 Johannes Marbach, Oliver Smith
+# Copyright 2021 Johannes Marbach, Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 
@@ -6,12 +6,12 @@ import pmb.chroot.root
 import pmb.config.pmaports
 import pmb.helpers.cli
 import pmb.helpers.run
-import pmb.helpers.run_core
 import pmb.parse.version
 
 
 def _run(args, command, chroot=False, suffix="native", output="log"):
-    """Run a command.
+    """
+    Run a command.
 
     :param command: command in list form
     :param chroot: whether to run the command inside the chroot or on the host
@@ -28,7 +28,8 @@ def _run(args, command, chroot=False, suffix="native", output="log"):
 
 
 def _prepare_fifo(args, chroot=False, suffix="native"):
-    """Prepare the progress fifo for reading / writing.
+    """
+    Prepare the progress fifo for reading / writing.
 
     :param chroot: whether to run the command inside the chroot or on the host
     :param suffix: chroot suffix. Only applies if the "chroot" parameter is
@@ -51,7 +52,9 @@ def _prepare_fifo(args, chroot=False, suffix="native"):
 
 
 def _create_command_with_progress(command, fifo):
-    """Build a full apk command from a subcommand, set up to redirect progress into a fifo.
+    """
+    Build a full apk command from a subcommand, set up to redirect progress
+    into a fifo.
 
     :param command: apk subcommand in list form
     :param fifo: path of the fifo
@@ -59,13 +62,14 @@ def _create_command_with_progress(command, fifo):
     """
     flags = ["--no-progress", "--progress-fd", "3"]
     command_full = [command[0]] + flags + command[1:]
-    command_flat = pmb.helpers.run_core.flat_cmd(command_full)
+    command_flat = pmb.helpers.run.flat_cmd(command_full)
     command_flat = f"exec 3>{fifo}; {command_flat}"
     return ["sh", "-c", command_flat]
 
 
 def _compute_progress(line):
-    """Compute the progress as a number between 0 and 1.
+    """
+    Compute the progress as a number between 0 and 1.
 
     :param line: line as read from the progress fifo
     :returns: progress as a number between 0 and 1
@@ -81,7 +85,8 @@ def _compute_progress(line):
 
 
 def apk_with_progress(args, command, chroot=False, suffix="native"):
-    """Run an apk subcommand while printing a progress bar to STDOUT.
+    """
+    Run an apk subcommand while printing a progress bar to STDOUT.
 
     :param command: apk subcommand in list form
     :param chroot: whether to run commands inside the chroot or on the host
@@ -106,10 +111,10 @@ def apk_with_progress(args, command, chroot=False, suffix="native"):
 
 
 def check_outdated(args, version_installed, action_msg):
-    """Check if the provided alpine version is outdated.
+    """
-
+    Check if the provided alpine version is outdated, depending on the alpine
-    This depends on the alpine mirrordir (edge, v3.12, ...) related to currently checked out
+    mirrordir (edge, v3.12, ...) related to currently checked out pmaports
-    pmaports branch.
+    branch.
 
     :param version_installed: currently installed apk version, e.g. "2.12.1-r0"
     :param action_msg: string explaining what the user should do to resolve