Compare commits

..

No commits in common. "4ff0b1f6c2466c3bef8f18f2671c1af09eb12357" and "248fe447bdc0611ca55a0379939aaa15ab8d166c" have entirely different histories.

176 changed files with 2346 additions and 2837 deletions

View File

@ -1,9 +0,0 @@
# Allow this repository to be used with the 'b4' tool. See
# https://postmarketos.org/patch-review for details.
[b4]
midmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%s
linkmask = https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%%3C%s%%3E
send-series-to = ~postmarketos/pmbootstrap-devel@lists.sr.ht
send-endpoint-web = NONE
backend = sourcehut

View File

@ -1,26 +0,0 @@
image: alpine/edge
packages:
- sudo
sources:
- https://git.sr.ht/~postmarketos/pmbootstrap
tasks:
- note: |
pmbootstrap/.ci/note.sh
- shellcheck: |
cd pmbootstrap
sudo .ci/shellcheck.sh
- ruff: |
cd pmbootstrap
sudo .ci/ruff.sh
- vermin: |
cd pmbootstrap
sudo .ci/vermin.sh
- codespell: |
cd pmbootstrap
sudo .ci/codespell.sh
- pytest: |
cd pmbootstrap
sudo .ci/pytest.sh
artifacts:
- ".local/var/pmbootstrap/log.txt"
- ".local/var/pmbootstrap/log_testsuite.txt"

View File

@ -1,20 +0,0 @@
#!/bin/sh -ex
# SPDX-License-Identifier: GPL-3.0-or-later
# Copyright 2023 Oliver Smith
# Description: find typos
# https://postmarketos.org/pmb-ci
if [ "$(id -u)" = 0 ]; then
set -x
apk -q add \
py3-codespell
exec su "${TESTUSER:-build}" -c "sh -e $0"
fi
set -x
# -L: words to ignore
codespell \
-L crate \
-L hda \
.

View File

@ -1,6 +0,0 @@
#!/bin/sh -e
printf "\n"
printf "PROTIP: use"
printf " \e[1;32mpmbootstrap ci\e[0m"
printf " to run these scripts locally.\n"

36
.ci/prepare.sh Executable file
View File

@ -0,0 +1,36 @@
#!/bin/sh -e
# Install pmbootstrap depends, set up pmos user with sudo
if [ "$(id -u)" != 0 ]; then
echo "ERROR: this script is meant to be executed in the gitlab-ci"
echo "environment only."
exit 1
fi
topdir="$(realpath "$(dirname "$0")/..")"
cd "$topdir"
ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
apk add -q \
git \
openssl \
py3-pytest \
py3-pytest-cov \
sudo
adduser -D pmos
chown -R pmos:pmos .
echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
su pmos -c "git config --global user.email postmarketos-ci@localhost"
su pmos -c "git config --global user.name postmarketOS_CI"
echo "Initializing pmbootstrap"
if ! su pmos -c "yes '' | pmbootstrap \
--details-to-stdout \
init \
>/tmp/pmb_init 2>&1"; then
cat /tmp/pmb_init
exit 1
fi
echo ""

View File

@ -1,39 +1,6 @@
#!/bin/sh -e
# Description: run pmbootstrap python testsuite
# Options: native slow
# https://postmarketos.org/pmb-ci
if [ "$(id -u)" = 0 ]; then
set -x
apk -q add \
git \
openssl \
py3-pytest \
py3-pytest-cov \
sudo
exec su "${TESTUSER:-build}" -c "sh -e $0"
fi
# Require pytest to be installed on the host system
if [ -z "$(command -v pytest)" ]; then
echo "ERROR: pytest command not found, make sure it is in your PATH."
exit 1
fi
# Use pytest-cov if it is installed to display code coverage
cov_arg=""
if python -c "import pytest_cov" >/dev/null 2>&1; then
cov_arg="--cov=pmb"
fi
echo "Initializing pmbootstrap..."
if ! yes '' | ./pmbootstrap.py \
--details-to-stdout \
init \
>/tmp/pmb_init 2>&1; then
cat /tmp/pmb_init
exit 1
fi
topdir="$(realpath "$(dirname "$0")/..")"
cd "$topdir"
# Make sure that the work folder format is up to date, and that there are no
# mounts from aborted test cases (#1595)
@ -54,19 +21,4 @@ if ! [ -e "$deviceinfo" ]; then
exit 1
fi
# Make sure pmaports is clean, some of the tests will fail otherwise
if [ -n "$(git -C "$pmaports" status --porcelain)" ]; then
echo "ERROR: pmaports dir is not clean"
exit 1
fi
echo "Running pytest..."
echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
pytest \
--color=yes \
-vv \
-x \
$cov_arg \
test \
-m "not skip_ci" \
"$@"
pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"

View File

@ -1,19 +0,0 @@
#!/bin/sh -e
# Description: lint all python scripts
# https://postmarketos.org/pmb-ci
if [ "$(id -u)" = 0 ]; then
set -x
apk -q add ruff
exec su "${TESTUSER:-build}" -c "sh -e $0"
fi
set -x
# __init__.py with additional ignore:
# F401: imported, but not used
# shellcheck disable=SC2046
ruff --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py')
# Check all other files
ruff --exclude=__init__.py .

View File

@ -1,15 +0,0 @@
#!/bin/sh -e
# Description: lint all shell scripts
# https://postmarketos.org/pmb-ci
if [ "$(id -u)" = 0 ]; then
set -x
apk -q add shellcheck
exec su "${TESTUSER:-build}" -c "sh -e $0"
fi
find . -name '*.sh' |
while read -r file; do
echo "shellcheck: $file"
shellcheck "$file"
done

View File

@ -1,23 +1,17 @@
#!/bin/sh -e
# Description: verify that we don't use too new python features
# https://postmarketos.org/pmb-ci
if [ "$(id -u)" = 0 ]; then
set -x
apk -q add vermin
exec su "${TESTUSER:-build}" -c "sh -e $0"
fi
_vermin() {
if ! vermin -q "$@" >/dev/null 2>&1; then
vermin -vv "$@"
fi
}
# shellcheck disable=SC2046
vermin \
-t=3.7- \
_vermin \
-t=3.6- \
--backport argparse \
--backport configparser \
--backport enum \
--backport typing \
--lint \
--no-parse-comments \
--eval-annotations \
$(find . -name '*.py' \
-a -not -path "./.venv/*" \
-a -not -path "./venv/*")

66
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,66 @@
---
# Author: Clayton Craft <clayton@craftyguy.net>
image: alpine:latest
# defaults for "only"
# We need to run the CI jobs in a "merge request specific context", if CI is
# running in a merge request. Otherwise the environment variable that holds the
# merge request ID is not available. This means, we must set the "only"
# variable accordingly - and if we only do it for one job, all other jobs will
# not get executed. So have the defaults here, and use them in all jobs that
# should run on both the master branch, and in merge requests.
# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
.only-default: &only-default
only:
- master
- merge_requests
- tags
static-code-analysis:
<<: *only-default
before_script:
- .ci/prepare.sh
script:
- su pmos -c "test/static_code_analysis.sh"
vermin:
image: alpine:latest
<<: *only-default
before_script:
- "apk -q add py3-pip"
- "pip3 -q --disable-pip-version-check install vermin"
script:
- ".ci/vermin.sh"
# MR settings
# (Checks for "Allow commits from members who can merge to the target branch")
mr-settings:
only:
- merge_requests
before_script:
- apk -q add python3
script:
- wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
- python3 ./check_mr_settings.py
pytest:
<<: *only-default
before_script:
- .ci/prepare.sh
script:
- su pmos -c .ci/pytest.sh
after_script:
# Move logs so it can be saved as artifacts
- "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
- "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
- "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
- "dmesg > $CI_PROJECT_DIR/dmesg.txt"
artifacts:
when: always
paths:
- "log.txt"
- "log_testsuite.txt"
- "dmesg.txt"
- "pmbootstrap.cfg"
expire_in: 1 week

146
README.md
View File

@ -1,62 +1,22 @@
# pmbootstrap
[**Introduction**](https://postmarketos.org/blog/2017/05/26/intro/) | [**Security Warning**](https://ollieparanoid.github.io/post/security-warning/) | [**Devices**](https://wiki.postmarketos.org/wiki/Devices)
Sophisticated chroot/build/flash tool to develop and install
[postmarketOS](https://postmarketos.org).
Sophisticated chroot/build/flash tool to develop and install [postmarketOS](https://postmarketos.org).
## Development
pmbootstrap is being developed on SourceHut
([what](https://postmarketos.org/blog/2022/07/25/considering-sourcehut/)):
https://git.sr.ht/~postmarketos/pmbootstrap
Send patches via mail or web UI to
[pmbootstrap-devel](https://lists.sr.ht/~postmarketos/pmbootstrap-devel)
([subscribe](mailto:~postmarketos/pmbootstrap-devel+subscribe@lists.sr.ht)):
```
~postmarketos/pmbootstrap-devel@lists.sr.ht
```
You can set the default values for sending email in the git checkout
```
$ git config sendemail.to "~postmarketos/pmbootstrap-devel@lists.sr.ht"
$ git config format.subjectPrefix "PATCH pmbootstrap"
```
Run CI scripts locally with:
```
$ pmbootstrap ci
```
Run a single test file:
```
$ pytest -vv ./test/test_keys.py
```
## Issues
Issues are being tracked
[here](https://gitlab.com/postmarketOS/pmbootstrap/-/issues).
Package build scripts live in the [`pmaports`](https://gitlab.com/postmarketOS/pmaports) repository now.
## Requirements
* Linux distribution on the host system (`x86`, `x86_64`, `aarch64` or `armv7`)
* [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux)
does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
* 2 GB of RAM recommended for compiling
* Linux distribution on the host system (`x86`, `x86_64`, or `aarch64`)
* [Windows subsystem for Linux (WSL)](https://en.wikipedia.org/wiki/Windows_Subsystem_for_Linux) does **not** work! Please use [VirtualBox](https://www.virtualbox.org/) instead.
* Kernels based on the grsec patchset [do **not** work](https://github.com/postmarketOS/pmbootstrap/issues/107)
* [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
* Python 3.7+
* Python 3.6+
* OpenSSL
* git
* ps
* tar
## Usage Examples
Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for
in-depth coverage of topics such as
[porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device)
or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The
help output (`pmbootstrap -h`) has detailed usage instructions for every
command. Read on for some generic examples of what can be done with
`pmbootstrap`.
Please refer to the [postmarketOS wiki](https://wiki.postmarketos.org) for in-depth coverage of topics such as [porting to a new device](https://wiki.postmarketos.org/wiki/Porting_to_a_new_device) or [installation](https://wiki.postmarketos.org/wiki/Installation_guide). The help output (`pmbootstrap -h`) has detailed usage instructions for every command. Read on for some generic examples of what can be done with `pmbootstrap`.
### Installing pmbootstrap
<https://wiki.postmarketos.org/wiki/Installing_pmbootstrap>
@ -103,27 +63,6 @@ Generate a template for a new package:
$ pmbootstrap newapkbuild "https://gitlab.com/postmarketOS/osk-sdl/-/archive/0.52/osk-sdl-0.52.tar.bz2"
```
#### Default architecture
Packages will be compiled for the architecture of the device running
pmbootstrap by default. For example, if your `x86_64` PC runs pmbootstrap, it
would build a package for `x86_64` with this command:
```
$ pmbootstrap build hello-world
```
If you would rather build for the target device selected in `pmbootstrap init`
by default, then use the `build_default_device_arch` option:
```
$ pmbootstrap config build_default_device_arch True
```
If your target device is `pine64-pinephone` for example, pmbootstrap will now
build this package for `aarch64`:
```
$ pmbootstrap build hello-world
```
### Chroots
Enter the `armhf` building chroot:
```
@ -141,9 +80,7 @@ $ pmbootstrap zap
```
### Device Porting Assistance
Analyze Android
[`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also
works with recovery OS images like TWRP):
Analyze Android [`boot.img`](https://wiki.postmarketos.org/wiki/Glossary#boot.img) files (also works with recovery OS images like TWRP):
```
$ pmbootstrap bootimg_analyze ~/Downloads/twrp-3.2.1-0-fp2.img
```
@ -207,14 +144,12 @@ List pmaports that don't have a binary package:
$ pmbootstrap repo_missing --arch=armhf --overview
```
Increase the `pkgrel` for each aport where the binary package has outdated
dependencies (e.g. after soname bumps):
Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps):
```
$ pmbootstrap pkgrel_bump --auto
```
Generate cross-compiler aports based on the latest version from Alpine's
aports:
Generate cross-compiler aports based on the latest version from Alpine's aports:
```
$ pmbootstrap aportgen binutils-armhf gcc-armhf
```
@ -255,24 +190,27 @@ $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hel
$ pmbootstrap stats --arch=armhf
```
`distccd` log:
```
$ pmbootstrap log_distccd
```
### Use alternative sudo
pmbootstrap supports `doas` and `sudo`.
If multiple sudo implementations are installed, pmbootstrap will use `doas`.
You can set the `PMB_SUDO` environmental variable to define the sudo
implementation you want to use.
You can set the `PMB_SUDO` environmental variable to define the sudo implementation you want to use.
### Select SSH keys to include and make authorized in new images
If the config file option `ssh_keys` is set to `True` (it defaults to `False`),
then all files matching the glob `~/.ssh/id_*.pub` will be placed in
`~/.ssh/authorized_keys` in the user's home directory in newly-built images.
If the config file option `ssh_keys` is set to `True` (it defaults to `False`), then all files
matching the glob `~/.ssh/id_*.pub` will be placed in `~/.ssh/authorized_keys` in the user's
home directory in newly-built images.
Sometimes, for example if you have a large number of SSH keys, you may wish to
select a different set of public keys to include in an image. To do this, set
the `ssh_key_glob` configuration parameter in the pmbootstrap config file to a
string containing a glob that is to match the file or files you wish to
include.
Sometimes, for example if you have a large number of SSH keys, you may wish to select a
different set of public keys to include in an image. To do this, set the `ssh_key_glob`
configuration parameter in the pmbootstrap config file to a string containing a glob that is to
match the file or files you wish to include.
For example, a `~/.config/pmbootstrap.cfg` may contain:
@ -282,5 +220,39 @@ For example, a `~/.config/pmbootstrap.cfg` may contain:
ssh_key_glob = ~/.ssh/postmarketos-dev.pub
# ...
## Development
### Requirements for running tests
* [Shellcheck](https://shellcheck.net/)
You also need to install the following python packages (pip can be useful if you distribution hasn't got them packaged):
* `pytest`
* `pytest-cov`
* `flake8`
On Alpine Linux it can be done with:
```shell
$ sudo apk add grep shellcheck py3-pytest py3-pytest-cov py3-flake8
```
### Running linters
The easiest way is to run the same script CI runs:
```shell
$ ./test/static_code_analysis.sh
```
### Running tests
You can now run `pytest -vv` inside the pmbootstrap folder to run all available tests.
CI runs slightly reduces set of tests (it skips tests that require running qemu) by this:
```shell
$ .ci/pytest.sh
```
This is the easiest way to do the same as CI.
Alternatively you can run a single test file if you wish:
```shell
$ pytest -vv ./test/test_keys.py
```
## License
[GPLv3](LICENSE)

View File

@ -49,7 +49,7 @@ export_pmbootstrap_dir() {
# See also: <https://stackoverflow.com/a/29835459>
# shellcheck disable=SC3054
if [ -n "${BASH_SOURCE[0]}" ]; then
script_dir="$(dirname "$(realpath "$BASH_SOURCE")")"
script_dir="$(dirname "${BASH_SOURCE[0]}")"
else
script_dir="$(dirname "$1")"
fi
@ -121,8 +121,7 @@ initialize_chroot() {
arch_substr="${host_arch:0:3}"
if [ "$arch" = "$host_arch" ] || \
{ [ "$arch_substr" = "arm" ] && [ "$arch_substr" = "$arch" ]; } || \
{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; } || \
{ [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; }; then
{ [ "$arch" = "arm64" ] && [ "$host_arch" = "aarch64" ]; }; then
need_cross_compiler=0
fi
@ -169,9 +168,7 @@ initialize_chroot() {
musl-dev \
ncurses-dev \
perl \
py3-dt-schema \
sed \
yamllint \
yaml-dev \
xz || return 1
@ -219,10 +216,6 @@ set_alias_make() {
cross_compiler="/usr/bin/$prefix-"
fi
if [ "$arch" = "x86" ] && [ "$host_arch" = "x86_64" ]; then
cc=$hostcc
fi
# Build make command
cmd="echo '*** pmbootstrap envkernel.sh active for $PWD! ***';"
cmd="$cmd pmbootstrap -q chroot --user --"
@ -315,7 +308,7 @@ set_reactivate() {
check_and_deactivate() {
if [ "$POSTMARKETOS_ENVKERNEL_ENABLED" = 1 ]; then
# we already are running in envkernel
# we already are runnning in envkernel
deactivate
fi
}
@ -324,7 +317,9 @@ check_and_deactivate() {
print_usage() {
# shellcheck disable=SC3054
if [ -n "${BASH_SOURCE[0]}" ]; then
echo "usage: source $(basename "$(realpath "$BASH_SOURCE")")"
echo "usage: source $(basename "${BASH_SOURCE[0]}")"
else
echo "usage: source $(basename "$1")"
fi
echo "optional arguments:"
echo " --fish Print fish alias syntax (internally used)"
@ -410,7 +405,7 @@ main() {
# Print fish alias syntax (when called from envkernel.fish)
fish_compat() {
[ "$1" = "--fish" ] || return 0
[ "$1" = "--fish" ] || return
for name in make kernelroot pmbootstrap pmbroot; do
echo "alias $(alias $name | sed 's/=/ /')"
done

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
# PYTHON_ARGCOMPLETE_OK
import sys
@ -14,17 +14,6 @@ from .helpers import logging as pmb_logging
from .helpers import mount
from .helpers import other
# pmbootstrap version
__version__ = "2.0.0"
# Python version check
version = sys.version_info
if version < (3, 7):
print("You need at least Python 3.7 to run pmbootstrap")
print("(You are running it with Python " + str(version.major) +
"." + str(version.minor) + ")")
sys.exit()
def main():
# Wrap everything to display nice error messages
@ -34,9 +23,6 @@ def main():
args = parse.arguments()
os.umask(0o22)
# Store script invocation command
os.environ["PMBOOTSTRAP_CMD"] = sys.argv[0]
# Sanity checks
other.check_grsec()
if not args.as_root and os.geteuid() == 0:
@ -70,10 +56,6 @@ def main():
" shutdown' as necessary)")
logging.info("DONE!")
except KeyboardInterrupt:
print("\nCaught KeyboardInterrupt, exiting …")
sys.exit(130) # SIGINT(2) + 128
except Exception as e:
# Dump log to stdout when args (and therefore logging) init failed
if not args:
@ -89,14 +71,7 @@ def main():
log_hint += (" Alternatively you can use '--details-to-stdout' to"
" get more output, e.g. 'pmbootstrap"
" --details-to-stdout init'.")
print()
print(log_hint)
print()
print("Before you report this error, ensure that pmbootstrap is "
"up to date.")
print("Find the latest version here:"
" https://git.sr.ht/~postmarketos/pmbootstrap/refs")
print(f"Your version: {__version__}")
return 1

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.aportgen.core
import pmb.helpers.git
@ -13,14 +13,9 @@ def generate(args, pkgname):
# Rewrite APKBUILD
fields = {
"arch": pmb.config.arch_native,
"makedepends_host": "zlib-dev jansson-dev zstd-dev",
"pkgdesc": f"Tools necessary to build programs for {arch} targets",
"pkgname": pkgname,
}
replace_simple = {
"*--with-bugurl=*": "\t\t--with-bugurl=\"https://postmarketos.org/issues\" \\"
"pkgdesc": f"Tools necessary to build programs for {arch} targets",
"arch": pmb.config.arch_native,
}
below_header = """
@ -29,5 +24,4 @@ def generate(args, pkgname):
"""
pmb.aportgen.core.rewrite(args, pkgname, "main/binutils", fields,
"binutils", replace_simple=replace_simple,
below_header=below_header)
"binutils", below_header=below_header)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.aportgen.core
import pmb.build
@ -67,7 +67,7 @@ def generate(args, pkgname):
handle.write(line[12:].replace(" " * 4, "\t") + "\n")
# Generate checksums
pmb.build.init_abuild_minimal(args)
pmb.build.init(args)
pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import fnmatch
import logging
@ -205,18 +205,16 @@ def get_upstream_aport(args, pkgname, arch=None):
if compare == 0:
return aport_path
# APKBUILD > binary: this is fine
if compare == 1:
logging.info(f"NOTE: {pkgname} {arch} binary package has a lower"
f" version {package['version']} than the APKBUILD"
f" {apkbuild_version}")
return aport_path
# APKBUILD < binary: aports.git is outdated
logging.error("ERROR: Package '" + pkgname + "' has a lower version in"
# Different version message
logging.error("ERROR: Package '" + pkgname + "' has a different version in"
" local checkout of Alpine's aports (" + apkbuild_version +
") compared to Alpine's binary package (" +
package["version"] + ")!")
raise RuntimeError("You can update your local checkout with: "
"'pmbootstrap pull'")
# APKBUILD < binary
if compare == -1:
raise RuntimeError("You can update your local checkout with: "
"'pmbootstrap pull'")
# APKBUILD > binary
raise RuntimeError("You can force an update of your binary package"
" APKINDEX files with: 'pmbootstrap update'")

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -16,7 +16,7 @@ def ask_for_architecture():
architectures.remove("armhf")
while True:
ret = pmb.helpers.cli.ask("Device architecture", architectures,
"aarch64", complete=architectures)
architectures[0], complete=architectures)
if ret in architectures:
return ret
logging.fatal("ERROR: Invalid architecture specified. If you want to"
@ -73,8 +73,7 @@ def ask_for_flash_method():
logging.info("Which flash method does the device support?")
method = pmb.helpers.cli.ask("Flash method",
pmb.config.flash_methods,
"none",
complete=pmb.config.flash_methods)
pmb.config.flash_methods[0])
if method in pmb.config.flash_methods:
if method == "heimdall":
@ -179,6 +178,7 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
deviceinfo_codename="{codename}"
deviceinfo_year="{year}"
deviceinfo_dtb=""
deviceinfo_modules_initfs=""
deviceinfo_arch="{arch}"
# Device related
@ -194,13 +194,13 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
content_heimdall_bootimg = """\
deviceinfo_flash_heimdall_partition_kernel=""
deviceinfo_flash_heimdall_partition_rootfs=""
deviceinfo_flash_heimdall_partition_system=""
"""
content_heimdall_isorec = """\
deviceinfo_flash_heimdall_partition_kernel=""
deviceinfo_flash_heimdall_partition_initfs=""
deviceinfo_flash_heimdall_partition_rootfs=""
deviceinfo_flash_heimdall_partition_system=""
"""
content_0xffff = """\
@ -231,24 +231,6 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
handle.write(line.lstrip() + "\n")
def generate_modules_initfs(args):
content = """\
# Remove this comment after reading, or the file if unnecessary (CHANGEME!)
# This file can contain a list of modules to be included in the initramfs,
# so that they are available in early boot stages. It should have one
# module name per line. If there are multiple kernel variants with different
# requirements for modules into the initramfs, one modules-initfs.$variant
# file should be created for each of them.
"""
# Write to file
pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/aportgen"])
path = args.work + "/aportgen/modules-initfs"
with open(path, "w", encoding="utf-8") as handle:
for line in content.rstrip().split("\n"):
handle.write(line.lstrip() + "\n")
def generate_apkbuild(args, pkgname, name, arch, flash_method):
# Dependencies
depends = ["postmarketos-base",
@ -257,6 +239,7 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
depends.append("mkbootimg")
if flash_method == "0xffff":
depends.append("uboot-tools")
depends.append("mesa-dri-gallium")
# Whole APKBUILD
depends.sort()
@ -275,10 +258,7 @@ def generate_apkbuild(args, pkgname, name, arch, flash_method):
{depends}
"
makedepends="devicepkg-dev"
source="
deviceinfo
modules-initfs
"
source="deviceinfo"
build() {{
devicepkg_build $startdir $pkgname
@ -315,5 +295,4 @@ def generate(args, pkgname):
generate_deviceinfo(args, pkgname, name, manufacturer, year, arch,
chassis, has_keyboard, has_external_storage,
flash_method, bootimg)
generate_modules_initfs(args)
generate_apkbuild(args, pkgname, name, arch, flash_method)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.aportgen.core
import pmb.helpers.git

View File

@ -1,4 +1,4 @@
# Copyright 2023 Nick Reitemeyer, Oliver Smith
# Copyright 2022 Nick Reitemeyer, Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.aportgen.core
import pmb.build
@ -57,7 +57,7 @@ def generate(args, pkgname):
handle.write(line[12:].replace(" " * 4, "\t") + "\n")
# Generate checksums
pmb.build.init_abuild_minimal(args)
pmb.build.init(args)
pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.helpers.run
import pmb.aportgen.core
@ -10,7 +10,7 @@ def generate_apkbuild(args, pkgname, deviceinfo, patches):
device = "-".join(pkgname.split("-")[1:])
carch = pmb.parse.arch.alpine_to_kernel(deviceinfo["arch"])
makedepends = ["bash", "bc", "bison", "devicepkg-dev", "findutils", "flex",
makedepends = ["bash", "bc", "bison", "devicepkg-dev", "flex",
"openssl-dev", "perl"]
build = """

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.aportgen.core
import pmb.build
@ -93,7 +93,7 @@ def generate(args, pkgname):
handle.write(line[12:].replace(" " * 4, "\t") + "\n")
# Generate checksums
pmb.build.init_abuild_minimal(args)
pmb.build.init(args)
pmb.chroot.root(args, ["chown", "-R", "pmos:pmos", tempdir])
pmb.chroot.user(args, ["abuild", "checksum"], working_dir=tempdir)
pmb.helpers.run.user(args, ["cp", apkbuild_path, f"{args.work}/aportgen"])

View File

@ -1,8 +1,8 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.build.init import init, init_abuild_minimal, init_compiler
from pmb.build.init import init, init_compiler
from pmb.build.envkernel import package_kernel
from pmb.build.kconfig import menuconfig
from pmb.build.menuconfig import menuconfig
from pmb.build.newapkbuild import newapkbuild
from pmb.build.other import copy_to_buildpath, is_necessary, \
index_repo

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import datetime
import logging
@ -8,6 +8,7 @@ import pmb.build
import pmb.build.autodetect
import pmb.chroot
import pmb.chroot.apk
import pmb.chroot.distccd
import pmb.helpers.pmaports
import pmb.helpers.repo
import pmb.parse
@ -189,7 +190,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
just initialized the build environment for nothing) and then setup the
whole build environment (abuild, gcc, dependencies, cross-compiler).
:param cross: None, "native", or "crossdirect"
:param cross: None, "native", "distcc", or "crossdirect"
:param skip_init_buildenv: can be set to False to avoid initializing the
build environment. Use this when building
something during initialization of the build
@ -213,10 +214,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
if not skip_init_buildenv:
pmb.build.init(args, suffix)
pmb.build.other.configure_abuild(args, suffix)
if args.ccache:
pmb.build.other.configure_ccache(args, suffix)
if "rust" in depends or "cargo" in depends:
pmb.chroot.apk.install(args, ["sccache"], suffix)
pmb.build.other.configure_ccache(args, suffix)
if not strict and "pmb:strict" not in apkbuild["options"] and len(depends):
pmb.chroot.apk.install(args, depends, suffix)
if src:
@ -225,12 +223,35 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
# Cross-compiler init
if cross:
pmb.build.init_compiler(args, depends, cross, arch)
if cross == "distcc":
pmb.chroot.distccd.start(args, arch)
if cross == "crossdirect":
pmb.chroot.mount_native_into_foreign(args, suffix)
# Workaround: this specific version currently in pmaports.git master
# was built with !tracedeps, so it doesn't pull in the isl dependency
# and we need to install it manually. Doing this is easier than bumping
# the pkgrel and going out of sync with Alpine's gcc package. This
# workaround can be removed once a newer gcc is in Alpine and we
# rebuild our cross gcc based on the new APKBUILD. See pmaports#1732.
if get_gcc_version(args, arch) == "12.2.1_git20220924-r2":
pmb.chroot.apk.install(args, ["isl25"], build=False)
return True
def get_gcc_version(args, arch):
"""
Get the GCC version for a specific arch from parsing the right APKINDEX.
We feed this to ccache, so it knows the right GCC version, when
cross-compiling in a foreign arch chroot with distcc. See the "using
ccache with other compiler wrappers" section of their man page:
<https://linux.die.net/man/1/ccache>
:returns: a string like "6.4.0-r5"
"""
return pmb.parse.apkindex.package(args, "gcc-" + arch,
pmb.config.arch_native)["version"]
def get_pkgver(original_pkgver, original_source=False, now=None):
"""
Get the original pkgver when using the original source. Otherwise, get the
@ -263,7 +284,7 @@ def override_source(args, apkbuild, pkgver, src, suffix="native"):
return
# Mount source in chroot
mount_path = "/mnt/pmbootstrap/source-override/"
mount_path = "/mnt/pmbootstrap-source-override/"
mount_path_outside = args.work + "/chroot_" + suffix + mount_path
pmb.helpers.mount.bind(args, src, mount_path_outside, umount=True)
@ -368,7 +389,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
depending on the cross-compiler method and target architecture), copy
the aport to the chroot and execute abuild.
:param cross: None, "native", or "crossdirect"
:param cross: None, "native", "distcc", or "crossdirect"
:param src: override source used to build the package with a local folder
:returns: (output, cmd, env), output is the destination apk path relative
to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
@ -397,28 +418,24 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "distcc":
env["CCACHE_PREFIX"] = "distcc"
env["CCACHE_PATH"] = f"/usr/lib/arch-bin-masquerade/{arch}:/usr/bin"
env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd"
env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" +
args.port_distccd)
env["DISTCC_BACKOFF_PERIOD"] = "0"
if not args.distcc_fallback:
env["DISTCC_FALLBACK"] = "0"
if args.verbose:
env["DISTCC_VERBOSE"] = "1"
if cross == "crossdirect":
env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch,
pmb.config.chroot_path])
if not args.ccache:
env["CCACHE_DISABLE"] = "1"
# Use sccache without crossdirect (crossdirect uses it via rustc.sh)
if args.ccache and cross != "crossdirect":
env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
# Cache binary objects from go in this path (like ccache)
env["GOCACHE"] = "/home/pmos/.cache/go-build"
# Cache go modules (git repositories). Usually these should be bundled and
# it should not be required to download them at build time, in that case
# the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But
# e.g. when using --src they are not bundled, in that case it makes sense
# to point GOMODCACHE at pmbootstrap's work dir so the modules are only
# downloaded once.
if args.go_mod_cache:
env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
# Build the abuild command
cmd = ["abuild", "-D", "postmarketOS"]
if strict or "pmb:strict" in apkbuild["options"]:

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -40,7 +40,7 @@ def arch(args, pkgname):
:returns: arch string like "x86_64" or "armhf". Preferred order, depending
on what is supported by the APKBUILD:
* native arch
* device arch (this will be preferred instead if build_default_device_arch is true)
* device arch
* first arch in the APKBUILD
"""
aport = pmb.helpers.pmaports.find(args, pkgname)
@ -50,19 +50,14 @@ def arch(args, pkgname):
apkbuild = pmb.parse.apkbuild(f"{aport}/APKBUILD")
arches = apkbuild["arch"]
if ("noarch" in arches or
"all" in arches or
pmb.config.arch_native in arches):
return pmb.config.arch_native
if args.build_default_device_arch:
preferred_arch = args.deviceinfo["arch"]
preferred_arch_2nd = pmb.config.arch_native
else:
preferred_arch = pmb.config.arch_native
preferred_arch_2nd = args.deviceinfo["arch"]
if "noarch" in arches or "all" in arches or preferred_arch in arches:
return preferred_arch
if preferred_arch_2nd in arches:
return preferred_arch_2nd
arch_device = args.deviceinfo["arch"]
if arch_device in arches:
return arch_device
try:
return apkbuild["arch"][0]
@ -82,7 +77,7 @@ def suffix(apkbuild, arch):
def crosscompile(args, apkbuild, arch, suffix):
"""
:returns: None, "native", "crossdirect"
:returns: None, "native", "crossdirect" or "distcc"
"""
if not args.cross:
return None
@ -90,6 +85,6 @@ def crosscompile(args, apkbuild, arch, suffix):
return None
if suffix == "native":
return "native"
if "!pmb:crossdirect" in apkbuild["options"]:
return None
if args.no_crossdirect or "!pmb:crossdirect" in apkbuild["options"]:
return "distcc"
return "crossdirect"

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
@ -10,7 +10,7 @@ import pmb.helpers.pmaports
def update(args, pkgname):
""" Fetch all sources and update the checksums in the APKBUILD. """
pmb.build.init_abuild_minimal(args)
pmb.build.init(args)
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) generate checksums for " + pkgname)
pmb.chroot.user(args, ["abuild", "checksum"],
@ -24,7 +24,7 @@ def update(args, pkgname):
def verify(args, pkgname):
""" Fetch all sources and verify their checksums. """
pmb.build.init_abuild_minimal(args)
pmb.build.init(args)
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) verify checksums for " + pkgname)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Robert Yang
# Copyright 2022 Robert Yang
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -104,23 +104,6 @@ def modify_apkbuild(args, pkgname, aport):
pmb.aportgen.core.rewrite(args, pkgname, apkbuild_path, fields=fields)
def host_build_bindmount(args, chroot, flag_file, mount=False):
"""
Check if the bind mount already exists and unmount it.
Then bindmount the current directory into the chroot as
/mnt/linux so it can be used by the envkernel abuild wrapper
"""
flag_path = f"{chroot}/{flag_file}"
if os.path.exists(flag_path):
logging.info("Cleaning up kernel sources bind-mount")
pmb.helpers.run.root(args, ["umount", chroot + "/mnt/linux"], check=False)
pmb.helpers.run.root(args, ["rm", flag_path])
if mount:
pmb.helpers.mount.bind(args, ".", f"{chroot}/mnt/linux")
pmb.helpers.run.root(args, ["touch", flag_path])
def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
"""
Prepare build environment and run abuild.
@ -134,30 +117,10 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
build_path = "/home/pmos/build"
kbuild_out_source = "/mnt/linux/.output"
# If the kernel was cross-compiled on the host rather than with the envkernel
# helper, we can still use the envkernel logic to package the artifacts for
# development, making it easy to quickly sideload a new kernel or pmbootstrap
# to create a boot image
# This handles bind mounting the current directory (assumed to be kernel sources)
# into the chroot so we can run abuild against it for the currently selected
# devices kernel package.
flag_file = "envkernel-bind-mounted"
host_build = False
if not pmb.helpers.mount.ismount(chroot + "/mnt/linux"):
logging.info("envkernel.sh hasn't run, assuming the kernel was cross compiled"
"on host and using current dir as source")
host_build = True
host_build_bindmount(args, chroot, flag_file, mount=host_build)
if not os.path.exists(chroot + kbuild_out_source):
raise RuntimeError("No '.output' dir found in your kernel source dir. "
"Compile the " + args.device + " kernel first and "
"then try again. See https://postmarketos.org/envkernel"
"for details. If building on your host and only using "
"--envkernel for packaging, make sure you have O=.output "
"as an argument to make.")
raise RuntimeError("No '.output' dir found in your kernel source dir."
"Compile the " + args.device + " kernel with "
"envkernel.sh first, then try again.")
# Create working directory for abuild
pmb.build.copy_to_buildpath(args, pkgname)
@ -184,9 +147,6 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out):
cmd = ["abuild", "rootpkg"]
pmb.chroot.user(args, cmd, working_dir=build_path, env=env)
# Clean up bindmount if needed
host_build_bindmount(args, chroot, flag_file)
# Clean up symlinks
if build_output != "":
if os.path.islink(chroot + "/mnt/linux/" + build_output) and \
@ -223,8 +183,6 @@ def package_kernel(args):
depends, _ = pmb.build._package.build_depends(
args, apkbuild, pmb.config.arch_native, strict=False)
pmb.build.init(args, suffix)
if pmb.parse.arch.cpu_emulation_required(arch):
depends.append("binutils-" + arch)
pmb.chroot.apk.install(args, depends, suffix)
output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +

View File

@ -1,9 +1,8 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import logging
import os
import pathlib
import logging
import glob
import pmb.build
import pmb.config
@ -12,14 +11,15 @@ import pmb.chroot.apk
import pmb.helpers.run
def init_abuild_minimal(args, suffix="native"):
""" Initialize a minimal chroot with abuild where one can do
'abuild checksum'. """
marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_abuild_init_done"
if os.path.exists(marker):
def init(args, suffix="native"):
# Check if already initialized
marker = "/var/local/pmbootstrap_chroot_build_init_done"
if os.path.exists(args.work + "/chroot_" + suffix + marker):
return
pmb.chroot.apk.install(args, ["abuild"], suffix, build=False)
# Initialize chroot, install packages
pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
build=False)
# Fix permissions
pmb.chroot.root(args, ["chown", "root:abuild",
@ -27,24 +27,6 @@ def init_abuild_minimal(args, suffix="native"):
pmb.chroot.root(args, ["chmod", "g+w",
"/var/cache/distfiles"], suffix)
# Add user to group abuild
pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
pathlib.Path(marker).touch()
def init(args, suffix="native"):
""" Initialize a chroot for building packages with abuild. """
marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_build_init_done"
if os.path.exists(marker):
return
init_abuild_minimal(args, suffix)
# Initialize chroot, install packages
pmb.chroot.apk.install(args, pmb.config.build_packages, suffix,
build=False)
# Generate package signing keys
chroot = args.work + "/chroot_" + suffix
if not os.path.exists(args.work + "/config_abuild/abuild.conf"):
@ -54,7 +36,7 @@ def init(args, suffix="native"):
# Copy package signing key to /etc/apk/keys
for key in glob.glob(chroot +
"/mnt/pmbootstrap/abuild-config/*.pub"):
"/mnt/pmbootstrap-abuild-config/*.pub"):
key = key[len(chroot):]
pmb.chroot.root(args, ["cp", key, "/etc/apk/keys/"], suffix)
@ -81,6 +63,9 @@ def init(args, suffix="native"):
"/usr/local/bin/gzip"], suffix)
pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix)
# Add user to group abuild
pmb.chroot.root(args, ["adduser", "pmos", "abuild"], suffix)
# abuild.conf: Don't clean the build folder after building, so we can
# inspect it afterwards for debugging
pmb.chroot.root(args, ["sed", "-i", "-e", "s/^CLEANUP=.*/CLEANUP=''/",
@ -92,7 +77,8 @@ def init(args, suffix="native"):
"s/^ERROR_CLEANUP=.*/ERROR_CLEANUP=''/",
"/etc/abuild.conf"], suffix)
pathlib.Path(marker).touch()
# Mark the chroot as initialized
pmb.chroot.root(args, ["touch", marker], suffix)
def init_compiler(args, depends, cross, arch):
@ -108,11 +94,6 @@ def init_compiler(args, depends, cross, arch):
if cross == "crossdirect":
cross_pkgs += ["crossdirect"]
if "rust" in depends or "cargo" in depends:
if args.ccache:
cross_pkgs += ["sccache"]
# crossdirect for rust installs all build dependencies in the
# native chroot too, as some of them can be required for building
# native macros / build scripts
cross_pkgs += depends
cross_pkgs += ["rust"]
pmb.chroot.apk.install(args, cross_pkgs)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging
@ -78,16 +78,6 @@ def get_outputdir(args, pkgname, apkbuild):
" template with: pmbootstrap aportgen " + pkgname)
def extract_and_patch_sources(args, pkgname, arch):
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) extract kernel source")
pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
logging.info("(native) apply patches")
pmb.chroot.user(args, ["abuild", "prepare"], "native",
"/home/pmos/build", output="interactive",
env={"CARCH": arch})
def menuconfig(args, pkgname, use_oldconfig):
# Pkgname: allow omitting "linux-" prefix
if not pkgname.startswith("linux-"):
@ -129,10 +119,14 @@ def menuconfig(args, pkgname, use_oldconfig):
if copy_xauth:
pmb.chroot.other.copy_xauthority(args)
extract_and_patch_sources(args, pkgname, arch)
# Check for background color variable
color = os.environ.get("MENUCONFIG_COLOR")
# Patch and extract sources
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) extract kernel source")
pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/pmos/build")
logging.info("(native) apply patches")
pmb.chroot.user(args, ["abuild", "prepare"], "native",
"/home/pmos/build", output="interactive",
env={"CARCH": arch})
# Run make menuconfig
outputdir = get_outputdir(args, pkgname, apkbuild)
@ -143,8 +137,6 @@ def menuconfig(args, pkgname, use_oldconfig):
if cross:
env["CROSS_COMPILE"] = f"{hostspec}-"
env["CC"] = f"{hostspec}-gcc"
if color:
env["MENUCONFIG_COLOR"] = color
pmb.chroot.user(args, ["make", kopt], "native",
outputdir, output="tui", env=env)
@ -161,4 +153,12 @@ def menuconfig(args, pkgname, use_oldconfig):
pmb.build.checksum.update(args, pkgname)
# Check config
pmb.parse.kconfig.check(args, apkbuild["_flavor"], details=True)
pmb.parse.kconfig.check(args, apkbuild["_flavor"],
force_waydroid_check=False,
force_nftables_check=False,
force_containers_check=False,
force_zram_check=False,
force_netboot_check=False,
force_community_check=False,
force_uefi_check=False,
details=True)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import logging
@ -28,16 +28,7 @@ def copy_to_buildpath(args, package, suffix="native"):
pmb.chroot.root(args, ["rm", "-rf", "/home/pmos/build"], suffix)
# Copy aport contents with resolved symlinks
pmb.helpers.run.root(args, ["mkdir", "-p", build])
for entry in os.listdir(aport):
# Don't copy those dirs, as those have probably been generated by running `abuild`
# on the host system directly and not cleaning up after itself.
# Those dirs might contain broken symlinks and cp fails resolving them.
if entry in ["src", "pkg"]:
logging.warn(f"WARNING: Not copying {entry}, looks like a leftover from abuild")
continue
pmb.helpers.run.root(args, ["cp", "-rL", f"{aport}/{entry}", f"{build}/{entry}"])
pmb.helpers.run.root(args, ["cp", "-rL", aport + "/", build])
pmb.chroot.root(args, ["chown", "-R", "pmos:pmos",
"/home/pmos/build"], suffix)
@ -152,7 +143,7 @@ def configure_abuild(args, suffix, verify=False):
suffix)
configure_abuild(args, suffix, True)
return
pmb.chroot.root(args, ["sed", "-i", f"$ a\\{prefix}{args.jobs}", "/etc/abuild.conf"], suffix)
raise RuntimeError("Could not find " + prefix + " line in " + path)
def configure_ccache(args, suffix="native", verify=False):

View File

@ -1,7 +1,7 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.chroot.init import init, init_keys
from pmb.chroot.mount import mount, mount_native_into_foreign, remove_mnt_pmbootstrap
from pmb.chroot.mount import mount, mount_native_into_foreign
from pmb.chroot.root import root
from pmb.chroot.user import user
from pmb.chroot.user import exists as user_exists

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging
@ -143,7 +143,7 @@ def packages_get_locally_built_apks(args, packages, arch):
:param packages: list of pkgnames
:param arch: architecture that the locally built packages should have
:returns: list of apk file paths that are valid inside the chroots, e.g.
["/mnt/pmbootstrap/packages/x86_64/hello-world-1-r6.apk", ...]
["/mnt/pmbootstrap-packages/x86_64/hello-world-1-r6.apk", ...]
"""
channel = pmb.config.pmaports.read_config(args)["channel"]
ret = []
@ -157,7 +157,7 @@ def packages_get_locally_built_apks(args, packages, arch):
if not os.path.exists(f"{args.work}/packages/{channel}/{arch}/{apk_file}"):
continue
ret.append(f"/mnt/pmbootstrap/packages/{arch}/{apk_file}")
ret.append(f"/mnt/pmbootstrap-packages/{arch}/{apk_file}")
return ret

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

250
pmb/chroot/distccd.py Normal file
View File

@ -0,0 +1,250 @@
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import errno
import json
import logging
import os
import pmb.chroot
import pmb.config
import pmb.chroot.apk
""" Packages for foreign architectures (e.g. armhf) get built in chroots
running with QEMU. While this works, it is painfully slow. So we speed it
up by using distcc to let cross compilers running in the native chroots do
the heavy lifting.
This file sets up an SSH server in the native chroot, which will then be
used by the foreign arch chroot to communicate with the distcc daemon. We
make sure that only the foreign arch chroot can connect to the sshd by only
listening on localhost, as well as generating dedicated ssh keys.
Using the SSH server instead of running distccd directly is a security
measure. Distccd does not authenticate its clients and would therefore
allow any process of the host system (not related to pmbootstrap) to
execute compilers in the native chroot. By modifying the compiler's options
or sending malicious data to the compiler, it is likely that the process
can gain remote code execution [1]. That way, a compromised, but sandboxed
process could gain privilege escalation.
[1]: <https://github.com/distcc/distcc/issues/155#issuecomment-374014645>
"""
def init_server(args):
"""
Install dependencies and generate keys for the server.
"""
# Install dependencies
pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
"openssh-server"])
# Config folder (nothing to do if existing)
dir = "/home/pmos/.distcc-sshd"
dir_outside = args.work + "/chroot_native" + dir
if os.path.exists(dir_outside):
return
# Generate keys
logging.info("(native) generate distcc-sshd server keys")
pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"])
pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir])
def init_client(args, suffix):
"""
Install dependencies and generate keys for the client.
"""
# Install dependencies
pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc",
"openssh-client"], suffix)
# Public key path (nothing to do if existing)
pub = "/home/pmos/id_ed25519.pub"
pub_outside = args.work + "/chroot_" + suffix + pub
if os.path.exists(pub_outside):
return
# Generate keys
logging.info("(" + suffix + ") generate distcc-sshd client keys")
pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "",
"-f", "/home/pmos/.ssh/id_ed25519"], suffix)
pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub],
suffix)
def configure_authorized_keys(args, suffix):
"""
Exclusively allow one foreign arch chroot to access the sshd.
"""
auth = "/home/pmos/.distcc-sshd/authorized_keys"
auth_outside = args.work + "/chroot_native/" + auth
pub = "/home/pmos/id_ed25519.pub"
pub_outside = args.work + "/chroot_" + suffix + pub
pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside])
def configure_cmdlist(args, arch):
"""
Create a whitelist of all the cross compiler wrappers.
Distcc 3.3 and above requires such a whitelist, or else it will only run
with the --make-me-a-botnet parameter (even in ssh mode).
"""
dir = "/home/pmos/.distcc-sshd"
with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle:
for cmd in ["c++", "cc", "cpp", "g++", "gcc"]:
cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd
handle.write(cmd_full + "\n")
pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"])
pmb.chroot.user(args, ["cat", dir + "/cmdlist"])
def configure_distccd_wrapper(args):
"""
Wrap distccd in a shell script, so we can pass the compiler whitelist and
set the verbose flag (when pmbootstrap is running with --verbose).
"""
dir = "/home/pmos/.distcc-sshd"
with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle:
handle.write("#!/bin/sh\n"
"export DISTCC_CMDLIST='" + dir + "/cmdlist'\n"
"distccd --log-file /home/pmos/distccd.log --nice 19")
if args.verbose:
handle.write(" --verbose")
handle.write(" \"$@\"\n")
pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"])
pmb.chroot.user(args, ["cat", dir + "/distccd"])
pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"])
def configure_sshd(args):
"""
Configure the SSH daemon in the native chroot.
"""
dir = "/home/pmos/.distcc-sshd"
config = """AllowAgentForwarding no
AllowTcpForwarding no
AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys
HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key
ListenAddress 127.0.0.1
PasswordAuthentication no
PidFile /home/pmos/.distcc-sshd/sshd.pid
Port """ + args.port_distccd + """
X11Forwarding no"""
with open(args.work + "/chroot_native/tmp/cfg", "w") as handle:
for line in config.split("\n"):
handle.write(line.lstrip() + "\n")
pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"])
pmb.chroot.user(args, ["cat", dir + "/sshd_config"])
def get_running_pid(args):
"""
:returns: the running distcc-sshd's pid as integer or None
"""
# PID file must exist
pidfile = "/home/pmos/.distcc-sshd/sshd.pid"
pidfile_outside = args.work + "/chroot_native" + pidfile
if not os.path.exists(pidfile_outside):
return None
# Verify, if it still exists by sending a kill signal
with open(pidfile_outside, "r") as handle:
pid = int(handle.read()[:-1])
try:
os.kill(pid, 0)
except OSError as err:
if err.errno == errno.ESRCH: # no such process
pmb.helpers.run.root(args, ["rm", pidfile_outside])
return None
return pid
def get_running_parameters(args):
"""
Get the parameters of the currently running distcc-sshd instance.
:returns: a dictionary in the form of
{"arch": "armhf", "port": 1234, "verbose": False}
If the information can not be read, "arch" is set to "unknown"
"""
# Return defaults
path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
if not os.path.exists(path):
return {"arch": "unknown", "port": 0, "verbose": False}
# Parse the file as JSON
with open(path, "r") as handle:
return json.loads(handle.read())
def set_running_parameters(args, arch):
"""
Set the parameters of the currently running distcc-sshd instance.
"""
parameters = {"arch": arch,
"port": args.port_distccd,
"verbose": args.verbose}
path = args.work + "/chroot_native/tmp/distcc_sshd_parameters"
with open(path, "w") as handle:
json.dump(parameters, handle)
def is_running_with_same_parameters(args, arch):
"""
Check whether we can use the already running distcc-sshd instance with our
current set of parameters. In case we can use it directly, we save some
time, otherwise we need to stop it, configure it again, and start it once
more.
"""
if not get_running_pid(args):
return False
parameters = get_running_parameters(args)
return (parameters["arch"] == arch and
parameters["port"] == args.port_distccd and
parameters["verbose"] == args.verbose)
def stop(args):
"""
Kill the sshd process (by using its pid).
"""
pid = get_running_pid(args)
if not pid:
return
parameters = get_running_parameters(args)
logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")")
pmb.chroot.user(args, ["kill", str(pid)])
def start(args, arch):
"""
Set up a new distcc-sshd instance or use an already running one.
"""
if is_running_with_same_parameters(args, arch):
return
stop(args)
# Initialize server and client
suffix = "buildroot_" + arch
init_server(args)
init_client(args, suffix)
logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" +
args.port_distccd)
# Configure server parameters (arch, port, verbose)
configure_authorized_keys(args, suffix)
configure_distccd_wrapper(args)
configure_cmdlist(args, arch)
configure_sshd(args)
# Run
dir = "/home/pmos/.distcc-sshd"
pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config",
"-E", dir + "/log.txt"])
set_running_parameters(args, arch)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import glob

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import logging
@ -106,18 +106,3 @@ def mount_native_into_foreign(args, suffix):
if not os.path.lexists(musl_link):
pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,
musl_link])
def remove_mnt_pmbootstrap(args, suffix):
""" Safely remove /mnt/pmbootstrap directories from the chroot, without
running rm -r as root and potentially removing data inside the
mountpoint in case it was still mounted (bug in pmbootstrap, or user
ran pmbootstrap 2x in parallel). This is similar to running 'rm -r -d',
but we don't assume that the host's rm has the -d flag (busybox does
not). """
mnt_dir = f"{args.work}/chroot_{suffix}/mnt/pmbootstrap"
if not os.path.exists(mnt_dir):
return
for path in glob.glob(f"{mnt_dir}/*") + [mnt_dir]:
pmb.helpers.run.root(args, ["rmdir", path])

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import glob

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import shutil
@ -65,17 +65,21 @@ def root(args, cmd, suffix="native", working_dir="/", output="log",
for key, value in env.items():
env_all[key] = value
# Preserve proxy environment variables
for var in ["FTP_PROXY", "ftp_proxy", "HTTP_PROXY", "http_proxy",
"HTTPS_PROXY", "https_proxy", "HTTP_PROXY_AUTH"]:
if var in os.environ:
env_all[var] = os.environ[var]
# Build the command in steps and run it, e.g.:
# cmd: ["echo", "test"]
# cmd_chroot: ["/sbin/chroot", "/..._native", "/bin/sh", "-c", "echo test"]
# cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
executables = executables_absolute_path()
cmd_chroot = [executables["chroot"], chroot, "/bin/sh", "-c",
pmb.helpers.run_core.flat_cmd(cmd, working_dir)]
cmd_sudo = pmb.config.sudo([
"env", "-i", executables["sh"], "-c",
pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)]
)
pmb.helpers.run.flat_cmd(cmd, working_dir)]
cmd_sudo = [pmb.config.sudo, "env", "-i", executables["sh"], "-c",
pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)]
return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output,
output_return, check, True,
disable_timeout)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import glob
@ -7,6 +7,7 @@ import socket
from contextlib import closing
import pmb.chroot
import pmb.chroot.distccd
import pmb.helpers.mount
import pmb.install.losetup
import pmb.parse.arch
@ -22,17 +23,6 @@ def kill_adb(args):
pmb.chroot.root(args, ["adb", "-P", str(port), "kill-server"])
def kill_sccache(args):
"""
Kill sccache daemon if it's running. Unlike ccache it automatically spawns
a daemon when you call it and exits after some time of inactivity.
"""
port = 4226
with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
if sock.connect_ex(("127.0.0.1", port)) == 0:
pmb.chroot.root(args, ["sccache", "--stop-server"])
def shutdown_cryptsetup_device(args, name):
"""
:param name: cryptsetup device name, usually "pm_crypt" in pmbootstrap
@ -59,9 +49,10 @@ def shutdown_cryptsetup_device(args, name):
def shutdown(args, only_install_related=False):
# Stop daemons
pmb.chroot.distccd.stop(args)
# Stop adb server
kill_adb(args)
kill_sccache(args)
# Umount installation-related paths (order is important!)
pmb.helpers.mount.umount_all(args, args.work +

View File

@ -1,8 +1,7 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.chroot.root
import pmb.helpers.run
import pmb.helpers.run_core
def user(args, cmd, suffix="native", working_dir="/", output="log",
@ -22,7 +21,7 @@ def user(args, cmd, suffix="native", working_dir="/", output="log",
if "HOME" not in env:
env["HOME"] = "/home/pmos"
flat_cmd = pmb.helpers.run_core.flat_cmd(cmd, env=env)
flat_cmd = pmb.helpers.run.flat_cmd(cmd, env=env)
cmd = ["busybox", "su", "pmos", "-c", flat_cmd]
return pmb.chroot.root(args, cmd, suffix, working_dir, output,
output_return, check, {}, auto_init)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import logging
@ -17,7 +17,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False,
rust=False, netboot=False):
"""
Shutdown everything inside the chroots (e.g. adb), umount
Shutdown everything inside the chroots (e.g. distccd, adb), umount
everything and then safely remove folders from the work-directory.
:param dry: Only show what would be deleted, do not delete for real

View File

@ -1,169 +0,0 @@
# Copyright 2023 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import collections
import glob
import logging
import os
import shlex
import pmb.chroot
import pmb.helpers.cli
def get_ci_scripts(topdir):
""" Find 'pmbootstrap ci'-compatible scripts inside a git repository, and
parse their metadata (description, options). The reference is at:
https://postmarketos.org/pmb-ci
:param topdir: top directory of the git repository, get it with:
pmb.helpers.git.get_topdir()
:returns: a dict of CI scripts found in the git repository, e.g.
{"ruff": {"description": "lint all python scripts",
"options": []},
...} """
ret = {}
for script in glob.glob(f"{topdir}/.ci/*.sh"):
is_pmb_ci_script = False
description = ""
options = []
with open(script) as handle:
for line in handle:
if line.startswith("# https://postmarketos.org/pmb-ci"):
is_pmb_ci_script = True
elif line.startswith("# Description: "):
description = line.split(": ", 1)[1].rstrip()
elif line.startswith("# Options: "):
options = line.split(": ", 1)[1].rstrip().split(" ")
elif not line.startswith("#"):
# Stop parsing after the block of comments on top
break
if not is_pmb_ci_script:
continue
if not description:
logging.error(f"ERROR: {script}: missing '# Description: …' line")
exit(1)
for option in options:
if option not in pmb.config.ci_valid_options:
raise RuntimeError(f"{script}: unsupported option '{option}'."
" Typo in script or pmbootstrap too old?")
short_name = os.path.basename(script).split(".", -1)[0]
ret[short_name] = {"description": description,
"options": options}
return ret
def sort_scripts_by_speed(scripts):
""" Order the scripts, so fast scripts run before slow scripts. Whether a
script is fast or not is determined by the '# Options: slow' comment in
the file.
:param scripts: return of get_ci_scripts()
:returns: same format as get_ci_scripts(), but as ordered dict with
fast scripts before slow scripts """
ret = collections.OrderedDict()
# Fast scripts first
for script_name, script in scripts.items():
if "slow" in script["options"]:
continue
ret[script_name] = script
# Then slow scripts
for script_name, script in scripts.items():
if "slow" not in script["options"]:
continue
ret[script_name] = script
return ret
def ask_which_scripts_to_run(scripts_available):
""" Display an interactive prompt about which of the scripts the user
wishes to run, or all of them.
:param scripts_available: same format as get_ci_scripts()
:returns: either full scripts_available (all selected), or a subset """
count = len(scripts_available.items())
choices = ["all"]
logging.info(f"Available CI scripts ({count}):")
for script_name, script in scripts_available.items():
extra = ""
if "slow" in script["options"]:
extra += " (slow)"
logging.info(f"* {script_name}: {script['description']}{extra}")
choices += [script_name]
selection = pmb.helpers.cli.ask("Which script?", None, "all",
complete=choices)
if selection == "all":
return scripts_available
ret = {}
ret[selection] = scripts_available[selection]
return ret
def copy_git_repo_to_chroot(args, topdir):
""" Create a tarball of the git repo (including unstaged changes and new
files) and extract it in chroot_native.
:param topdir: top directory of the git repository, get it with:
pmb.helpers.git.get_topdir() """
pmb.chroot.init(args)
tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz"
files = pmb.helpers.git.get_files(args, topdir)
with open(f"{tarball_path}.files", "w") as handle:
for file in files:
handle.write(file)
handle.write("\n")
pmb.helpers.run.user(args, ["tar", "-cf", tarball_path, "-T",
f"{tarball_path}.files"], topdir)
ci_dir = "/home/pmos/ci"
pmb.chroot.user(args, ["rm", "-rf", ci_dir])
pmb.chroot.user(args, ["mkdir", ci_dir])
pmb.chroot.user(args, ["tar", "-xf", "/tmp/git.tar.gz"],
working_dir=ci_dir)
def run_scripts(args, topdir, scripts):
""" Run one of the given scripts after another, either natively or in a
chroot. Display a progress message and stop on error (without printing
a python stack trace).
:param topdir: top directory of the git repository, get it with:
pmb.helpers.git.get_topdir()
:param scripts: return of get_ci_scripts() """
steps = len(scripts)
step = 0
repo_copied = False
for script_name, script in scripts.items():
step += 1
where = "pmbootstrap chroot"
if "native" in script["options"]:
where = "native"
script_path = f".ci/{script_name}.sh"
logging.info(f"*** ({step}/{steps}) RUNNING CI SCRIPT: {script_path}"
f" [{where}] ***")
if "native" in script["options"]:
rc = pmb.helpers.run.user(args, [script_path], topdir,
output="tui")
continue
else:
# Run inside pmbootstrap chroot
if not repo_copied:
copy_git_repo_to_chroot(args, topdir)
repo_copied = True
env = {"TESTUSER": "pmos"}
rc = pmb.chroot.root(args, [script_path], check=False, env=env,
working_dir="/home/pmos/ci",
output="tui")
if rc:
logging.error(f"ERROR: CI script failed: {script_name}")
exit(1)

View File

@ -1,10 +1,9 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import multiprocessing
import os
import pmb.parse.arch
import sys
from typing import List
#
# Exported functions
@ -18,6 +17,7 @@ from pmb.config.sudo import which_sudo
#
# Exported variables (internal configuration)
#
version = "1.47.1"
pmb_src = os.path.normpath(os.path.realpath(__file__) + "/../../..")
apk_keys_path = pmb_src + "/pmb/data/keys"
arch_native = pmb.parse.arch.alpine_native()
@ -27,9 +27,7 @@ arch_native = pmb.parse.arch.alpine_native()
# Update this frequently to prevent a MITM attack with an outdated version
# (which may contain a vulnerable apk/openssl, and allows an attacker to
# exploit the system!)
apk_tools_min_version = {"edge": "2.14.0-r5",
"v3.18": "2.14.0-r2",
"v3.17": "2.12.10-r1",
apk_tools_min_version = {"edge": "2.12.9-r7",
"v3.16": "2.12.9-r3",
"v3.15": "2.12.7-r3",
"v3.14": "2.12.7-r0",
@ -53,53 +51,36 @@ ondev_min_version = "0.2.0"
# Programs that pmbootstrap expects to be available from the host system. Keep
# in sync with README.md, and try to keep the list as small as possible. The
# idea is to run almost everything in Alpine chroots.
required_programs = [
"git",
"openssl",
"ps",
"tar",
]
def sudo(cmd: List[str]) -> List[str]:
"""Adapt a command to run as root."""
sudo = which_sudo()
if sudo:
return [sudo, *cmd]
else:
return cmd
required_programs = ["git", "openssl", "ps"]
sudo = which_sudo()
# Keys saved in the config file (mostly what we ask in 'pmbootstrap init')
config_keys = [
"aports",
"boot_size",
"build_default_device_arch",
"build_pkgs_on_install",
"ccache_size",
"device",
"extra_packages",
"extra_space",
"hostname",
"is_default_channel",
"jobs",
"kernel",
"keymap",
"locale",
"mirror_alpine",
"mirrors_postmarketos",
"nonfree_firmware",
"nonfree_userland",
"qemu_redir_stdio",
"ssh_key_glob",
"ssh_keys",
"sudo_timer",
"timezone",
"ui",
"ui_extras",
"user",
"work",
]
config_keys = ["aports",
"ccache_size",
"device",
"extra_packages",
"hostname",
"build_pkgs_on_install",
"is_default_channel",
"jobs",
"kernel",
"keymap",
"locale",
"mirror_alpine",
"mirrors_postmarketos",
"nonfree_firmware",
"nonfree_userland",
"ssh_keys",
"ssh_key_glob",
"timezone",
"ui",
"ui_extras",
"user",
"work",
"boot_size",
"extra_space",
"sudo_timer",
"qemu_redir_stdio"]
# Config file/commandline default values
# $WORK gets replaced with the actual value for args.work (which may be
@ -123,7 +104,7 @@ defaults = {
"jobs": str(multiprocessing.cpu_count() + 1),
"kernel": "stable",
"keymap": "",
"locale": "en_US.UTF-8",
"locale": "C.UTF-8",
"log": "$WORK/log.txt",
"mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/",
# NOTE: mirrors_postmarketos variable type is supposed to be
@ -131,18 +112,18 @@ defaults = {
"mirrors_postmarketos": "http://mirror.postmarketos.org/postmarketos/",
"nonfree_firmware": True,
"nonfree_userland": False,
"port_distccd": "33632",
"ssh_keys": False,
"ssh_key_glob": "~/.ssh/id_*.pub",
"timezone": "GMT",
"ui": "console",
"ui": "weston",
"ui_extras": False,
"user": "user",
"work": os.path.expanduser("~") + "/.local/var/pmbootstrap",
"boot_size": "256",
"extra_space": "0",
"sudo_timer": False,
"qemu_redir_stdio": False,
"build_default_device_arch": False,
"qemu_redir_stdio": False
}
@ -167,6 +148,26 @@ if "NO_COLOR" in os.environ:
for style in styles.keys():
styles[style] = ""
# List of available locales taken from musl-locales package; see
# https://pkgs.alpinelinux.org/contents?name=musl-locales
locales = [
"C.UTF-8",
"ch_DE.UTF-8",
"de_CH.UTF-8",
"de_DE.UTF-8",
"en_GB.UTF-8",
"en_US.UTF-8",
"es_ES.UTF-8",
"fr_FR.UTF-8",
"it_IT.UTF-8",
"nb_NO.UTF-8",
"nl_NL.UTF-8",
"pt_BR.UTF-8",
"ru_RU.UTF-8",
"sv_SE.UTF-8"
]
# Supported filesystems and their fstools packages
filesystems = {"btrfs": "btrfs-progs",
"ext2": "e2fsprogs",
@ -206,21 +207,17 @@ chroot_host_path = os.environ["PATH"] + ":/usr/sbin/"
# $WORK gets replaced with args.work
# $ARCH gets replaced with the chroot architecture (eg. x86_64, armhf)
# $CHANNEL gets replaced with the release channel (e.g. edge, v21.03)
# Use no more than one dir after /mnt/pmbootstrap, see remove_mnt_pmbootstrap.
chroot_mount_bind = {
"/proc": "/proc",
"$WORK/cache_apk_$ARCH": "/var/cache/apk",
"$WORK/cache_appstream/$ARCH/$CHANNEL": "/mnt/appstream-data",
"$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap/ccache",
"$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap-ccache",
"$WORK/cache_distfiles": "/var/cache/distfiles",
"$WORK/cache_git": "/mnt/pmbootstrap/git",
"$WORK/cache_go": "/mnt/pmbootstrap/go",
"$WORK/cache_rust": "/mnt/pmbootstrap/rust",
"$WORK/config_abuild": "/mnt/pmbootstrap/abuild-config",
"$WORK/cache_git": "/mnt/pmbootstrap-git",
"$WORK/cache_rust": "/mnt/pmbootstrap-rust",
"$WORK/config_abuild": "/mnt/pmbootstrap-abuild-config",
"$WORK/config_apk_keys": "/etc/apk/keys",
"$WORK/cache_sccache": "/mnt/pmbootstrap/sccache",
"$WORK/images_netboot": "/mnt/pmbootstrap/netboot",
"$WORK/packages/$CHANNEL": "/mnt/pmbootstrap/packages",
"$WORK/images_netboot": "/mnt/pmbootstrap-netboot",
"$WORK/packages/$CHANNEL": "/mnt/pmbootstrap-packages",
}
# Building chroots (all chroots, except for the rootfs_ chroot) get symlinks in
@ -231,20 +228,13 @@ chroot_mount_bind = {
# a no-go, but at least until this is resolved properly, let's cache the
# dependencies and downloads as suggested in "Caching the Cargo home in CI":
# https://doc.rust-lang.org/cargo/guide/cargo-home.html
# Go: cache the directories "go env GOMODCACHE" and "go env GOCACHE" point to,
# to avoid downloading dependencies over and over (GOMODCACHE, similar to the
# rust depends caching described above) and to cache build artifacts (GOCACHE,
# similar to ccache).
chroot_home_symlinks = {
"/mnt/pmbootstrap/abuild-config": "/home/pmos/.abuild",
"/mnt/pmbootstrap/ccache": "/home/pmos/.ccache",
"/mnt/pmbootstrap/go/gocache": "/home/pmos/.cache/go-build",
"/mnt/pmbootstrap/go/gomodcache": "/home/pmos/go/pkg/mod",
"/mnt/pmbootstrap/packages": "/home/pmos/packages/pmos",
"/mnt/pmbootstrap/rust/git/db": "/home/pmos/.cargo/git/db",
"/mnt/pmbootstrap/rust/registry/cache": "/home/pmos/.cargo/registry/cache",
"/mnt/pmbootstrap/rust/registry/index": "/home/pmos/.cargo/registry/index",
"/mnt/pmbootstrap/sccache": "/home/pmos/.cache/sccache",
"/mnt/pmbootstrap-abuild-config": "/home/pmos/.abuild",
"/mnt/pmbootstrap-ccache": "/home/pmos/.ccache",
"/mnt/pmbootstrap-packages": "/home/pmos/packages/pmos",
"/mnt/pmbootstrap-rust/registry/index": "/home/pmos/.cargo/registry/index",
"/mnt/pmbootstrap-rust/registry/cache": "/home/pmos/.cargo/registry/cache",
"/mnt/pmbootstrap-rust/git/db": "/home/pmos/.cargo/git/db",
}
# Device nodes to be created in each chroot. Syntax for each entry:
@ -272,7 +262,7 @@ chroot_outdated = 3600 * 24 * 2
# specify architectures supported by Alpine here. For cross-compiling,
# we need to generate the "musl-$ARCH", "binutils-$ARCH" and "gcc-$ARCH"
# packages (use "pmbootstrap aportgen musl-armhf" etc.).
build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86", "riscv64"]
build_device_architectures = ["armhf", "armv7", "aarch64", "x86_64", "x86"]
# Packages that will be installed in a chroot before it builds packages
# for the first time
@ -294,7 +284,7 @@ build_packages = ["abuild", "build-base", "ccache", "git"]
# this e.g. if the order of the elements is important.
# Necessary kernel config options
kconfig_options = {
necessary_kconfig_options = {
">=0.0.0": { # all versions
"all": { # all arches
"ANDROID_PARANOID_NETWORK": False,
@ -315,16 +305,6 @@ kconfig_options = {
"VT": True,
}
},
">=2.6.0": {
"all": {
"BINFMT_ELF": True,
},
},
">=3.10.0": {
"all": {
"BINFMT_SCRIPT": True,
},
},
">=4.0.0": {
"all": {
"UEVENT_HELPER": True,
@ -350,31 +330,26 @@ kconfig_options = {
}
# Necessary waydroid kernel config options (android app support)
kconfig_options_waydroid = {
necessary_kconfig_options_waydroid = {
">=0.0.0": { # all versions
"all": { # all arches
"SQUASHFS": True,
"SQUASHFS_XZ": True,
"SQUASHFS_XATTR": True,
"TMPFS_XATTR": True,
"ANDROID_BINDER_IPC": True,
"ANDROID_BINDERFS": False,
"ANDROID_BINDER_DEVICES": ["binder", "hwbinder", "vndbinder"],
"ANDROID_BINDER_IPC": True,
"ANDROID_BINDER_IPC_SELFTEST": False,
"BLK_DEV_LOOP": True,
"BPF_SYSCALL": True,
"BRIDGE": True,
"BRIDGE_VLAN_FILTERING": True,
"CGROUP_BPF": True,
"FUSE_FS": True,
"IP_NF_MANGLE": True,
"NETFILTER_XTABLES": True,
"NETFILTER_XT_MATCH_COMMENT": True,
"PSI": True,
"PSI_DEFAULT_DISABLED": False,
"SQUASHFS": True,
"SQUASHFS_XATTR": True,
"SQUASHFS_XZ": True,
"TMPFS_XATTR": True,
"IP_NF_MANGLE": True,
"FUSE_FS": True,
"BLK_DEV_LOOP": True,
"TUN": True,
"VETH": True,
"VLAN_8021Q": True, # prerequisite for bridge
"BRIDGE": True,
"BRIDGE_VLAN_FILTERING": True,
}
},
">=3.5": {
@ -388,7 +363,7 @@ kconfig_options_waydroid = {
"PSI_DEFAULT_DISABLED": False,
}
},
"<5.18": { # option has been dropped
"<5.18_rc1": { # option has been dropped
"all": {
"ASHMEM": True,
}
@ -397,7 +372,7 @@ kconfig_options_waydroid = {
# Necessary iwd kernel config options (inet wireless daemon)
# Obtained from 'grep ADD_MISSING src/main.c' in iwd.git
kconfig_options_iwd = {
necessary_kconfig_options_iwd = {
">=0.0.0": { # all versions
"all": { # all arches
"ASYMMETRIC_KEY_TYPE": True,
@ -424,7 +399,7 @@ kconfig_options_iwd = {
}
# Necessary nftables kernel config options (firewall)
kconfig_options_nftables = {
necessary_kconfig_options_nftables = {
">=3.13.0": { # nftables support introduced here
"all": { # all arches
"NETFILTER": True,
@ -451,7 +426,7 @@ kconfig_options_nftables = {
"IP6_NF_NAT": True,
}
},
">=3.13.0 <5.17": { # option has been dropped
">=3.13.0 <5.17_rc1": { # option has been dropped
"all": { # all arches
"NFT_COUNTER": True,
},
@ -459,7 +434,7 @@ kconfig_options_nftables = {
}
# Necessary kernel config options for containers (lxc, Docker)
kconfig_options_containers = {
necessary_kconfig_options_containers = {
">=0.0.0": { # all versions, more specifically - since >=2.5~2.6
"all": { # all arches
"NAMESPACES": True,
@ -520,19 +495,14 @@ kconfig_options_containers = {
">=3.6": {
"all": { # all arches
"MEMCG": True,
"MEMCG_SWAP": True,
"DM_THIN_PROVISIONING": True, # Storage Drivers
"SWAP": True,
},
"x86 x86_64": { # only for x86, x86_64 (and sparc64, ia64)
"HUGETLB_PAGE": True,
"CGROUP_HUGETLB": True, # Optional section
}
},
">=3.6 <6.1_rc1": { # option has been dropped
"all": {
"MEMCG_SWAP": True,
}
},
">=3.7 <5.0": {
"all": {
"NF_NAT_IPV4": True, # Needed for lxc
@ -575,7 +545,7 @@ kconfig_options_containers = {
}
# Necessary zram kernel config options (RAM disk with on-the-fly compression)
kconfig_options_zram = {
necessary_kconfig_options_zram = {
">=3.14.0": { # zram support introduced here
"all": { # all arches
"ZRAM": True,
@ -588,7 +558,7 @@ kconfig_options_zram = {
}
# Necessary netboot kernel config options
kconfig_options_netboot = {
necessary_kconfig_options_netboot = {
">=0.0.0": { # all versions
"all": { # all arches
"BLK_DEV_NBD": True,
@ -596,67 +566,8 @@ kconfig_options_netboot = {
},
}
# Necessary wireguard & wg-quick kernel config options
# From https://gitweb.gentoo.org/repo/gentoo.git/tree/net-vpn/wireguard-tools/wireguard-tools-1.0.20210914.ebuild?id=76aaa1eeb6f001baaa68e6946f917ebb091bbd9d # noqa
kconfig_options_wireguard = {
">=5.6_rc1": { # all versions
"all": { # all arches
"WIREGUARD": True,
"IP_ADVANCED_ROUTER": True,
"IP_MULTIPLE_TABLES": True,
"IPV6_MULTIPLE_TABLES": True,
"NF_TABLES": True,
"NF_TABLES_IPV4": True,
"NF_TABLES_IPV6": True,
"NFT_CT": True,
"NFT_FIB": True,
"NFT_FIB_IPV4": True,
"NFT_FIB_IPV6": True,
"NF_CONNTRACK_MARK": True,
},
},
}
# Necessary file system config options
kconfig_options_filesystems = {
">=0.0.0": { # all versions
"all": { # all arches
"BTRFS_FS": True,
"EXFAT_FS": True,
"EXT4_FS": True,
"F2FS_FS": True,
},
},
}
kconfig_options_usb_gadgets = {
">=0.0.0": { # all versions
"all": { # all arches
# disable legacy gadgets
"USB_ETH": False,
"USB_FUNCTIONFS": False,
"USB_MASS_STORAGE": False,
"USB_G_SERIAL": False,
# enable configfs gadgets
"USB_CONFIGFS_RNDIS": True, # USB networking via RNDIS
},
},
}
# Various other kernel config options
kconfig_options_community = {
">=0.0.0": { # all versions
"all": { # all arches
"INPUT_UINPUT": True, # buffyboard
"LEDS_TRIGGER_TIMER": True, # hfd-service
"NETFILTER_XT_MATCH_TCPMSS": True, # change MTU, e.g. for Wireguard
"NETFILTER_XT_TARGET_TCPMSS": True, # change MTU, e.g. for Wireguard
},
},
}
# Necessary UEFI boot config options
kconfig_options_uefi = {
necessary_kconfig_options_uefi = {
">=0.0.0": { # all versions
"all": { # all arches
"EFI_STUB": True,
@ -728,9 +639,6 @@ apkbuild_attributes = {
"_outdir": {},
"_config": {},
# linux-edge
"_depends_dev": {"array": True},
# mesa
"_llvmver": {},
@ -751,14 +659,9 @@ apkbuild_attributes = {
apkbuild_custom_valid_options = [
"!pmb:crossdirect",
"!pmb:kconfig-check",
"pmb:kconfigcheck-community",
"pmb:kconfigcheck-containers",
"pmb:kconfigcheck-iwd",
"pmb:kconfigcheck-netboot",
"pmb:kconfigcheck-nftables",
"pmb:kconfigcheck-uefi",
"pmb:kconfigcheck-waydroid",
"pmb:kconfigcheck-zram",
"pmb:kconfigcheck-containers",
"pmb:kconfigcheck-nftables",
"pmb:cross-native",
"pmb:gpu-accel",
"pmb:strict",
@ -773,6 +676,7 @@ deviceinfo_attributes = [
"codename",
"year",
"dtb",
"modules_initfs",
"arch",
# device
@ -792,22 +696,13 @@ deviceinfo_attributes = [
# flash
"flash_heimdall_partition_kernel",
"flash_heimdall_partition_initfs",
"flash_heimdall_partition_rootfs",
"flash_heimdall_partition_system", # deprecated
"flash_heimdall_partition_system",
"flash_heimdall_partition_vbmeta",
"flash_heimdall_partition_dtbo",
"flash_fastboot_partition_kernel",
"flash_fastboot_partition_rootfs",
"flash_fastboot_partition_system", # deprecated
"flash_fastboot_partition_system",
"flash_fastboot_partition_vbmeta",
"flash_fastboot_partition_dtbo",
"flash_rk_partition_kernel",
"flash_rk_partition_rootfs",
"flash_rk_partition_system", # deprecated
"flash_mtkclient_partition_kernel",
"flash_mtkclient_partition_rootfs",
"flash_mtkclient_partition_vbmeta",
"flash_mtkclient_partition_dtbo",
"generate_legacy_uboot_initfs",
"kernel_cmdline",
"generate_bootimg",
@ -845,7 +740,7 @@ deviceinfo_attributes = [
"keymaps",
]
# Valid types for the 'chassis' attribute in deviceinfo
# Valid types for the 'chassis' atribute in deviceinfo
# See https://www.freedesktop.org/software/systemd/man/machine-info.html
deviceinfo_chassis_types = [
"desktop",
@ -876,6 +771,9 @@ default_ip = "172.16.42.1"
install_native_packages = ["cryptsetup", "util-linux", "parted"]
install_device_packages = ["postmarketos-base"]
# Groups for the default user
install_user_groups = ["wheel", "video", "audio", "input", "plugdev", "netdev"]
#
# FLASH
#
@ -884,7 +782,6 @@ flash_methods = [
"0xffff",
"fastboot",
"heimdall",
"mtkclient",
"none",
"rkdeveloptool",
"uuu",
@ -909,7 +806,7 @@ $IMAGE: Path to the combined boot/rootfs image
$IMAGE_SPLIT_BOOT: Path to the (split) boot image
$IMAGE_SPLIT_ROOT: Path to the (split) rootfs image
$PARTITION_KERNEL: Partition to flash the kernel/boot.img to
$PARTITION_ROOTFS: Partition to flash the rootfs to
$PARTITION_SYSTEM: Partition to flash the rootfs to
Fastboot specific: $KERNEL_CMDLINE
Heimdall specific: $PARTITION_INITFS
@ -917,10 +814,10 @@ uuu specific: $UUU_SCRIPT
"""
flashers = {
"fastboot": {
"depends": [], # pmaports.cfg: supported_fastboot_depends
"depends": ["android-tools", "avbtool"],
"actions": {
"list_devices": [["fastboot", "devices", "-l"]],
"flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
"flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
"$IMAGE"]],
"flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
"$BOOT/boot.img$FLAVOR"]],
@ -951,7 +848,7 @@ flashers = {
"depends": ["android-tools"],
"actions": {
"list_devices": [["fastboot", "devices", "-l"]],
"flash_rootfs": [["fastboot", "flash", "$PARTITION_ROOTFS",
"flash_rootfs": [["fastboot", "flash", "$PARTITION_SYSTEM",
"$IMAGE_SPLIT_ROOT"]],
"flash_kernel": [["fastboot", "flash", "$PARTITION_KERNEL",
"$IMAGE_SPLIT_BOOT"]],
@ -969,7 +866,7 @@ flashers = {
"list_devices": [["heimdall", "detect"]],
"flash_rootfs": [
["heimdall_wait_for_device.sh"],
["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
"flash_kernel": [["heimdall_flash_kernel.sh",
"$BOOT/initramfs$FLAVOR", "$PARTITION_INITFS",
"$BOOT/vmlinuz$FLAVOR$DTB",
@ -979,12 +876,12 @@ flashers = {
# Some Samsung devices need a 'boot.img' file, just like the one generated
# fastboot compatible devices. Example: s7562, n7100
"heimdall-bootimg": {
"depends": [], # pmaports.cfg: supported_heimdall_depends
"depends": ["heimdall", "avbtool"],
"actions": {
"list_devices": [["heimdall", "detect"]],
"flash_rootfs": [
["heimdall_wait_for_device.sh"],
["heimdall", "flash", "--$PARTITION_ROOTFS", "$IMAGE"]],
["heimdall", "flash", "--$PARTITION_SYSTEM", "$IMAGE"]],
"flash_kernel": [
["heimdall_wait_for_device.sh"],
["heimdall", "flash", "--$PARTITION_KERNEL",
@ -1027,7 +924,7 @@ flashers = {
"actions": {
"list_devices": [["rkdeveloptool", "list"]],
"flash_rootfs": [
["rkdeveloptool", "write-partition", "$PARTITION_ROOTFS",
["rkdeveloptool", "write-partition", "$PARTITION_SYSTEM",
"$IMAGE_SPLIT_ROOT"]
],
"flash_kernel": [
@ -1035,27 +932,6 @@ flashers = {
"$IMAGE_SPLIT_BOOT"]
],
},
},
"mtkclient": {
"depends": ["mtkclient"],
"actions": {
"flash_rootfs": [["mtk", "w", "$PARTITION_ROOTFS",
"$IMAGE"]],
"flash_kernel": [["mtk", "w", "$PARTITION_KERNEL",
"$BOOT/boot.img$FLAVOR"]],
"flash_vbmeta": [
# Generate vbmeta image with "disable verification" flag
["avbtool", "make_vbmeta_image", "--flags", "2",
"--padding_size", "$FLASH_PAGESIZE",
"--output", "/vbmeta.img"],
["mtk", "w", "$PARTITION_VBMETA", "/vbmeta.img"],
["rm", "-f", "/vbmeta.img"]
],
"flash_dtbo": [["mtk", "w", "$PARTITION_DTBO",
"$BOOT/dtbo.img"]],
"flash_lk2nd": [["mtk", "w", "$PARTITION_KERNEL",
"$BOOT/lk2nd.img"]]
}
}
}
@ -1123,15 +999,9 @@ newapkbuild_arguments_switches_other = [
# Patterns of package names to ignore for automatic pmaport upgrading
# ("pmbootstrap aportupgrade --all")
upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*",
"*-aarch64", "*-armhf", "*-armv7", "*-riscv64"]
"*-aarch64", "*-armhf", "*-armv7"]
#
# SIDELOAD
#
sideload_sudo_prompt = "[sudo] password for %u@%h: "
#
# CI
#
# Valid options für 'pmbootstrap ci', see https://postmarketos.org/pmb-ci
ci_valid_options = ["native", "slow"]

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import glob
@ -11,7 +11,6 @@ import pmb.config
import pmb.config.pmaports
import pmb.helpers.cli
import pmb.helpers.devices
import pmb.helpers.git
import pmb.helpers.http
import pmb.helpers.logging
import pmb.helpers.other
@ -34,23 +33,6 @@ def require_programs():
f" {', '.join(missing)}")
def ask_for_username(args):
"""
Ask for a reasonable username for the non-root user.
:returns: the username
"""
while True:
ret = pmb.helpers.cli.ask("Username", None, args.user, False,
"[a-z_][a-z0-9_-]*")
if ret == "root":
logging.fatal("ERROR: don't put \"root\" here. This is about"
" creating an additional non-root user. Don't worry,"
" the root user will also be created ;)")
continue
return ret
def ask_for_work_path(args):
"""
Ask for the work path, until we can create it (when it does not exist) and
@ -110,10 +92,7 @@ def ask_for_channel(args):
# List channels
logging.info("Choose the postmarketOS release channel.")
logging.info(f"Available ({count}):")
# Only show the first 3 releases. This includes edge, the latest supported
# release plus one. Should be a good solution until new needs arrive when
# we might want to have a custom channels.cfg attribute.
for channel, channel_data in list(channels_cfg["channels"].items())[:3]:
for channel, channel_data in channels_cfg["channels"].items():
logging.info(f"* {channel}: {channel_data['description']}")
# Default for first run: "recommended" from channels.cfg
@ -149,22 +128,19 @@ def ask_for_ui(args, info):
ui_list.pop(i)
hidden_ui_count += 1
# Get default
default = args.ui
if default not in dict(ui_list).keys():
default = pmb.config.defaults["ui"]
logging.info(f"Available user interfaces ({len(ui_list) - 1}): ")
ui_completion_list = []
for ui in ui_list:
logging.info(f"* {ui[0]}: {ui[1]}")
ui_completion_list.append(ui[0])
if hidden_ui_count > 0:
logging.info(f"NOTE: {hidden_ui_count} UIs are hidden because"
" \"deviceinfo_gpu_accelerated\" is not set (see"
" https://postmarketos.org/deviceinfo).")
logging.info(f"NOTE: {hidden_ui_count} user interfaces are not"
" available. If device supports GPU acceleration,"
" set \"deviceinfo_gpu_accelerated\" to make UIs"
" available. See: <https://wiki.postmarketos.org/wiki/"
"Deviceinfo_reference")
while True:
ret = pmb.helpers.cli.ask("User interface", None, default, True,
ret = pmb.helpers.cli.ask("User interface", None, args.ui, True,
complete=ui_completion_list)
if ret in dict(ui_list).keys():
return ret
@ -626,36 +602,15 @@ def ask_build_pkgs_on_install(args):
default=args.build_pkgs_on_install)
def get_locales():
ret = []
list_path = f"{pmb.config.pmb_src}/pmb/data/locales"
with open(list_path, "r") as handle:
for line in handle:
ret += [line.rstrip()]
return ret
def ask_for_locale(args):
locales = get_locales()
logging.info("Choose your preferred locale, like e.g. en_US. Only UTF-8"
" is supported, it gets appended automatically. Use"
" tab-completion if needed.")
while True:
ret = pmb.helpers.cli.ask("Locale",
choices=None,
default=args.locale.replace(".UTF-8", ""),
lowercase_answer=False,
complete=locales)
ret = ret.replace(".UTF-8", "")
if ret not in locales:
logging.info("WARNING: this locale is not in the list of known"
" valid locales.")
if pmb.helpers.cli.ask() != "y":
# Ask again
continue
return f"{ret}.UTF-8"
locales = pmb.config.locales
logging.info(f"Available locales ({len(locales)}): {', '.join(locales)}")
return pmb.helpers.cli.ask("Choose default locale for installation",
choices=None,
default=args.locale,
lowercase_answer=False,
validation_regex="|".join(locales),
complete=locales)
def frontend(args):
@ -681,15 +636,6 @@ def frontend(args):
pmb.config.pmaports.switch_to_channel_branch(args, channel)
cfg["pmbootstrap"]["is_default_channel"] = "False"
# Copy the git hooks if master was checked out. (Don't symlink them and
# only do it on master, so the git hooks don't change unexpectedly when
# having a random branch checked out.)
branch_current = pmb.helpers.git.rev_parse(args, args.aports,
extra_args=["--abbrev-ref"])
if branch_current == "master":
logging.info("NOTE: pmaports is on master branch, copying git hooks.")
pmb.config.pmaports.install_githooks(args)
# Device
device, device_exists, kernel, nonfree = ask_for_device(args)
cfg["pmbootstrap"]["device"] = device
@ -707,7 +653,11 @@ def frontend(args):
if device_exists:
cfg["pmbootstrap"]["keymap"] = ask_for_keymaps(args, info)
cfg["pmbootstrap"]["user"] = ask_for_username(args)
# Username
cfg["pmbootstrap"]["user"] = pmb.helpers.cli.ask("Username", None,
args.user, False,
"[a-z_][a-z0-9_-]*")
ask_for_provider_select_pkg(args, "postmarketos-base", cfg["providers"])
# UI and various build options

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import configparser

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.config

View File

@ -1,9 +1,8 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import configparser
import logging
import os
import sys
import pmb.config
import pmb.helpers.git
@ -59,7 +58,7 @@ def check_version_pmaports(real):
def check_version_pmbootstrap(min):
# Compare versions
real = pmb.__version__
real = pmb.config.version
if pmb.parse.version.compare(real, min) >= 0:
return
@ -90,9 +89,9 @@ def read_config(args):
# Migration message
if not os.path.exists(args.aports):
logging.error(f"ERROR: pmaports dir not found: {args.aports}")
logging.error("Did you run 'pmbootstrap init'?")
sys.exit(1)
raise RuntimeError("We have split the aports repository from the"
" pmbootstrap repository (#383). Please run"
" 'pmbootstrap init' again to clone it.")
# Require the config
path_cfg = args.aports + "/pmaports.cfg"
@ -192,17 +191,3 @@ def switch_to_channel_branch(args, channel_new):
# Verify pmaports.cfg on new branch
read_config(args)
return True
def install_githooks(args):
hooks_dir = os.path.join(args.aports, ".githooks")
if not os.path.exists(hooks_dir):
logging.info("No .githooks dir found")
return
for h in os.listdir(hooks_dir):
src = os.path.join(hooks_dir, h)
# Use git default hooks dir so users can ignore our hooks
# if they dislike them by setting "core.hooksPath" git config
dst = os.path.join(args.aports, ".git", "hooks", h)
if pmb.helpers.run.user(args, ["cp", src, dst], check=False):
logging.warning(f"WARNING: Copying git hook failed: {dst}")

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

View File

@ -1,22 +1,14 @@
# Copyright 2023 Anjandev Momi
# Copyright 2022 Anjandev Momi
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import shutil
from functools import lru_cache
from typing import Optional
@lru_cache()
def which_sudo() -> Optional[str]:
"""Returns a command required to run commands as root, if any.
def which_sudo():
"""
Find whether sudo or doas is installed for commands that require root.
Allows user to override preferred sudo with PMB_SUDO env variable.
"""
if os.getuid() == 0:
return None
supported_sudos = ['doas', 'sudo']
user_set_sudo = os.getenv("PMB_SUDO")

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
""" Save, read, verify workdir state related information in $WORK/workdir.cfg,
for example the init dates of the chroots. This is not saved in

View File

@ -1,304 +0,0 @@
C
a_DJ
aa_ER
aa_ET
af_ZA
agr_PE
ak_GH
am_ET
an_ES
anp_IN
ar_AE
ar_BH
ar_DZ
ar_EG
ar_IN
ar_IQ
ar_JO
ar_KW
ar_LB
ar_LY
ar_MA
ar_OM
ar_QA
ar_SA
ar_SD
ar_SS
ar_SY
ar_TN
ar_YE
as_IN
ast_ES
ayc_PE
az_AZ
az_IR
be_BY
bem_ZM
ber_DZ
ber_MA
bg_BG
bhb_IN
bho_IN
bho_NP
bi_VU
bn_BD
bn_IN
bo_CN
bo_IN
br_FR
brx_IN
bs_BA
byn_ER
ca_AD
ca_ES
ca_FR
ca_IT
ce_RU
ch_DE
chr_US
cmn_TW
crh_UA
cs_CZ
csb_PL
cv_RU
cy_GB
da_DK
de_AT
de_BE
de_CH
de_DE
de_IT
de_LI
de_LU
doi_IN
dsb_DE
dv_MV
dz_BT
el_CY
el_GR
en_AG
en_AU
en_BW
en_CA
en_DK
en_GB
en_HK
en_IE
en_IL
en_IN
en_NG
en_NZ
en_PH
en_SC
en_SG
en_US
en_ZA
en_ZM
en_ZW
eo
es_AR
es_BO
es_CL
es_CO
es_CR
es_CU
es_DO
es_EC
es_ES
es_GT
es_HN
es_MX
es_NI
es_PA
es_PE
es_PR
es_PY
es_SV
es_US
es_UY
es_VE
et_EE
eu_ES
fa_IR
ff_SN
fi_FI
fil_PH
fo_FO
fr_BE
fr_CA
fr_CH
fr_FR
fr_LU
fur_IT
fy_DE
fy_NL
ga_IE
gd_GB
gez_ER
gez_ET
gl_ES
gu_IN
gv_GB
ha_NG
hak_TW
he_IL
hi_IN
hif_FJ
hne_IN
hr_HR
hsb_DE
ht_HT
hu_HU
hy_AM
ia_FR
id_ID
ig_NG
ik_CA
is_IS
it_CH
it_IT
iu_CA
ja_JP
ka_GE
kab_DZ
kk_KZ
kl_GL
km_KH
kn_IN
ko_KR
kok_IN
ks_IN
ku_TR
kw_GB
ky_KG
lb_LU
lg_UG
li_BE
li_NL
lij_IT
ln_CD
lo_LA
lt_LT
lv_LV
lzh_TW
mag_IN
mai_IN
mai_NP
mfe_MU
mg_MG
mhr_RU
mi_NZ
miq_NI
mjw_IN
mk_MK
ml_IN
mn_MN
mni_IN
mnw_MM
mr_IN
ms_MY
mt_MT
my_MM
nan_TW
nb_NO
nds_DE
nds_NL
ne_NP
nhn_MX
niu_NU
niu_NZ
nl_AW
nl_BE
nl_NL
nn_NO
nr_ZA
nso_ZA
oc_FR
om_ET
om_KE
or_IN
os_RU
pa_IN
pa_PK
pap_AW
pap_CW
pl_PL
ps_AF
pt_BR
pt_PT
quz_PE
raj_IN
ro_RO
ru_RU
ru_UA
rw_RW
sa_IN
sah_RU
sat_IN
sc_IT
sd_IN
se_NO
sgs_LT
shn_MM
shs_CA
si_LK
sid_ET
sk_SK
sl_SI
sm_WS
so_DJ
so_ET
so_KE
so_SO
sq_AL
sq_MK
sr_ME
sr_RS
ss_ZA
st_ZA
sv_FI
sv_SE
sw_KE
sw_TZ
szl_PL
ta_IN
ta_LK
tcy_IN
te_IN
tg_TJ
th_TH
the_NP
ti_ER
ti_ET
tig_ER
tk_TM
tl_PH
tn_ZA
to_TO
tpi_PG
tr_CY
tr_TR
ts_ZA
tt_RU
ug_CN
uk_UA
unm_US
ur_IN
ur_PK
uz_UZ
ve_ZA
vi_VN
wa_BE
wae_CH
wal_ET
wo_SN
xh_ZA
yi_US
yo_NG
yue_HK
yuw_PG
zh_CN
zh_HK
zh_SG
zh_TW
zu_ZA

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.export.frontend import frontend
from pmb.export.odin import odin

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os

View File

@ -1,7 +1,6 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.flasher.init import init
from pmb.flasher.init import install_depends
from pmb.flasher.run import run
from pmb.flasher.run import check_partition_blacklist
from pmb.flasher.variables import variables

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -88,8 +88,11 @@ def list_devices(args):
def sideload(args):
method = args.flash_method or args.deviceinfo["flash_method"]
cfg = pmb.config.flashers[method]
# Install depends
pmb.flasher.install_depends(args)
pmb.chroot.apk.install(args, cfg["depends"])
# Mount the buildroot
suffix = "buildroot_" + args.deviceinfo["arch"]
@ -110,37 +113,15 @@ def sideload(args):
def flash_lk2nd(args):
method = args.flash_method or args.deviceinfo["flash_method"]
if method == "fastboot":
# In the future this could be expanded to use "fastboot flash lk2nd $img"
# which reflashes/updates lk2nd from itself. For now let the user handle this
# manually since supporting the codepath with heimdall requires more effort.
pmb.flasher.init(args)
logging.info("(native) checking current fastboot product")
output = pmb.chroot.root(args, ["fastboot", "getvar", "product"],
output="interactive", output_return=True)
# Variable "product" is e.g. "LK2ND_MSM8974" or "lk2nd-msm8226" depending
# on the lk2nd version.
if "lk2nd" in output.lower():
raise RuntimeError("You are currently running lk2nd. Please reboot into the regular"
" bootloader mode to re-flash lk2nd.")
chroot_path = args.work + "/chroot_rootfs_" + args.device
lk2nd_path = "/boot/lk2nd.img"
if not os.path.exists(chroot_path + lk2nd_path):
raise RuntimeError(f"{chroot_path+lk2nd_path} doesn't exist. Your"
" device may not support lk2nd.")
# Get the lk2nd package (which is a dependency of the device package)
device_pkg = f"device-{args.device}"
apkbuild = pmb.helpers.pmaports.get(args, device_pkg)
lk2nd_pkg = None
for dep in apkbuild["depends"]:
if dep.startswith("lk2nd"):
lk2nd_pkg = dep
break
if not lk2nd_pkg:
raise RuntimeError(f"{device_pkg} does not depend on any lk2nd package")
suffix = "rootfs_" + args.device
pmb.chroot.apk.install(args, [lk2nd_pkg], suffix)
logging.info("(native) flash lk2nd image")
logging.info(lk2nd_path)
logging.info("It's normal if fastboot warns"
" \"Image not signed or corrupt\" or a similar warning")
pmb.flasher.run(args, "flash_lk2nd")
@ -148,6 +129,10 @@ def frontend(args):
action = args.action_flasher
method = args.flash_method or args.deviceinfo["flash_method"]
# Legacy alias
if action == "flash_system":
action = "flash_rootfs"
if method == "none" and action in ["boot", "flash_kernel", "flash_rootfs",
"flash_lk2nd"]:
logging.info("This device doesn't support any flash method.")
@ -155,17 +140,17 @@ def frontend(args):
if action in ["boot", "flash_kernel"]:
kernel(args)
elif action == "flash_rootfs":
if action == "flash_rootfs":
rootfs(args)
elif action == "flash_vbmeta":
if action == "flash_vbmeta":
flash_vbmeta(args)
elif action == "flash_dtbo":
if action == "flash_dtbo":
flash_dtbo(args)
elif action == "flash_lk2nd":
flash_lk2nd(args)
elif action == "list_flavors":
if action == "list_flavors":
list_flavors(args)
elif action == "list_devices":
if action == "list_devices":
list_devices(args)
elif action == "sideload":
if action == "sideload":
sideload(args)
if action in ["flash_lk2nd"]:
flash_lk2nd(args)

View File

@ -1,12 +1,12 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.chroot.apk
import pmb.config
import pmb.config.pmaports
import pmb.chroot.apk
import pmb.helpers.mount
def install_depends(args):
def init(args):
# Validate method
if hasattr(args, 'flash_method'):
method = args.flash_method or args.deviceinfo["flash_method"]
else:
@ -20,28 +20,10 @@ def install_depends(args):
"Make sure, it is packaged for Alpine Linux, or"
" package it yourself, and then add it to"
" pmb/config/__init__.py.")
depends = pmb.config.flashers[method]["depends"]
cfg = pmb.config.flashers[method]
# Depends for some flash methods may be different for various pmaports
# branches, so read them from pmaports.cfg.
if method == "fastboot":
pmaports_cfg = pmb.config.pmaports.read_config(args)
depends = pmaports_cfg.get("supported_fastboot_depends",
"android-tools,avbtool").split(",")
elif method == "heimdall-bootimg":
pmaports_cfg = pmb.config.pmaports.read_config(args)
depends = pmaports_cfg.get("supported_heimdall_depends",
"heimdall,avbtool").split(",")
elif method == "mtkclient":
pmaports_cfg = pmb.config.pmaports.read_config(args)
depends = pmaports_cfg.get("supported_mtkclient_depends",
"mtkclient,android-tools").split(",")
pmb.chroot.apk.install(args, depends)
def init(args):
install_depends(args)
# Install depends
pmb.chroot.apk.install(args, cfg["depends"])
# Mount folders from host system
for folder in pmb.config.flash_mount_bind:

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.flasher
import pmb.chroot.initfs

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import pmb.config.pmaports
@ -10,15 +10,11 @@ def variables(args, flavor, method):
flash_pagesize = args.deviceinfo['flash_pagesize']
# TODO Remove _partition_system deviceinfo support once pmaports has been
# updated and minimum pmbootstrap version bumped.
# See also https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2243
if method.startswith("fastboot"):
_partition_kernel = args.deviceinfo["flash_fastboot_partition_kernel"]\
or "boot"
_partition_rootfs = args.deviceinfo["flash_fastboot_partition_rootfs"]\
or args.deviceinfo["flash_fastboot_partition_system"] or "userdata"
_partition_system = args.deviceinfo["flash_fastboot_partition_system"]\
or "system"
_partition_vbmeta = args.deviceinfo["flash_fastboot_partition_vbmeta"]\
or None
_partition_dtbo = args.deviceinfo["flash_fastboot_partition_dtbo"]\
@ -27,24 +23,15 @@ def variables(args, flavor, method):
elif method.startswith("rkdeveloptool"):
_partition_kernel = args.deviceinfo["flash_rk_partition_kernel"]\
or None
_partition_rootfs = args.deviceinfo["flash_rk_partition_rootfs"]\
or args.deviceinfo["flash_rk_partition_system"] or None
_partition_system = args.deviceinfo["flash_rk_partition_system"]\
or None
_partition_vbmeta = None
_partition_dtbo = None
elif method.startswith("mtkclient"):
_partition_kernel = args.deviceinfo["flash_mtkclient_partition_kernel"]\
or "boot"
_partition_rootfs = args.deviceinfo["flash_mtkclient_partition_rootfs"]\
or "userdata"
_partition_vbmeta = args.deviceinfo["flash_mtkclient_partition_vbmeta"]\
or None
_partition_dtbo = args.deviceinfo["flash_mtkclient_partition_dtbo"]\
or None
else:
_partition_kernel = args.deviceinfo["flash_heimdall_partition_kernel"]\
or "KERNEL"
_partition_rootfs = args.deviceinfo["flash_heimdall_partition_rootfs"]\
or args.deviceinfo["flash_heimdall_partition_system"] or "SYSTEM"
_partition_system = args.deviceinfo["flash_heimdall_partition_system"]\
or "SYSTEM"
_partition_vbmeta = args.deviceinfo["flash_heimdall_partition_vbmeta"]\
or None
_partition_dtbo = args.deviceinfo["flash_heimdall_partition_dtbo"]\
@ -54,7 +41,7 @@ def variables(args, flavor, method):
# Only one operation is done at same time so it doesn't matter
# sharing the arg
_partition_kernel = args.partition
_partition_rootfs = args.partition
_partition_system = args.partition
_partition_vbmeta = args.partition
_partition_dtbo = args.partition
@ -72,7 +59,7 @@ def variables(args, flavor, method):
"$PARTITION_KERNEL": _partition_kernel,
"$PARTITION_INITFS": args.deviceinfo[
"flash_heimdall_partition_initfs"] or "RECOVERY",
"$PARTITION_ROOTFS": _partition_rootfs,
"$PARTITION_SYSTEM": _partition_system,
"$PARTITION_VBMETA": _partition_vbmeta,
"$PARTITION_DTBO": _partition_dtbo,
"$FLASH_PAGESIZE": flash_pagesize,

View File

@ -1,2 +1,2 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later

View File

@ -1,4 +1,4 @@
# Copyright 2023 Johannes Marbach, Oliver Smith
# Copyright 2022 Johannes Marbach, Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
@ -6,7 +6,6 @@ import pmb.chroot.root
import pmb.config.pmaports
import pmb.helpers.cli
import pmb.helpers.run
import pmb.helpers.run_core
import pmb.parse.version
@ -63,7 +62,7 @@ def _create_command_with_progress(command, fifo):
"""
flags = ["--no-progress", "--progress-fd", "3"]
command_full = [command[0]] + flags + command[1:]
command_flat = pmb.helpers.run_core.flat_cmd(command_full)
command_flat = pmb.helpers.run.flat_cmd(command_full)
command_flat = f"exec 3>{fifo}; {command_flat}"
return ["sh", "-c", command_flat]

View File

@ -1,4 +1,4 @@
# Copyright 2023 Luca Weiss
# Copyright 2022 Luca Weiss
# SPDX-License-Identifier: GPL-3.0-or-later
import datetime
import fnmatch
@ -34,7 +34,7 @@ def init_req_headers() -> None:
return
# Generic request headers
req_headers = {
'User-Agent': f'pmbootstrap/{pmb.__version__} aportupgrade'}
'User-Agent': f'pmbootstrap/{pmb.config.version} aportupgrade'}
# Request headers specific to GitHub
req_headers_github = dict(req_headers)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import copy
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import datetime
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import glob

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import json
@ -9,14 +9,13 @@ import sys
import pmb.aportgen
import pmb.build
import pmb.build.autodetect
import pmb.sideload
import pmb.chroot
import pmb.chroot.initfs
import pmb.chroot.other
import pmb.ci
import pmb.config
import pmb.export
import pmb.flasher
import pmb.helpers.aportupgrade
import pmb.helpers.devices
import pmb.helpers.git
import pmb.helpers.lint
@ -26,13 +25,13 @@ import pmb.helpers.pmaports
import pmb.helpers.repo
import pmb.helpers.repo_missing
import pmb.helpers.run
import pmb.helpers.aportupgrade
import pmb.helpers.status
import pmb.install
import pmb.install.blockdevice
import pmb.netboot
import pmb.parse
import pmb.qemu
import pmb.sideload
def _parse_flavor(args, autoinstall=True):
@ -377,17 +376,17 @@ def newapkbuild(args):
def kconfig(args):
if args.action_kconfig == "check":
details = args.kconfig_check_details
# Build the components list from cli arguments (--waydroid etc.)
components_list = []
for name in pmb.parse.kconfig.get_all_component_names():
if getattr(args, f"kconfig_check_{name}"):
components_list += [name]
# Handle passing a file directly
if args.file:
if pmb.parse.kconfig.check_file(args.package, components_list,
details=details):
if pmb.parse.kconfig.check_file(args.package,
waydroid=args.waydroid,
nftables=args.nftables,
containers=args.containers,
zram=args.zram,
netboot=args.netboot,
community=args.community,
uefi=args.uefi,
details=True):
logging.info("kconfig check succeeded!")
return
raise RuntimeError("kconfig check failed!")
@ -414,8 +413,17 @@ def kconfig(args):
if "!pmb:kconfigcheck" in apkbuild["options"]:
skipped += 1
continue
if not pmb.parse.kconfig.check(args, package, components_list,
details=details):
if not pmb.parse.kconfig.check(
args, package,
force_waydroid_check=args.waydroid,
force_iwd_check=args.iwd,
force_nftables_check=args.nftables,
force_containers_check=args.containers,
force_zram_check=args.zram,
force_netboot_check=args.netboot,
force_community_check=args.community,
force_uefi_check=args.uefi,
details=True):
error = True
# At least one failure
@ -531,25 +539,18 @@ def work_migrate(args):
def log(args):
log_testsuite = f"{args.work}/log_testsuite.txt"
if args.clear_log:
pmb.helpers.run.user(args, ["truncate", "-s", "0", args.log])
pmb.helpers.run.user(args, ["truncate", "-s", "0", log_testsuite])
pmb.helpers.run.user(args, ["tail", "-n", args.lines, "-F", args.log],
output="tui")
cmd = ["tail", "-n", args.lines, "-F"]
# Follow the testsuite's log file too if it exists. It will be created when
# starting a test case that writes to it (git -C test grep log_testsuite).
if os.path.exists(log_testsuite):
cmd += [log_testsuite]
# tail writes the last lines of the files to the terminal. Put the regular
# log at the end, so that output is visible at the bottom (where the user
# looks for an error / what's currently going on).
cmd += [args.log]
pmb.helpers.run.user(args, cmd, output="tui")
def log_distccd(args):
logpath = "/home/pmos/distccd.log"
if args.clear_log:
pmb.chroot.user(args, ["truncate", "-s", "0", logpath])
pmb.chroot.user(args, ["tail", "-n", args.lines, "-f", logpath],
output="tui")
def zap(args):
@ -609,47 +610,3 @@ def lint(args):
def status(args):
if not pmb.helpers.status.print_status(args, args.details):
sys.exit(1)
def ci(args):
topdir = pmb.helpers.git.get_topdir(args, os.getcwd())
if not os.path.exists(topdir):
logging.error("ERROR: change your current directory to a git"
" repository (e.g. pmbootstrap, pmaports) before running"
" 'pmbootstrap ci'.")
exit(1)
scripts_available = pmb.ci.get_ci_scripts(topdir)
scripts_available = pmb.ci.sort_scripts_by_speed(scripts_available)
if not scripts_available:
logging.error("ERROR: no supported CI scripts found in current git"
" repository, see https://postmarketos.org/pmb-ci")
exit(1)
scripts_selected = {}
if args.scripts:
if args.all:
raise RuntimeError("Combining --all with script names doesn't"
" make sense")
for script in args.scripts:
if script not in scripts_available:
logging.error(f"ERROR: script '{script}' not found in git"
" repository, found these:"
f" {', '.join(scripts_available.keys())}")
exit(1)
scripts_selected[script] = scripts_available[script]
elif args.all:
scripts_selected = scripts_available
if args.fast:
for script, script_data in scripts_available.items():
if "slow" not in script_data["options"]:
scripts_selected[script] = script_data
if not pmb.helpers.git.clean_worktree(args, topdir):
logging.warning("WARNING: this git repository has uncommitted changes")
if not scripts_selected:
scripts_selected = pmb.ci.ask_which_scripts_to_run(scripts_available)
pmb.ci.run_scripts(args, topdir, scripts_selected)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import configparser
import logging
@ -246,29 +246,3 @@ def is_outdated(path):
date_outdated = time.time() - pmb.config.git_repo_outdated
return date_head <= date_outdated
def get_topdir(args, path):
""" :returns: a string with the top dir of the git repository, or an
empty string if it's not a git repository. """
return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"],
path, output_return=True, check=False).rstrip()
def get_files(args, path):
""" Get all files inside a git repository, that are either already in the
git tree or are not in gitignore. Do not list deleted files. To be used
for creating a tarball of the git repository.
:param path: top dir of the git repository
:returns: all files in a git repository as list, relative to path """
ret = []
files = pmb.helpers.run.user(args, ["git", "ls-files"], path,
output_return=True).split("\n")
files += pmb.helpers.run.user(args, ["git", "ls-files",
"--exclude-standard", "--other"], path,
output_return=True).split("\n")
for file in files:
if os.path.exists(f"{path}/{file}"):
ret += [file]
return ret

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import hashlib
import json

View File

@ -1,4 +1,4 @@
# Copyright 2023 Danct12 <danct12@disroot.org>
# Copyright 2022 Danct12 <danct12@disroot.org>
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -17,6 +17,7 @@ def check(args, pkgnames):
:param pkgnames: Names of the packages to lint
"""
pmb.chroot.apk.install(args, ["atools"])
pmb.build.init(args)
# Mount pmaports.git inside the chroot so that we don't have to copy the
# package folders

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import pmb.helpers.run

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import logging
@ -54,12 +54,9 @@ def check_binfmt_misc(args):
if os.path.exists(path):
return
# check=False: this might be built-in instead of being a module
pmb.helpers.run.root(args, ["modprobe", "binfmt_misc"], check=False)
# check=False: we check it below and print a more helpful message on error
pmb.helpers.run.root(args, ["mount", "-t", "binfmt_misc", "none",
"/proc/sys/fs/binfmt_misc"], check=False)
"/proc/sys/fs/binfmt_misc"])
if not os.path.exists(path):
link = "https://postmarketos.org/binfmt_misc"
@ -267,17 +264,16 @@ def validate_hostname(hostname):
return False
# Check that it only contains valid chars
if not re.match(r"^[0-9a-z-\.]*$", hostname):
if not re.match("^[0-9a-z-]*$", hostname):
logging.fatal("ERROR: Hostname must only contain letters (a-z),"
" digits (0-9), minus signs (-), or periods (.)")
" digits (0-9) or minus signs (-)")
return False
# Check that doesn't begin or end with a minus sign or period
if re.search(r"^-|^\.|-$|\.$", hostname):
# Check that doesn't begin or end with a minus sign
if hostname[:1] == "-" or hostname[-1:] == "-":
logging.fatal("ERROR: Hostname must not begin or end with a minus"
" sign or period")
" sign")
return False
return True

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
"""
Functions that work with both pmaports and binary package repos. See also:

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
"""
Functions that work with pmaports. See also:

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
"""
Functions that work with binary package repos. See also:
@ -43,10 +43,10 @@ def hash(url, length=8):
def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
"""
Get a list of repository URLs, as they are in /etc/apk/repositories.
:param user_repository: add /mnt/pmbootstrap/packages
:param user_repository: add /mnt/pmbootstrap-packages
:param postmarketos_mirror: add postmarketos mirror URLs
:param alpine: add alpine mirror URLs
:returns: list of mirror strings, like ["/mnt/pmbootstrap/packages",
:returns: list of mirror strings, like ["/mnt/pmbootstrap-packages",
"http://...", ...]
"""
ret = []
@ -59,7 +59,7 @@ def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True):
# Local user repository (for packages compiled with pmbootstrap)
if user_repository:
ret.append("/mnt/pmbootstrap/packages")
ret.append("/mnt/pmbootstrap-packages")
# Upstream postmarketOS binary repository
if postmarketos_mirror:

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging

View File

@ -1,8 +1,38 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import shlex
import pmb.helpers.run_core
def flat_cmd(cmd, working_dir=None, env={}):
"""
Convert a shell command passed as list into a flat shell string with
proper escaping.
:param cmd: command as list, e.g. ["echo", "string with spaces"]
:param working_dir: when set, prepend "cd ...;" to execute the command
in the given working directory
:param env: dict of environment variables to be passed to the command, e.g.
{"JOBS": "5"}
:returns: the flat string, e.g.
echo 'string with spaces'
cd /home/pmos;echo 'string with spaces'
"""
# Merge env and cmd into escaped list
escaped = []
for key, value in env.items():
escaped.append(key + "=" + shlex.quote(value))
for i in range(len(cmd)):
escaped.append(shlex.quote(cmd[i]))
# Prepend working dir
ret = " ".join(escaped)
if working_dir:
ret = "cd " + shlex.quote(working_dir) + ";" + ret
return ret
def user(args, cmd, working_dir=None, output="log", output_return=False,
check=None, env={}, sudo=False):
"""
@ -24,7 +54,7 @@ def user(args, cmd, working_dir=None, output="log", output_return=False,
# Add environment variables and run
if env:
cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
return pmb.helpers.run_core.core(args, msg, cmd, working_dir, output,
output_return, check, sudo)
@ -41,8 +71,8 @@ def root(args, cmd, working_dir=None, output="log", output_return=False,
arguments and the return value.
"""
if env:
cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)]
cmd = pmb.config.sudo(cmd)
cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
cmd = [pmb.config.sudo] + cmd
return user(args, cmd, working_dir, output, output_return, check, env,
True)

View File

@ -1,14 +1,13 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import fcntl
import logging
import os
import selectors
import shlex
import subprocess
import sys
import threading
import time
import os
import pmb.helpers.run
""" For a detailed description of all output modes, read the description of
@ -16,35 +15,6 @@ import pmb.helpers.run
called by core(). """
def flat_cmd(cmd, working_dir=None, env={}):
"""
Convert a shell command passed as list into a flat shell string with
proper escaping.
:param cmd: command as list, e.g. ["echo", "string with spaces"]
:param working_dir: when set, prepend "cd ...;" to execute the command
in the given working directory
:param env: dict of environment variables to be passed to the command, e.g.
{"JOBS": "5"}
:returns: the flat string, e.g.
echo 'string with spaces'
cd /home/pmos;echo 'string with spaces'
"""
# Merge env and cmd into escaped list
escaped = []
for key, value in env.items():
escaped.append(key + "=" + shlex.quote(value))
for i in range(len(cmd)):
escaped.append(shlex.quote(cmd[i]))
# Prepend working dir
ret = " ".join(escaped)
if working_dir:
ret = "cd " + shlex.quote(working_dir) + ";" + ret
return ret
def sanity_checks(output="log", output_return=False, check=None):
"""
Raise an exception if the parameters passed to core() don't make sense
@ -257,10 +227,10 @@ def sudo_timer_iterate():
Run sudo -v and schedule a new timer to repeat the same.
"""
if pmb.config.which_sudo() == "sudo":
if pmb.config.sudo == "sudo":
subprocess.Popen(["sudo", "-v"]).wait()
else:
subprocess.Popen(pmb.config.sudo(["true"])).wait()
subprocess.Popen([pmb.config.sudo, "true"]).wait()
timer = threading.Timer(interval=60, function=sudo_timer_iterate)
timer.daemon = True
@ -318,7 +288,7 @@ def core(args, log_message, cmd, working_dir=None, output="log",
output value | timeout | out to log | out to stdout | wait | pass stdin
------------------------------------------------------------------------
"log" | x | x | | x |
"stdout" | x | x | x | x |
"stdout" | x | x | x | x | x
"interactive" | | x | x | x | x
"tui" | | | x | x | x
"background" | | x | | |
@ -340,15 +310,6 @@ def core(args, log_message, cmd, working_dir=None, output="log",
"""
sanity_checks(output, output_return, check)
# Preserve proxy environment variables
env = {}
for var in ["FTP_PROXY", "ftp_proxy", "HTTP_PROXY", "http_proxy",
"HTTPS_PROXY", "https_proxy", "HTTP_PROXY_AUTH"]:
if var in os.environ:
env[var] = os.environ[var]
if env:
cmd = ["sh", "-c", flat_cmd(cmd, env=env)]
if args.sudo_timer and sudo:
sudo_timer_start()
@ -377,7 +338,7 @@ def core(args, log_message, cmd, working_dir=None, output="log",
output_timeout = output in ["log", "stdout"] and not disable_timeout
stdin = subprocess.DEVNULL if output in ["log", "stdout"] else None
stdin = subprocess.DEVNULL if output == "log" else None
(code, output_after_run) = foreground_pipe(args, cmd, working_dir,
output_to_stdout,

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Clayton Craft
# Copyright 2022 Clayton Craft
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import glob

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.install._install import install
from pmb.install._install import get_kernel_package

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -164,8 +164,8 @@ def create_home_from_skel(args):
def configure_apk(args):
"""
Copy over all official keys, and the keys used to compile local packages
(unless --no-local-pkgs is set). Then copy the corresponding APKINDEX files
and remove the /mnt/pmbootstrap/packages repository.
(unless --no-local-pkgs is set). Then disable the /mnt/pmbootstrap-packages
repository.
"""
# Official keys
pattern = f"{pmb.config.apk_keys_path}/*.pub"
@ -179,15 +179,8 @@ def configure_apk(args):
for key in glob.glob(pattern):
pmb.helpers.run.root(args, ["cp", key, rootfs + "/etc/apk/keys/"])
# Copy over the corresponding APKINDEX files from cache
index_files = pmb.helpers.repo.apkindex_files(args,
arch=args.deviceinfo["arch"],
user_repository=False)
for f in index_files:
pmb.helpers.run.root(args, ["cp", f, rootfs + "/var/cache/apk/"])
# Disable pmbootstrap repository
pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap\/packages/d",
pmb.helpers.run.root(args, ["sed", "-i", r"/\/mnt\/pmbootstrap-packages/d",
rootfs + "/etc/apk/repositories"])
pmb.helpers.run.user(args, ["cat", rootfs + "/etc/apk/repositories"])
@ -204,13 +197,7 @@ def set_user(args):
if not pmb.chroot.user_exists(args, args.user, suffix):
pmb.chroot.root(args, ["adduser", "-D", "-u", "10000", args.user],
suffix)
pmaports_cfg = pmb.config.pmaports.read_config(args)
groups = []
groups += pmaports_cfg.get("install_user_groups",
"audio,input,netdev,plugdev,video,wheel").split(",")
groups += pmb.install.ui.get_groups(args)
groups = pmb.install.ui.get_groups(args) + pmb.config.install_user_groups
for group in groups:
pmb.chroot.root(args, ["addgroup", "-S", group], suffix,
check=False)
@ -360,26 +347,6 @@ def setup_keymap(args):
logging.info("NOTE: No valid keymap specified for device")
def setup_timezone(args):
suffix = f"rootfs_{args.device}"
arch = args.deviceinfo["arch"]
alpine_conf = pmb.helpers.package.get(args, "alpine-conf", arch)
version = alpine_conf["version"].split("-r")[0]
setup_tz_cmd = ["setup-timezone"]
# setup-timezone will, by default, copy the timezone to /etc/zoneinfo
# and disregard tzdata, to save space. If we actually have tzdata
# installed, make sure that setup-timezone makes use of it, since
# there's no space to be saved.
if "tzdata" in pmb.chroot.apk.installed(args, suffix):
setup_tz_cmd += ["-i"]
if not pmb.parse.version.check_string(version, ">=3.14.0"):
setup_tz_cmd += ["-z"]
setup_tz_cmd += [args.timezone]
pmb.chroot.root(args, setup_tz_cmd, suffix)
def setup_hostname(args):
"""
Set the hostname and update localhost address in /etc/hosts
@ -403,25 +370,6 @@ def setup_hostname(args):
pmb.chroot.root(args, ["sed", "-i", "-e", regex, "/etc/hosts"], suffix)
def setup_appstream(args):
"""
If alpine-appstream-downloader has been downloaded, execute it to have
update AppStream data on new installs
"""
suffix = "rootfs_" + args.device
installed_pkgs = pmb.chroot.apk.installed(args, suffix)
if "alpine-appstream-downloader" not in installed_pkgs or args.offline:
return
pmb.chroot.root(args, ["alpine-appstream-downloader",
"/mnt/appstream-data"], suffix)
pmb.chroot.root(args, ["mkdir", "-p", "/var/lib/swcatalog"], suffix)
pmb.chroot.root(args, ["cp", "-r", "/mnt/appstream-data/icons",
"/mnt/appstream-data/xml",
"-t", "/var/lib/swcatalog"], suffix)
def disable_sshd(args):
if not args.no_sshd:
return
@ -612,7 +560,7 @@ def write_cgpt_kpart(args, layout, suffix):
:param layout: partition layout from get_partition_layout()
:param suffix: of the chroot, which holds the image file to be flashed
"""
if not args.deviceinfo["cgpt_kpart"] or not args.install_cgpt:
if not args.deviceinfo["cgpt_kpart"]:
return
device_rootfs = mount_device_rootfs(args, suffix)
@ -700,74 +648,6 @@ def get_partition_layout(reserve, kernel):
return ret
def get_uuid(args, partition):
"""
Get UUID of a partition
:param partition: block device for getting UUID from
"""
return pmb.chroot.root(
args,
[
"blkid",
"-s", "UUID",
"-o", "value",
partition,
],
output_return=True
).rstrip()
def create_crypttab(args, layout, suffix):
"""
Create /etc/crypttab config
:param layout: partition layout from get_partition_layout()
:param suffix: of the chroot, which crypttab will be created to
"""
luks_uuid = get_uuid(args, f"/dev/installp{layout['root']}")
crypttab = f"root UUID={luks_uuid} none luks\n"
open(f"{args.work}/chroot_{suffix}/tmp/crypttab", "w").write(crypttab)
pmb.chroot.root(args, ["mv", "/tmp/crypttab", "/etc/crypttab"], suffix)
def create_fstab(args, layout, suffix):
"""
Create /etc/fstab config
:param layout: partition layout from get_partition_layout()
:param suffix: of the chroot, which fstab will be created to
"""
# Do not install fstab into target rootfs when using on-device
# installer. Provide fstab only to installer suffix
if args.on_device_installer and "rootfs_" in suffix:
return
boot_dev = f"/dev/installp{layout['boot']}"
root_dev = f"/dev/installp{layout['root']}"
boot_mount_point = f"UUID={get_uuid(args, boot_dev)}"
root_mount_point = "/dev/mapper/root" if args.full_disk_encryption \
else f"UUID={get_uuid(args, root_dev)}"
boot_filesystem = args.deviceinfo["boot_filesystem"] or "ext2"
root_filesystem = pmb.install.get_root_filesystem(args)
fstab = f"""
# <file system> <mount point> <type> <options> <dump> <pass>
{root_mount_point} / {root_filesystem} defaults 0 0
{boot_mount_point} /boot {boot_filesystem} defaults 0 0
""".lstrip()
with open(f"{args.work}/chroot_{suffix}/tmp/fstab", "w") as f:
f.write(fstab)
pmb.chroot.root(args, ["mv", "/tmp/fstab", "/etc/fstab"], suffix)
def install_system_image(args, size_reserve, suffix, step, steps,
boot_label="pmOS_boot", root_label="pmOS_root",
split=False, sdcard=None):
@ -786,13 +666,12 @@ def install_system_image(args, size_reserve, suffix, step, steps,
logging.info(f"*** ({step}/{steps}) PREPARE INSTALL BLOCKDEVICE ***")
pmb.chroot.shutdown(args, True)
(size_boot, size_root) = get_subpartitions_size(args, suffix)
layout = get_partition_layout(size_reserve, args.deviceinfo["cgpt_kpart"] \
and args.install_cgpt)
layout = get_partition_layout(size_reserve, args.deviceinfo["cgpt_kpart"])
if not args.rsync:
pmb.install.blockdevice.create(args, size_boot, size_root,
size_reserve, split, sdcard)
if not split:
if args.deviceinfo["cgpt_kpart"] and args.install_cgpt:
if args.deviceinfo["cgpt_kpart"]:
pmb.install.partition_cgpt(
args, layout, size_boot, size_reserve)
else:
@ -802,22 +681,6 @@ def install_system_image(args, size_reserve, suffix, step, steps,
pmb.install.format(args, layout, boot_label, root_label, sdcard)
# Create /etc/fstab and /etc/crypttab
logging.info("(native) create /etc/fstab")
create_fstab(args, layout, suffix)
if args.full_disk_encryption:
logging.info("(native) create /etc/crypttab")
create_crypttab(args, layout, suffix)
# Run mkinitfs to pass UUIDs to cmdline
logging.info(f"({suffix}) mkinitfs")
pmb.chroot.root(args, ["mkinitfs"], suffix)
# Clean up after running mkinitfs in chroot
pmb.helpers.mount.umount_all(args, f"{args.work}/chroot_{suffix}")
pmb.helpers.run.root(args, ["rm", f"{args.work}/chroot_{suffix}/in-pmbootstrap"])
pmb.chroot.remove_mnt_pmbootstrap(args, suffix)
# Just copy all the files
logging.info(f"*** ({step + 1}/{steps}) FILL INSTALL BLOCKDEVICE ***")
copy_files_from_chroot(args, suffix)
@ -1004,7 +867,7 @@ def install_on_device_installer(args, step, steps):
"ONDEV_CHANNEL_DESCRIPTION": channel_cfg["description"],
"ONDEV_CHANNEL_MIRRORDIR_ALPINE": channel_cfg["mirrordir_alpine"],
"ONDEV_CIPHER": args.cipher,
"ONDEV_PMBOOTSTRAP_VERSION": pmb.__version__,
"ONDEV_PMBOOTSTRAP_VERSION": pmb.config.version,
"ONDEV_UI": args.ui}
pmb.chroot.root(args, ["ondev-prepare"], suffix_installer, env=env)
@ -1132,22 +995,24 @@ def create_device_rootfs(args, step, steps):
setup_keymap(args)
# Set timezone
setup_timezone(args)
arch = args.deviceinfo["arch"]
package = pmb.helpers.package.get(args, "alpine-conf", arch)
version = package["version"].split("-r")[0]
if not pmb.parse.version.check_string(version, ">=3.14.0"):
pmb.chroot.root(args, ["setup-timezone", "-z", args.timezone], suffix)
else:
pmb.chroot.root(args, ["setup-timezone", args.timezone], suffix)
# Set locale
if locale_is_set:
# 10locale-pmos.sh gets sourced before 20locale.sh from
# alpine-baselayout by /etc/profile. Since they don't override the
# locale if it exists, it warranties we have preference
line = f"export LANG=${{LANG:-{shlex.quote(args.locale)}}}"
pmb.chroot.root(args, ["sh", "-c", f"echo {shlex.quote(line)}"
" > /etc/profile.d/10locale-pmos.sh"], suffix)
pmb.chroot.root(args, ["sed", "-i",
f"s/LANG=C.UTF-8/LANG={args.locale}/",
"/etc/profile.d/locale.sh"], suffix)
# Set the hostname as the device name
setup_hostname(args)
setup_appstream(args)
disable_sshd(args)
disable_firewall(args)

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import os
import logging
@ -103,7 +103,7 @@ def format_and_mount_root(args, device, root_label, sdcard):
mkfs_root_args = ["mkfs.ext4", "-O", "^metadata_csum", "-F",
"-q", "-L", root_label]
# When we don't know the file system size before hand like
# with non-block devices, we need to explicitly set a number of
# with non-block devices, we need to explicitely set a number of
# inodes. See #1717 and #1845 for details
if not sdcard:
mkfs_root_args = mkfs_root_args + ["-N", "100000"]

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import glob
import json

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -97,15 +97,6 @@ def partition(args, layout, size_boot, size_reserve):
["set", str(layout["boot"]), "boot", "on"]
]
# Not strictly necessary if the device doesn't use EFI boot, but marking
# it as an ESP will cover all situations where the device does use EFI
# boot. Marking it as ESP is helpful for EFI fw when it's looking for EFI
# system partitions. It's assumed that setting this bit is unlikely to
# cause problems for other situations, like when using Legacy BIOS boot
# or u-boot.
if partition_type.lower() == "gpt":
commands += [["set", str(layout["boot"]), "esp", "on"]]
for command in commands:
pmb.chroot.root(args, ["parted", "-s", "/dev/install"] +
command, check=False)
@ -121,7 +112,7 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
:param size_reserve: empty partition between root and boot in MiB (pma#463)
"""
pmb.chroot.apk.install(args, ["cgpt"], build=False)
pmb.chroot.root(args, ["apk", "add", "cgpt"])
cgpt = {
'kpart_start': args.deviceinfo["cgpt_kpart_start"],
@ -145,6 +136,27 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
commands = [
["parted", "-s", "/dev/install", "mktable", "gpt"],
["cgpt", "create", "/dev/install"],
[
"cgpt", "add",
# pmOS_boot is second partition, the first will be ChromeOS kernel
# partition
"-i", str(layout["boot"]), # Partition number
"-t", "data",
"-b", boot_part_start,
"-s", s_boot,
"-l", "pmOS_boot",
"/dev/install"
],
# Mark this partition as bootable for u-boot
[
"parted",
"-s", "/dev/install",
"set", str(layout["boot"]),
"boot", "on"
],
# For some reason cgpt switches all flags to 0 after marking
# any partition as bootable, so create ChromeOS kernel partition
# only after pmOS_boot is created and marked as bootable
[
"cgpt", "add",
"-i", str(layout["kernel"]),
@ -157,23 +169,11 @@ def partition_cgpt(args, layout, size_boot, size_reserve):
"-P", "10", # Priority flag
"/dev/install"
],
[
"cgpt", "add",
# pmOS_boot is second partition, the first will be ChromeOS kernel
# partition
"-i", str(layout["boot"]), # Partition number
"-t", "efi", # Mark this partition as bootable for u-boot
"-b", boot_part_start,
"-s", s_boot,
"-l", "pmOS_boot",
"/dev/install"
],
]
dev_size = pmb.chroot.root(
args, ["blockdev", "--getsz", "/dev/install"], output_return=True)
# 33: Sec GPT table (32) + Sec GPT header (1)
root_size = str(int(dev_size) - int(s_root_start) - 33)
root_size = str(int(dev_size) - int(s_root_start) - 1024)
commands += [
[

View File

@ -1,4 +1,4 @@
# Copyright 2023 Attila Szollosi
# Copyright 2022 Attila Szollosi
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
@ -35,8 +35,7 @@ def create_zip(args, suffix):
"ISOREC": method == "heimdall-isorec",
"KERNEL_PARTLABEL": vars["$PARTITION_KERNEL"],
"INITFS_PARTLABEL": vars["$PARTITION_INITFS"],
# Name is still "SYSTEM", not "ROOTFS" in the recovery installer
"SYSTEM_PARTLABEL": vars["$PARTITION_ROOTFS"],
"SYSTEM_PARTLABEL": vars["$PARTITION_SYSTEM"],
"INSTALL_PARTITION": args.recovery_install_partition,
"CIPHER": args.cipher,
"FDE": args.full_disk_encryption,

View File

@ -1,4 +1,4 @@
# Copyright 2023 Dylan Van Assche
# Copyright 2022 Dylan Van Assche
# SPDX-License-Identifier: GPL-3.0-or-later
import logging

View File

@ -1,4 +1,4 @@
# Copyright 2023 Mark Hargreaves, Luca Weiss
# Copyright 2022 Mark Hargreaves, Luca Weiss
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -20,7 +20,7 @@ def start_nbd_server(args, ip="172.16.42.2", port=9999):
chroot = f"{args.work}/chroot_native"
rootfs_path = f"/mnt/pmbootstrap/netboot/{args.device}.img"
rootfs_path = f"/mnt/pmbootstrap-netboot/{args.device}.img"
if not os.path.exists(chroot + rootfs_path) or args.replace:
rootfs_path2 = f"/home/pmos/rootfs/{args.device}.img"
if not os.path.exists(chroot + rootfs_path2):

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
from pmb.parse.arguments import arguments
from pmb.parse._apkbuild import apkbuild

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import logging
import os
@ -320,7 +320,8 @@ def apkbuild(path, check_pkgver=True, check_pkgname=True):
# Sanity check: pkgver
if check_pkgver:
if not pmb.parse.version.validate(ret["pkgver"]):
if ("-r" in ret["pkgver"] or not
pmb.parse.version.validate(ret["pkgver"])):
logging.info(
"NOTE: Valid pkgvers are described here: "
"https://wiki.alpinelinux.org/wiki/APKBUILD_Reference#pkgver")

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import collections
import logging
@ -308,7 +308,7 @@ def providers(args, package, arch=None, must_exist=True, indexes=None):
" higher)")
continue
# Add the provider to ret
# Add the provier to ret
logging.verbose(package + ": provided by: " + provider_pkgname +
"-" + version + " in " + path)
ret[provider_pkgname] = provider
@ -339,7 +339,7 @@ def provider_highest_priority(providers, pkgname):
if priority_providers:
logging.debug(
f"{pkgname}: picked provider(s) with highest priority "
f"{pkgname}: picked provider(s) with higest priority "
f"{max_priority}: {', '.join(priority_providers.keys())}")
return priority_providers

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import fnmatch
import platform
@ -12,9 +12,8 @@ def alpine_native():
"i686": "x86",
"x86_64": "x86_64",
"aarch64": "aarch64",
"arm64": "aarch64",
"armv6l": "armhf",
"armv7l": "armv7",
"armv7l": "armv7"
}
if machine in mapping:
return mapping[machine]
@ -37,7 +36,6 @@ def from_chroot_suffix(args, suffix):
def alpine_to_qemu(arch):
"""
Convert the architecture to the string used in the QEMU packaging.
This corresponds to the package name of e.g. qemu-system-aarch64.
"""
mapping = {
@ -46,7 +44,6 @@ def alpine_to_qemu(arch):
"armhf": "arm",
"armv7": "arm",
"aarch64": "aarch64",
"riscv64": "riscv64",
}
for pattern, arch_qemu in mapping.items():
if fnmatch.fnmatch(arch, pattern):
@ -64,8 +61,7 @@ def alpine_to_kernel(arch):
"aarch64*": "arm64",
"arm*": "arm",
"ppc*": "powerpc",
"s390*": "s390",
"riscv64*": "riscv",
"s390*": "s390"
}
for pattern, arch_kernel in mapping.items():
if fnmatch.fnmatch(arch, pattern):
@ -82,9 +78,6 @@ def alpine_to_hostspec(arch):
"armel": "armv5-alpine-linux-musleabi",
"armhf": "armv6-alpine-linux-musleabihf",
"armv7": "armv7-alpine-linux-musleabihf",
"loongarch32": "loongarch32-alpine-linux-musl",
"loongarchx32": "loongarchx32-alpine-linux-musl",
"loongarch64": "loongarch64-alpine-linux-musl",
"mips": "mips-alpine-linux-musl",
"mips64": "mips64-alpine-linux-musl",
"mipsel": "mipsel-alpine-linux-musl",
@ -92,8 +85,6 @@ def alpine_to_hostspec(arch):
"ppc": "powerpc-alpine-linux-musl",
"ppc64": "powerpc64-alpine-linux-musl",
"ppc64le": "powerpc64le-alpine-linux-musl",
"riscv32": "riscv32-alpine-linux-musl",
"riscv64": "riscv64-alpine-linux-musl",
"s390x": "s390x-alpine-linux-musl",
"x86": "i586-alpine-linux-musl",
"x86_64": "x86_64-alpine-linux-musl",

View File

@ -1,4 +1,4 @@
# Copyright 2023 Oliver Smith
# Copyright 2022 Oliver Smith
# SPDX-License-Identifier: GPL-3.0-or-later
import argparse
import copy
@ -79,8 +79,6 @@ def arguments_install(subparser):
" installation - will be handled in PLAIN TEXT during"
" install and may be logged to the logfile, do not use an"
" important password!")
ret.add_argument("--no-cgpt", help="do not use cgpt partition table",
dest="install_cgpt", action="store_false", default=True)
# Image type
group_desc = ret.add_argument_group(
@ -252,13 +250,13 @@ def arguments_flasher(subparser):
" default boot image partition ")
# Flash rootfs
flash_rootfs = sub.add_parser("flash_rootfs",
flash_rootfs = sub.add_parser("flash_rootfs", aliases=["flash_system"],
help="flash the rootfs to a partition on the"
" device (partition layout does not get"
" changed)")
flash_rootfs.add_argument("--partition", default=None,
help="partition to flash the rootfs to (defaults"
" to deviceinfo_flash_*_partition_rootfs,"
" to deviceinfo_flash_*_partition_system,"
" 'userdata' on Android may have more"
" space)")
@ -368,8 +366,6 @@ def arguments_qemu(subparser):
ret.add_argument("--host-qemu", dest="host_qemu", action='store_true',
help="Use the host system's qemu")
ret.add_argument("--efi", action="store_true",
help="Use EFI boot (default: direct kernel image boot)")
return ret
@ -468,14 +464,23 @@ def arguments_kconfig(subparser):
check.add_argument("--arch", choices=arch_choices, dest="arch")
check.add_argument("--file", action="store_true", help="check a file"
" directly instead of a config in a package")
check.add_argument("--no-details", action="store_false",
dest="kconfig_check_details",
help="print one generic error per component instead of"
" listing each option that needs to be adjusted")
for name in pmb.parse.kconfig.get_all_component_names():
check.add_argument(f"--{name}", action="store_true",
dest=f"kconfig_check_{name}",
help=f"check options needed for {name} too")
check.add_argument("--waydroid", action="store_true", help="check"
" options needed for waydroid too")
check.add_argument("--iwd", action="store_true", help="check"
" options needed for iwd too")
check.add_argument("--nftables", action="store_true", help="check"
" options needed for nftables too")
check.add_argument("--containers", action="store_true",
help="check options needed for containers too")
check.add_argument("--zram", action="store_true", help="check"
" options needed for zram support too")
check.add_argument("--netboot", action="store_true", help="check"
" options needed for netbooting too")
check.add_argument("--community", action="store_true", help="check"
" options needed for various features, required for"
" community/main devices")
check.add_argument("--uefi", action="store_true", help="check"
" options needed for uefi too")
add_kernel_arg(check)
# "pmbootstrap kconfig edit"
@ -541,21 +546,6 @@ def arguments_netboot(subparser):
return ret
def arguments_ci(subparser):
ret = subparser.add_parser("ci", help="run continuous integration scripts"
" locally of git repo in current"
" directory")
script_args = ret.add_mutually_exclusive_group()
script_args.add_argument("-a", "--all", action="store_true",
help="run all scripts")
script_args.add_argument("-f", "--fast", action="store_true",
help="run fast scripts only")
ret.add_argument("scripts", nargs="*", metavar="script",
help="name of the CI script to run, depending on the git"
" repository")
return ret
def package_completer(prefix, action, parser=None, parsed_args=None):
args = parsed_args
pmb.config.merge_with_args(args)
@ -590,8 +580,8 @@ def add_packages_arg(subparser, name="packages", *args, **kwargs):
arg.completer = package_completer
def add_kernel_arg(subparser, name="package", nargs="?", *args, **kwargs):
arg = subparser.add_argument("package", nargs=nargs, help="kernel package"
def add_kernel_arg(subparser, name="package", *args, **kwargs):
arg = subparser.add_argument("package", nargs='?', help="kernel package"
" (e.g. linux-postmarketos-allwinner)")
if argcomplete:
arg.completer = kernel_completer
@ -605,7 +595,7 @@ def arguments():
# Other
parser.add_argument("-V", "--version", action="version",
version=pmb.__version__)
version=pmb.config.version)
parser.add_argument("-c", "--config", dest="config",
default=pmb.config.defaults["config"],
help="path to pmbootstrap.cfg file (default in"
@ -613,6 +603,7 @@ def arguments():
parser.add_argument("--config-channels",
help="path to channels.cfg (which is by default"
" read from pmaports.git, origin/master branch)")
parser.add_argument("-d", "--port-distccd", dest="port_distccd")
parser.add_argument("-mp", "--mirror-pmOS", dest="mirrors_postmarketos",
help="postmarketOS mirror, disable with: -mp='',"
" specify multiple with: -mp='one' -mp='two',"
@ -653,6 +644,16 @@ def arguments():
# Compiler
parser.add_argument("--no-ccache", action="store_false",
dest="ccache", help="do not cache the compiled output")
parser.add_argument("--no-crossdirect", action="store_true",
help="Don't use the new, faster 'crossdirect' method,"
" use the old 'distcc-sshd' method instead. Use"
" if crossdirect broke something. This option"
" and the legacy 'distcc-sshd' code will be"
" removed soon if no problems turn up.")
parser.add_argument("--distcc-nofallback", action="store_false",
help="when using the cross compiler via distcc fails,"
"do not fall back to compiling slowly with QEMU",
dest="distcc_fallback")
parser.add_argument("--no-cross", action="store_false", dest="cross",
help="disable cross compiler, build only with QEMU and"
" gcc (slow!)")
@ -692,14 +693,17 @@ def arguments():
arguments_newapkbuild(sub)
arguments_lint(sub)
arguments_status(sub)
arguments_ci(sub)
# Action: log
log = sub.add_parser("log", help="follow the pmbootstrap logfile")
log.add_argument("-n", "--lines", default="60",
help="count of initial output lines")
log.add_argument("-c", "--clear", help="clear the log",
action="store_true", dest="clear_log")
log_distccd = sub.add_parser(
"log_distccd",
help="follow the distccd logfile")
for action in [log, log_distccd]:
action.add_argument("-n", "--lines", default="60",
help="count of initial output lines")
action.add_argument("-c", "--clear", help="clear the log",
action="store_true", dest="clear_log")
# Action: zap
zap = sub.add_parser("zap", help="safely delete chroot folders")
@ -835,21 +839,9 @@ def arguments():
build.add_argument("-n", "--no-depends", action="store_true",
help="never build dependencies, abort instead",
dest="no_depends")
build.add_argument("--go-mod-cache", action="store_true", default=None,
help="for go packages: Usually they should bundle the"
" dependency sources instead of downloading them"
" at build time. But if they don't (e.g. with"
" pmbootstrap build --src), then this option can"
" be used to let GOMODCACHE point into"
" pmbootstrap's work dir to only download"
" dependencies once. (default: true with --src,"
" false otherwise)")
build.add_argument("--no-go-mod-cache",
action="store_false", dest="go_mod_cache", default=None,
help="don't set GOMODCACHE")
build.add_argument("--envkernel", action="store_true",
help="Create an apk package from the build output of"
" a kernel compiled locally on the host or with envkernel.sh.")
" a kernel compiled with envkernel.sh.")
add_packages_arg(build, nargs="+")
# Action: deviceinfo_parse
@ -901,9 +893,4 @@ def arguments():
setattr(args, "from_argparse", copy.deepcopy(args))
setattr(args.from_argparse, "from_argparse", args.from_argparse)
pmb.helpers.args.init(args)
if getattr(args, "go_mod_cache", None) is None:
gomodcache = True if getattr(args, "src", None) else False
setattr(args, "go_mod_cache", gomodcache)
return args

Some files were not shown because too many files have changed in this diff Show More