synit
/
pmbootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

pmb.config: mount go caching directories 

Always set GOCACHE during build and let it point to pmbootstrap's work
dir. This has a similar effect as using ccache for C/C++.

Set GOMODCACHE conditionally: this is for caching the go modules (git
repositories) that get cloned during a build if they are not bundled
with the source. Usually APKBUILDs should cache them, but when using
pmbootstrap build --src, they would get downloaded over and over again.
Set GOMODCACHE automatically for --src and allow enabling/disabling this
manually with new --go-mod-cache / --no-go-mod-cache.

This speeds up multiple iterations of building the same package
significantly. I'm using it for:

$ pmbootstrap build postmarketos-mkinitfs --src=...

I've verified that using the same GOCACHE dir for multiple architectures
doesn't result in build artifacts for the wrong arch getting used.

Reviewed-by: Clayton Craft <clayton@craftyguy.net>
Link: https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%3C20230223064743.1773-1-ollieparanoid@postmarketos.org%3E
This commit is contained in:
Oliver Smith 2023-02-23 07:47:43 +01:00
parent 3ad159f98d
commit a0cea6255a
No known key found for this signature in database
GPG Key ID: 5AE7F5513E0885CB
4 changed files with 41 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pmb/build/_package.py
View File

@ -436,6 +436,18 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
if not args.ccache:
env["CCACHE_DISABLE"] = "1"
# Cache binary objects from go in this path (like ccache)
env["GOCACHE"] = "/home/pmos/.cache/go-build"
# Cache go modules (git repositories). Usually these should be bundled and
# it should not be required to download them at build time, in that case
# the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But
# e.g. when using --src they are not bundled, in that case it makes sense
# to point GOMODCACHE at pmbootstrap's work dir so the modules are only
# downloaded once.
if args.go_mod_cache:
env["GOMODCACHE"] = "/home/pmos/go/pkg/mod"
# Build the abuild command
cmd = ["abuild", "-D", "postmarketOS"]
if strict or "pmb:strict" in apkbuild["options"]:

7
pmb/config/__init__.py
View File

@ -219,6 +219,7 @@ chroot_mount_bind = {
"$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap-ccache",
"$WORK/cache_distfiles": "/var/cache/distfiles",
"$WORK/cache_git": "/mnt/pmbootstrap-git",
"$WORK/cache_go": "/mnt/pmbootstrap-go",
"$WORK/cache_rust": "/mnt/pmbootstrap-rust",
"$WORK/config_abuild": "/mnt/pmbootstrap-abuild-config",
"$WORK/config_apk_keys": "/etc/apk/keys",
@ -234,10 +235,16 @@ chroot_mount_bind = {
# a no-go, but at least until this is resolved properly, let's cache the
# dependencies and downloads as suggested in "Caching the Cargo home in CI":
# https://doc.rust-lang.org/cargo/guide/cargo-home.html
# Go: cache the directories "go env GOMODCACHE" and "go env GOCACHE" point to,
# to avoid downloading dependencies over and over (GOMODCACHE, similar to the
# rust depends caching described above) and to cache build artifacts (GOCACHE,
# similar to ccache).
chroot_home_symlinks = {
"/mnt/pmbootstrap-abuild-config": "/home/pmos/.abuild",
"/mnt/pmbootstrap-ccache": "/home/pmos/.ccache",
"/mnt/pmbootstrap-packages": "/home/pmos/packages/pmos",
"/mnt/pmbootstrap-go/gocache": "/home/pmos/.cache/go-build",
"/mnt/pmbootstrap-go/gomodcache": "/home/pmos/go/pkg/mod",
"/mnt/pmbootstrap-rust/registry/index": "/home/pmos/.cargo/registry/index",
"/mnt/pmbootstrap-rust/registry/cache": "/home/pmos/.cargo/registry/cache",
"/mnt/pmbootstrap-rust/git/db": "/home/pmos/.cargo/git/db",

17
pmb/parse/arguments.py
View File

@ -855,6 +855,18 @@ def arguments():
build.add_argument("-n", "--no-depends", action="store_true",
help="never build dependencies, abort instead",
dest="no_depends")
build.add_argument("--go-mod-cache", action="store_true", default=None,
help="for go packages: Usually they should bundle the"
" dependency sources instead of downloading them"
" at build time. But if they don't (e.g. with"
" pmbootstrap build --src), then this option can"
" be used to let GOMODCACHE point into"
" pmbootstrap's work dir to only download"
" dependencies once. (default: true with --src,"
" false otherwise)")
build.add_argument("--no-go-mod-cache",
action="store_false", dest="go_mod_cache", default=None,
help="don't set GOMODCACHE")
build.add_argument("--envkernel", action="store_true",
help="Create an apk package from the build output of"
" a kernel compiled locally on the host or with envkernel.sh.")
@ -909,4 +921,9 @@ def arguments():
setattr(args, "from_argparse", copy.deepcopy(args))
setattr(args.from_argparse, "from_argparse", args.from_argparse)
pmb.helpers.args.init(args)
if getattr(args, "go_mod_cache", None) is None:
gomodcache = True if getattr(args, "src", None) else False
setattr(args, "go_mod_cache", gomodcache)
return args

6
test/test_build_package.py
View File

@ -273,7 +273,9 @@ def test_run_abuild(args, monkeypatch):
# Normal run
output = "armhf/test-1-r2.apk"
env = {"CARCH": "armhf", "SUDO_APK": "abuild-apk --no-progress"}
env = {"CARCH": "armhf",
"GOCACHE": "/home/pmos/.cache/go-build",
"SUDO_APK": "abuild-apk --no-progress"}
cmd = ["abuild", "-D", "postmarketOS", "-d"]
assert func(args, apkbuild, "armhf") == (output, cmd, env)
@ -283,6 +285,7 @@ def test_run_abuild(args, monkeypatch):
# cross=native
env = {"CARCH": "armhf",
"GOCACHE": "/home/pmos/.cache/go-build",
"SUDO_APK": "abuild-apk --no-progress",
"CROSS_COMPILE": "armv6-alpine-linux-musleabihf-",
"CC": "armv6-alpine-linux-musleabihf-gcc"}
@ -293,6 +296,7 @@ def test_run_abuild(args, monkeypatch):
(output, cmd, env) = func(args, apkbuild, "armhf", cross="distcc")
assert output == "armhf/test-1-r2.apk"
assert env["CARCH"] == "armhf"
assert env["GOCACHE"] == "/home/pmos/.cache/go-build"
assert env["CCACHE_PREFIX"] == "distcc"
assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin"
assert env["CCACHE_COMPILERCHECK"].startswith("string:")