From a0cea6255ac8fd894fa19f822a8cfc3f33a8c6cb Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 23 Feb 2023 07:47:43 +0100 Subject: [PATCH] pmb.config: mount go caching directories Always set GOCACHE during build and let it point to pmbootstrap's work dir. This has a similar effect as using ccache for C/C++. Set GOMODCACHE conditionally: this is for caching the go modules (git repositories) that get cloned during a build if they are not bundled with the source. Usually APKBUILDs should cache them, but when using pmbootstrap build --src, they would get downloaded over and over again. Set GOMODCACHE automatically for --src and allow enabling/disabling this manually with new --go-mod-cache / --no-go-mod-cache. This speeds up multiple iterations of building the same package significantly. I'm using it for: $ pmbootstrap build postmarketos-mkinitfs --src=... I've verified that using the same GOCACHE dir for multiple architectures doesn't result in build artifacts for the wrong arch getting used. Reviewed-by: Clayton Craft Link: https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%3C20230223064743.1773-1-ollieparanoid@postmarketos.org%3E --- pmb/build/_package.py | 12 ++++++++++++ pmb/config/__init__.py | 7 +++++++ pmb/parse/arguments.py | 17 +++++++++++++++++ test/test_build_package.py | 6 +++++- 4 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pmb/build/_package.py b/pmb/build/_package.py index 751a1b77..79c6a1a9 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -436,6 +436,18 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, if not args.ccache: env["CCACHE_DISABLE"] = "1" + # Cache binary objects from go in this path (like ccache) + env["GOCACHE"] = "/home/pmos/.cache/go-build" + + # Cache go modules (git repositories). Usually these should be bundled and + # it should not be required to download them at build time, in that case + # the APKBUILD sets the GOPATH (and therefore indirectly GOMODCACHE). But + # e.g. when using --src they are not bundled, in that case it makes sense + # to point GOMODCACHE at pmbootstrap's work dir so the modules are only + # downloaded once. + if args.go_mod_cache: + env["GOMODCACHE"] = "/home/pmos/go/pkg/mod" + # Build the abuild command cmd = ["abuild", "-D", "postmarketOS"] if strict or "pmb:strict" in apkbuild["options"]: diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 98a7e462..f4679914 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -219,6 +219,7 @@ chroot_mount_bind = { "$WORK/cache_ccache_$ARCH": "/mnt/pmbootstrap-ccache", "$WORK/cache_distfiles": "/var/cache/distfiles", "$WORK/cache_git": "/mnt/pmbootstrap-git", + "$WORK/cache_go": "/mnt/pmbootstrap-go", "$WORK/cache_rust": "/mnt/pmbootstrap-rust", "$WORK/config_abuild": "/mnt/pmbootstrap-abuild-config", "$WORK/config_apk_keys": "/etc/apk/keys", @@ -234,10 +235,16 @@ chroot_mount_bind = { # a no-go, but at least until this is resolved properly, let's cache the # dependencies and downloads as suggested in "Caching the Cargo home in CI": # https://doc.rust-lang.org/cargo/guide/cargo-home.html +# Go: cache the directories "go env GOMODCACHE" and "go env GOCACHE" point to, +# to avoid downloading dependencies over and over (GOMODCACHE, similar to the +# rust depends caching described above) and to cache build artifacts (GOCACHE, +# similar to ccache). chroot_home_symlinks = { "/mnt/pmbootstrap-abuild-config": "/home/pmos/.abuild", "/mnt/pmbootstrap-ccache": "/home/pmos/.ccache", "/mnt/pmbootstrap-packages": "/home/pmos/packages/pmos", + "/mnt/pmbootstrap-go/gocache": "/home/pmos/.cache/go-build", + "/mnt/pmbootstrap-go/gomodcache": "/home/pmos/go/pkg/mod", "/mnt/pmbootstrap-rust/registry/index": "/home/pmos/.cargo/registry/index", "/mnt/pmbootstrap-rust/registry/cache": "/home/pmos/.cargo/registry/cache", "/mnt/pmbootstrap-rust/git/db": "/home/pmos/.cargo/git/db", diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index e934b791..15c6c4df 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -855,6 +855,18 @@ def arguments(): build.add_argument("-n", "--no-depends", action="store_true", help="never build dependencies, abort instead", dest="no_depends") + build.add_argument("--go-mod-cache", action="store_true", default=None, + help="for go packages: Usually they should bundle the" + " dependency sources instead of downloading them" + " at build time. But if they don't (e.g. with" + " pmbootstrap build --src), then this option can" + " be used to let GOMODCACHE point into" + " pmbootstrap's work dir to only download" + " dependencies once. (default: true with --src," + " false otherwise)") + build.add_argument("--no-go-mod-cache", + action="store_false", dest="go_mod_cache", default=None, + help="don't set GOMODCACHE") build.add_argument("--envkernel", action="store_true", help="Create an apk package from the build output of" " a kernel compiled locally on the host or with envkernel.sh.") @@ -909,4 +921,9 @@ def arguments(): setattr(args, "from_argparse", copy.deepcopy(args)) setattr(args.from_argparse, "from_argparse", args.from_argparse) pmb.helpers.args.init(args) + + if getattr(args, "go_mod_cache", None) is None: + gomodcache = True if getattr(args, "src", None) else False + setattr(args, "go_mod_cache", gomodcache) + return args diff --git a/test/test_build_package.py b/test/test_build_package.py index b90c3946..0d18b793 100644 --- a/test/test_build_package.py +++ b/test/test_build_package.py @@ -273,7 +273,9 @@ def test_run_abuild(args, monkeypatch): # Normal run output = "armhf/test-1-r2.apk" - env = {"CARCH": "armhf", "SUDO_APK": "abuild-apk --no-progress"} + env = {"CARCH": "armhf", + "GOCACHE": "/home/pmos/.cache/go-build", + "SUDO_APK": "abuild-apk --no-progress"} cmd = ["abuild", "-D", "postmarketOS", "-d"] assert func(args, apkbuild, "armhf") == (output, cmd, env) @@ -283,6 +285,7 @@ def test_run_abuild(args, monkeypatch): # cross=native env = {"CARCH": "armhf", + "GOCACHE": "/home/pmos/.cache/go-build", "SUDO_APK": "abuild-apk --no-progress", "CROSS_COMPILE": "armv6-alpine-linux-musleabihf-", "CC": "armv6-alpine-linux-musleabihf-gcc"} @@ -293,6 +296,7 @@ def test_run_abuild(args, monkeypatch): (output, cmd, env) = func(args, apkbuild, "armhf", cross="distcc") assert output == "armhf/test-1-r2.apk" assert env["CARCH"] == "armhf" + assert env["GOCACHE"] == "/home/pmos/.cache/go-build" assert env["CCACHE_PREFIX"] == "distcc" assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin" assert env["CCACHE_COMPILERCHECK"].startswith("string:")