From 99127111a1b08de11df0b4a8386348535ae1b825 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Sun, 4 Mar 2018 13:44:27 +0000 Subject: [PATCH] Only download APKINDEX for relevant architectures (#1201) ### Only download APKINDEX for relevant architectures We're downloading the APKINDEX files for all architectures supported by postmarketOS currently (x86, x86_64, armhf, aarch64). Most of the time, we only need it for the native and device arch, so this PR reduces the downloaded files to what is really necessary. ### Intuitive pmbootstrap update logic * pmb.helpers.repo.update(): * Default is updating all arches where the APKBUILD files exist * Add existing_only parameter * Return True when files have been downloaded * Properly print which arches will be updated * Print update reason only in verbose log * Add and improve comments * pmb.parse.arguments(), update action: * Add --non-existing parameter * Default for --arch is None (instead of arch.native) * pmb.helpers.frontend.update(): * Inform about --non-existing if no APKBUILDs have been updated --- pmb/build/_package.py | 5 ++-- pmb/chroot/apk.py | 7 +++--- pmb/chroot/apk_static.py | 2 +- pmb/chroot/init.py | 3 ++- pmb/helpers/frontend.py | 9 ++++++- pmb/helpers/repo.py | 39 ++++++++++++++++++++++------- pmb/parse/arguments.py | 10 +++++++- test/test_upstream_compatibility.py | 4 +-- 8 files changed, 58 insertions(+), 21 deletions(-) diff --git a/pmb/build/_package.py b/pmb/build/_package.py index c89c7f64..3effe799 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -57,7 +57,7 @@ def get_apkbuild(args, pkgname, arch): :returns: None or full path to APKBUILD """ # Get existing binary package indexes - pmb.helpers.repo.update(args) + pmb.helpers.repo.update(args, arch) # Get aport, skip upstream only packages aport = pmb.build.find_aport(args, pkgname, False) @@ -214,7 +214,8 @@ def get_gcc_version(args, arch): :returns: a string like "6.4.0-r5" """ - return pmb.parse.apkindex.package(args, "gcc", arch)["version"] + return pmb.parse.apkindex.package(args, "gcc-" + arch, + args.arch_native)["version"] def get_pkgver(original_pkgver, original_source=False, now=None): diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index f7ab02ce..6b3b6457 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -237,10 +237,9 @@ def upgrade(args, suffix="native"): """ Upgrade all packages installed in a chroot """ - # Prepare apk and update index - check_min_version(args, suffix) - pmb.chroot.init(args, suffix) - pmb.helpers.repo.update(args) + # Update APKINDEX files + arch = pmb.parse.arch.from_chroot_suffix(args, suffix) + pmb.helpers.repo.update(args, arch) # Rebuild and upgrade out-of-date packages packages = installed(args, suffix).keys() diff --git a/pmb/chroot/apk_static.py b/pmb/chroot/apk_static.py index 609f1a4e..9041f6eb 100644 --- a/pmb/chroot/apk_static.py +++ b/pmb/chroot/apk_static.py @@ -160,7 +160,7 @@ def init(args): Download, verify, extract $WORK/apk.static. """ # Get the APKINDEX - pmb.helpers.repo.update(args) + pmb.helpers.repo.update(args, args.arch_native) url = args.mirror_alpine + args.alpine_version + "/main" apkindex = (args.work + "/cache_apk_" + args.arch_native + "/APKINDEX." + pmb.helpers.repo.hash(url) + ".tar.gz") diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index d44a0ea7..a4bfd9d8 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -134,7 +134,8 @@ def init(args, suffix="native"): "/chroot_native/usr/bin/qemu-" + arch_debian + "-static", chroot + "/usr/bin/qemu-" + arch_debian + "-static"]) - # Install alpine-base (no clean exit for non-native chroot!) + # Install alpine-base + pmb.helpers.repo.update(args, arch) pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot, "--cache-dir", apk_cache, "--initdb", "--arch", arch, "add", "alpine-base"]) diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index dce690b9..c2d68f05 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -207,7 +207,14 @@ def menuconfig(args): def update(args): - pmb.helpers.repo.update(args, True) + existing_only = not args.non_existing + if not pmb.helpers.repo.update(args, args.arch, True, existing_only): + logging.info("No APKINDEX files exist, so none have been updated." + " The pmbootstrap command downloads the APKINDEX files on" + " demand.") + logging.info("If you want to force downloading the APKINDEX files for" + " all architectures (not recommended), use:" + " pmbootstrap update --non-existing") def newapkbuild(args): diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 2811829b..95201220 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -106,25 +106,40 @@ def apkindex_files(args, arch=None): return ret -def update(args, force=False): - """ - Download the APKINDEX files for all URLs and architectures. - :arg force: even update when the APKINDEX file is fairly recent +def update(args, arch=None, force=False, existing_only=False): """ + Download the APKINDEX files for all URLs depending on the architectures. - architectures = pmb.config.build_device_architectures + :param arch: * one Alpine architecture name ("x86_64", "armhf", ...) + * None for all architectures + :param force: even update when the APKINDEX file is fairly recent + :param existing_only: only update the APKBUILD files that already exist, + this is used by "pmbootstrap update" + + :returns: True when files have been downloaded, False otherwise + """ + # Architectures and retention time + architectures = [arch] if arch else pmb.config.build_device_architectures retention_hours = pmb.config.apkindex_retention_time retention_seconds = retention_hours * 3600 + # Find outdated APKINDEX files. Formats: + # outdated: {URL: apkindex_path, ... } + # outdated_arches: ["armhf", "x86_64", ... ] outdated = {} + outdated_arches = [] for url in urls(args, False): for arch in architectures: + # APKINDEX file name from the URL url_full = url + "/" + arch + "/APKINDEX.tar.gz" cache_apk_outside = args.work + "/cache_apk_" + arch apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz" + # Find update reason, possibly skip non-existing files reason = None if not os.path.exists(apkindex): + if existing_only: + continue reason = "file does not exist yet" elif force: reason = "forced update" @@ -133,19 +148,25 @@ def update(args, force=False): if not reason: continue + # Update outdated and outdated_arches logging.debug("APKINDEX outdated (" + reason + "): " + url_full) outdated[url_full] = apkindex + if arch not in outdated_arches: + outdated_arches.append(arch) + # Bail out or show log message if not len(outdated): - return + return False + logging.info("Update package index for " + ", ".join(outdated_arches) + + " (" + str(len(outdated)) + " file(s))") - # Show one message only - logging.info("Update package index (" + str(len(outdated)) + "x)") + # Download and move to right location for url, target in outdated.items(): - # Download and move to right location temp = pmb.helpers.http.download(args, url, "APKINDEX", False, logging.DEBUG) target_folder = os.path.dirname(target) if not os.path.exists(target_folder): pmb.helpers.run.root(args, ["mkdir", "-p", target_folder]) pmb.helpers.run.root(args, ["cp", temp, target]) + + return True diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index f4e76cf9..65f686d0 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -207,7 +207,6 @@ def arguments(): sub.add_parser("shutdown", help="umount, unregister binfmt") sub.add_parser("index", help="re-index all repositories with custom built" " packages (do this after manually removing package files)") - sub.add_parser("update", help="update all APKINDEX files") arguments_export(sub) arguments_flasher(sub) arguments_initfs(sub) @@ -247,6 +246,15 @@ def arguments(): stats = sub.add_parser("stats", help="show ccache stats") stats.add_argument("--arch", default=arch_native, choices=arch_choices) + # Action: update + update = sub.add_parser("update", help="update all existing APKINDEX" + " files") + update.add_argument("--arch", default=None, choices=arch_choices, + help="only update a specific architecture") + update.add_argument("--non-existing", action="store_true", help="do not" + " only update the existing APKINDEX files, but all of" + " them", dest="non_existing") + # Action: build_init / chroot build_init = sub.add_parser("build_init", help="initialize build" " environment (usually you do not need to call this)") diff --git a/test/test_upstream_compatibility.py b/test/test_upstream_compatibility.py index e428ffb5..bf27582f 100644 --- a/test/test_upstream_compatibility.py +++ b/test/test_upstream_compatibility.py @@ -46,7 +46,7 @@ def test_qt_versions(args): qt5-qtbase version. """ # Upstream version - pmb.helpers.repo.update(args) + pmb.helpers.repo.update(args, "armhf") repository = args.mirror_alpine + args.alpine_version + "/community" hash = pmb.helpers.repo.hash(repository) index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash + @@ -85,7 +85,7 @@ def test_aportgen_versions(args): """ # Get Alpine's "main" repository APKINDEX path - pmb.helpers.repo.update(args) + pmb.helpers.repo.update(args, "armhf") repository = args.mirror_alpine + args.alpine_version + "/main" hash = pmb.helpers.repo.hash(repository) index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +