Only download APKINDEX for relevant architectures (#1201)
### Only download APKINDEX for relevant architectures We're downloading the APKINDEX files for all architectures supported by postmarketOS currently (x86, x86_64, armhf, aarch64). Most of the time, we only need it for the native and device arch, so this PR reduces the downloaded files to what is really necessary. ### Intuitive pmbootstrap update logic * pmb.helpers.repo.update(): * Default is updating all arches where the APKBUILD files exist * Add existing_only parameter * Return True when files have been downloaded * Properly print which arches will be updated * Print update reason only in verbose log * Add and improve comments * pmb.parse.arguments(), update action: * Add --non-existing parameter * Default for --arch is None (instead of arch.native) * pmb.helpers.frontend.update(): * Inform about --non-existing if no APKBUILDs have been updated
This commit is contained in:
Oliver Smith
GitHub
parent
e6bb96426b
commit
99127111a1
|
|
@ -57,7 +57,7 @@ def get_apkbuild(args, pkgname, arch):
|
|||
:returns: None or full path to APKBUILD
|
||||
"""
|
||||
# Get existing binary package indexes
|
||||
pmb.helpers.repo.update(args)
|
||||
pmb.helpers.repo.update(args, arch)
|
||||
|
||||
# Get aport, skip upstream only packages
|
||||
aport = pmb.build.find_aport(args, pkgname, False)
|
||||
|
|
@ -214,7 +214,8 @@ def get_gcc_version(args, arch):
|
|||
<https://linux.die.net/man/1/ccache>
|
||||
:returns: a string like "6.4.0-r5"
|
||||
"""
|
||||
return pmb.parse.apkindex.package(args, "gcc", arch)["version"]
|
||||
return pmb.parse.apkindex.package(args, "gcc-" + arch,
|
||||
args.arch_native)["version"]
|
||||
|
||||
|
||||
def get_pkgver(original_pkgver, original_source=False, now=None):
|
||||
|
|
|
|||
|
|
@ -237,10 +237,9 @@ def upgrade(args, suffix="native"):
|
|||
"""
|
||||
Upgrade all packages installed in a chroot
|
||||
"""
|
||||
# Prepare apk and update index
|
||||
check_min_version(args, suffix)
|
||||
pmb.chroot.init(args, suffix)
|
||||
pmb.helpers.repo.update(args)
|
||||
# Update APKINDEX files
|
||||
arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
|
||||
pmb.helpers.repo.update(args, arch)
|
||||
|
||||
# Rebuild and upgrade out-of-date packages
|
||||
packages = installed(args, suffix).keys()
|
||||
|
|
|
|||
|
|
@ -160,7 +160,7 @@ def init(args):
|
|||
Download, verify, extract $WORK/apk.static.
|
||||
"""
|
||||
# Get the APKINDEX
|
||||
pmb.helpers.repo.update(args)
|
||||
pmb.helpers.repo.update(args, args.arch_native)
|
||||
url = args.mirror_alpine + args.alpine_version + "/main"
|
||||
apkindex = (args.work + "/cache_apk_" + args.arch_native + "/APKINDEX." +
|
||||
pmb.helpers.repo.hash(url) + ".tar.gz")
|
||||
|
|
|
|||
|
|
@ -134,7 +134,8 @@ def init(args, suffix="native"):
|
|||
"/chroot_native/usr/bin/qemu-" + arch_debian + "-static",
|
||||
chroot + "/usr/bin/qemu-" + arch_debian + "-static"])
|
||||
|
||||
# Install alpine-base (no clean exit for non-native chroot!)
|
||||
# Install alpine-base
|
||||
pmb.helpers.repo.update(args, arch)
|
||||
pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot,
|
||||
"--cache-dir", apk_cache, "--initdb", "--arch", arch,
|
||||
"add", "alpine-base"])
|
||||
|
|
|
|||
|
|
@ -207,7 +207,14 @@ def menuconfig(args):
|
|||
|
||||
|
||||
def update(args):
|
||||
pmb.helpers.repo.update(args, True)
|
||||
existing_only = not args.non_existing
|
||||
if not pmb.helpers.repo.update(args, args.arch, True, existing_only):
|
||||
logging.info("No APKINDEX files exist, so none have been updated."
|
||||
" The pmbootstrap command downloads the APKINDEX files on"
|
||||
" demand.")
|
||||
logging.info("If you want to force downloading the APKINDEX files for"
|
||||
" all architectures (not recommended), use:"
|
||||
" pmbootstrap update --non-existing")
|
||||
|
||||
|
||||
def newapkbuild(args):
|
||||
|
|
|
|||
|
|
@ -106,25 +106,40 @@ def apkindex_files(args, arch=None):
|
|||
return ret
|
||||
|
||||
|
||||
def update(args, force=False):
|
||||
"""
|
||||
Download the APKINDEX files for all URLs and architectures.
|
||||
:arg force: even update when the APKINDEX file is fairly recent
|
||||
def update(args, arch=None, force=False, existing_only=False):
|
||||
"""
|
||||
Download the APKINDEX files for all URLs depending on the architectures.
|
||||
|
||||
architectures = pmb.config.build_device_architectures
|
||||
:param arch: * one Alpine architecture name ("x86_64", "armhf", ...)
|
||||
* None for all architectures
|
||||
:param force: even update when the APKINDEX file is fairly recent
|
||||
:param existing_only: only update the APKBUILD files that already exist,
|
||||
this is used by "pmbootstrap update"
|
||||
|
||||
:returns: True when files have been downloaded, False otherwise
|
||||
"""
|
||||
# Architectures and retention time
|
||||
architectures = [arch] if arch else pmb.config.build_device_architectures
|
||||
retention_hours = pmb.config.apkindex_retention_time
|
||||
retention_seconds = retention_hours * 3600
|
||||
|
||||
# Find outdated APKINDEX files. Formats:
|
||||
# outdated: {URL: apkindex_path, ... }
|
||||
# outdated_arches: ["armhf", "x86_64", ... ]
|
||||
outdated = {}
|
||||
outdated_arches = []
|
||||
for url in urls(args, False):
|
||||
for arch in architectures:
|
||||
# APKINDEX file name from the URL
|
||||
url_full = url + "/" + arch + "/APKINDEX.tar.gz"
|
||||
cache_apk_outside = args.work + "/cache_apk_" + arch
|
||||
apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
|
||||
|
||||
# Find update reason, possibly skip non-existing files
|
||||
reason = None
|
||||
if not os.path.exists(apkindex):
|
||||
if existing_only:
|
||||
continue
|
||||
reason = "file does not exist yet"
|
||||
elif force:
|
||||
reason = "forced update"
|
||||
|
|
@ -133,19 +148,25 @@ def update(args, force=False):
|
|||
if not reason:
|
||||
continue
|
||||
|
||||
# Update outdated and outdated_arches
|
||||
logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
|
||||
outdated[url_full] = apkindex
|
||||
if arch not in outdated_arches:
|
||||
outdated_arches.append(arch)
|
||||
|
||||
# Bail out or show log message
|
||||
if not len(outdated):
|
||||
return
|
||||
return False
|
||||
logging.info("Update package index for " + ", ".join(outdated_arches) +
|
||||
" (" + str(len(outdated)) + " file(s))")
|
||||
|
||||
# Show one message only
|
||||
logging.info("Update package index (" + str(len(outdated)) + "x)")
|
||||
# Download and move to right location
|
||||
for url, target in outdated.items():
|
||||
# Download and move to right location
|
||||
temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
|
||||
logging.DEBUG)
|
||||
target_folder = os.path.dirname(target)
|
||||
if not os.path.exists(target_folder):
|
||||
pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
|
||||
pmb.helpers.run.root(args, ["cp", temp, target])
|
||||
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -207,7 +207,6 @@ def arguments():
|
|||
sub.add_parser("shutdown", help="umount, unregister binfmt")
|
||||
sub.add_parser("index", help="re-index all repositories with custom built"
|
||||
" packages (do this after manually removing package files)")
|
||||
sub.add_parser("update", help="update all APKINDEX files")
|
||||
arguments_export(sub)
|
||||
arguments_flasher(sub)
|
||||
arguments_initfs(sub)
|
||||
|
|
@ -247,6 +246,15 @@ def arguments():
|
|||
stats = sub.add_parser("stats", help="show ccache stats")
|
||||
stats.add_argument("--arch", default=arch_native, choices=arch_choices)
|
||||
|
||||
# Action: update
|
||||
update = sub.add_parser("update", help="update all existing APKINDEX"
|
||||
" files")
|
||||
update.add_argument("--arch", default=None, choices=arch_choices,
|
||||
help="only update a specific architecture")
|
||||
update.add_argument("--non-existing", action="store_true", help="do not"
|
||||
" only update the existing APKINDEX files, but all of"
|
||||
" them", dest="non_existing")
|
||||
|
||||
# Action: build_init / chroot
|
||||
build_init = sub.add_parser("build_init", help="initialize build"
|
||||
" environment (usually you do not need to call this)")
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ def test_qt_versions(args):
|
|||
qt5-qtbase version.
|
||||
"""
|
||||
# Upstream version
|
||||
pmb.helpers.repo.update(args)
|
||||
pmb.helpers.repo.update(args, "armhf")
|
||||
repository = args.mirror_alpine + args.alpine_version + "/community"
|
||||
hash = pmb.helpers.repo.hash(repository)
|
||||
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
||||
|
|
@ -85,7 +85,7 @@ def test_aportgen_versions(args):
|
|||
"""
|
||||
|
||||
# Get Alpine's "main" repository APKINDEX path
|
||||
pmb.helpers.repo.update(args)
|
||||
pmb.helpers.repo.update(args, "armhf")
|
||||
repository = args.mirror_alpine + args.alpine_version + "/main"
|
||||
hash = pmb.helpers.repo.hash(repository)
|
||||
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
||||
|
|
|
|||
Loading…
Reference in New Issue