Only download APKINDEX for relevant architectures (#1201)
### Only download APKINDEX for relevant architectures We're downloading the APKINDEX files for all architectures supported by postmarketOS currently (x86, x86_64, armhf, aarch64). Most of the time, we only need it for the native and device arch, so this PR reduces the downloaded files to what is really necessary. ### Intuitive pmbootstrap update logic * pmb.helpers.repo.update(): * Default is updating all arches where the APKBUILD files exist * Add existing_only parameter * Return True when files have been downloaded * Properly print which arches will be updated * Print update reason only in verbose log * Add and improve comments * pmb.parse.arguments(), update action: * Add --non-existing parameter * Default for --arch is None (instead of arch.native) * pmb.helpers.frontend.update(): * Inform about --non-existing if no APKBUILDs have been updated
This commit is contained in:
parent
e6bb96426b
commit
99127111a1
|
@ -57,7 +57,7 @@ def get_apkbuild(args, pkgname, arch):
|
||||||
:returns: None or full path to APKBUILD
|
:returns: None or full path to APKBUILD
|
||||||
"""
|
"""
|
||||||
# Get existing binary package indexes
|
# Get existing binary package indexes
|
||||||
pmb.helpers.repo.update(args)
|
pmb.helpers.repo.update(args, arch)
|
||||||
|
|
||||||
# Get aport, skip upstream only packages
|
# Get aport, skip upstream only packages
|
||||||
aport = pmb.build.find_aport(args, pkgname, False)
|
aport = pmb.build.find_aport(args, pkgname, False)
|
||||||
|
@ -214,7 +214,8 @@ def get_gcc_version(args, arch):
|
||||||
<https://linux.die.net/man/1/ccache>
|
<https://linux.die.net/man/1/ccache>
|
||||||
:returns: a string like "6.4.0-r5"
|
:returns: a string like "6.4.0-r5"
|
||||||
"""
|
"""
|
||||||
return pmb.parse.apkindex.package(args, "gcc", arch)["version"]
|
return pmb.parse.apkindex.package(args, "gcc-" + arch,
|
||||||
|
args.arch_native)["version"]
|
||||||
|
|
||||||
|
|
||||||
def get_pkgver(original_pkgver, original_source=False, now=None):
|
def get_pkgver(original_pkgver, original_source=False, now=None):
|
||||||
|
|
|
@ -237,10 +237,9 @@ def upgrade(args, suffix="native"):
|
||||||
"""
|
"""
|
||||||
Upgrade all packages installed in a chroot
|
Upgrade all packages installed in a chroot
|
||||||
"""
|
"""
|
||||||
# Prepare apk and update index
|
# Update APKINDEX files
|
||||||
check_min_version(args, suffix)
|
arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
|
||||||
pmb.chroot.init(args, suffix)
|
pmb.helpers.repo.update(args, arch)
|
||||||
pmb.helpers.repo.update(args)
|
|
||||||
|
|
||||||
# Rebuild and upgrade out-of-date packages
|
# Rebuild and upgrade out-of-date packages
|
||||||
packages = installed(args, suffix).keys()
|
packages = installed(args, suffix).keys()
|
||||||
|
|
|
@ -160,7 +160,7 @@ def init(args):
|
||||||
Download, verify, extract $WORK/apk.static.
|
Download, verify, extract $WORK/apk.static.
|
||||||
"""
|
"""
|
||||||
# Get the APKINDEX
|
# Get the APKINDEX
|
||||||
pmb.helpers.repo.update(args)
|
pmb.helpers.repo.update(args, args.arch_native)
|
||||||
url = args.mirror_alpine + args.alpine_version + "/main"
|
url = args.mirror_alpine + args.alpine_version + "/main"
|
||||||
apkindex = (args.work + "/cache_apk_" + args.arch_native + "/APKINDEX." +
|
apkindex = (args.work + "/cache_apk_" + args.arch_native + "/APKINDEX." +
|
||||||
pmb.helpers.repo.hash(url) + ".tar.gz")
|
pmb.helpers.repo.hash(url) + ".tar.gz")
|
||||||
|
|
|
@ -134,7 +134,8 @@ def init(args, suffix="native"):
|
||||||
"/chroot_native/usr/bin/qemu-" + arch_debian + "-static",
|
"/chroot_native/usr/bin/qemu-" + arch_debian + "-static",
|
||||||
chroot + "/usr/bin/qemu-" + arch_debian + "-static"])
|
chroot + "/usr/bin/qemu-" + arch_debian + "-static"])
|
||||||
|
|
||||||
# Install alpine-base (no clean exit for non-native chroot!)
|
# Install alpine-base
|
||||||
|
pmb.helpers.repo.update(args, arch)
|
||||||
pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot,
|
pmb.chroot.apk_static.run(args, ["--no-progress", "--root", chroot,
|
||||||
"--cache-dir", apk_cache, "--initdb", "--arch", arch,
|
"--cache-dir", apk_cache, "--initdb", "--arch", arch,
|
||||||
"add", "alpine-base"])
|
"add", "alpine-base"])
|
||||||
|
|
|
@ -207,7 +207,14 @@ def menuconfig(args):
|
||||||
|
|
||||||
|
|
||||||
def update(args):
|
def update(args):
|
||||||
pmb.helpers.repo.update(args, True)
|
existing_only = not args.non_existing
|
||||||
|
if not pmb.helpers.repo.update(args, args.arch, True, existing_only):
|
||||||
|
logging.info("No APKINDEX files exist, so none have been updated."
|
||||||
|
" The pmbootstrap command downloads the APKINDEX files on"
|
||||||
|
" demand.")
|
||||||
|
logging.info("If you want to force downloading the APKINDEX files for"
|
||||||
|
" all architectures (not recommended), use:"
|
||||||
|
" pmbootstrap update --non-existing")
|
||||||
|
|
||||||
|
|
||||||
def newapkbuild(args):
|
def newapkbuild(args):
|
||||||
|
|
|
@ -106,25 +106,40 @@ def apkindex_files(args, arch=None):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
|
||||||
def update(args, force=False):
|
def update(args, arch=None, force=False, existing_only=False):
|
||||||
"""
|
|
||||||
Download the APKINDEX files for all URLs and architectures.
|
|
||||||
:arg force: even update when the APKINDEX file is fairly recent
|
|
||||||
"""
|
"""
|
||||||
|
Download the APKINDEX files for all URLs depending on the architectures.
|
||||||
|
|
||||||
architectures = pmb.config.build_device_architectures
|
:param arch: * one Alpine architecture name ("x86_64", "armhf", ...)
|
||||||
|
* None for all architectures
|
||||||
|
:param force: even update when the APKINDEX file is fairly recent
|
||||||
|
:param existing_only: only update the APKBUILD files that already exist,
|
||||||
|
this is used by "pmbootstrap update"
|
||||||
|
|
||||||
|
:returns: True when files have been downloaded, False otherwise
|
||||||
|
"""
|
||||||
|
# Architectures and retention time
|
||||||
|
architectures = [arch] if arch else pmb.config.build_device_architectures
|
||||||
retention_hours = pmb.config.apkindex_retention_time
|
retention_hours = pmb.config.apkindex_retention_time
|
||||||
retention_seconds = retention_hours * 3600
|
retention_seconds = retention_hours * 3600
|
||||||
|
|
||||||
|
# Find outdated APKINDEX files. Formats:
|
||||||
|
# outdated: {URL: apkindex_path, ... }
|
||||||
|
# outdated_arches: ["armhf", "x86_64", ... ]
|
||||||
outdated = {}
|
outdated = {}
|
||||||
|
outdated_arches = []
|
||||||
for url in urls(args, False):
|
for url in urls(args, False):
|
||||||
for arch in architectures:
|
for arch in architectures:
|
||||||
|
# APKINDEX file name from the URL
|
||||||
url_full = url + "/" + arch + "/APKINDEX.tar.gz"
|
url_full = url + "/" + arch + "/APKINDEX.tar.gz"
|
||||||
cache_apk_outside = args.work + "/cache_apk_" + arch
|
cache_apk_outside = args.work + "/cache_apk_" + arch
|
||||||
apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
|
apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
|
||||||
|
|
||||||
|
# Find update reason, possibly skip non-existing files
|
||||||
reason = None
|
reason = None
|
||||||
if not os.path.exists(apkindex):
|
if not os.path.exists(apkindex):
|
||||||
|
if existing_only:
|
||||||
|
continue
|
||||||
reason = "file does not exist yet"
|
reason = "file does not exist yet"
|
||||||
elif force:
|
elif force:
|
||||||
reason = "forced update"
|
reason = "forced update"
|
||||||
|
@ -133,19 +148,25 @@ def update(args, force=False):
|
||||||
if not reason:
|
if not reason:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Update outdated and outdated_arches
|
||||||
logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
|
logging.debug("APKINDEX outdated (" + reason + "): " + url_full)
|
||||||
outdated[url_full] = apkindex
|
outdated[url_full] = apkindex
|
||||||
|
if arch not in outdated_arches:
|
||||||
|
outdated_arches.append(arch)
|
||||||
|
|
||||||
|
# Bail out or show log message
|
||||||
if not len(outdated):
|
if not len(outdated):
|
||||||
return
|
return False
|
||||||
|
logging.info("Update package index for " + ", ".join(outdated_arches) +
|
||||||
|
" (" + str(len(outdated)) + " file(s))")
|
||||||
|
|
||||||
# Show one message only
|
|
||||||
logging.info("Update package index (" + str(len(outdated)) + "x)")
|
|
||||||
for url, target in outdated.items():
|
|
||||||
# Download and move to right location
|
# Download and move to right location
|
||||||
|
for url, target in outdated.items():
|
||||||
temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
|
temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
|
||||||
logging.DEBUG)
|
logging.DEBUG)
|
||||||
target_folder = os.path.dirname(target)
|
target_folder = os.path.dirname(target)
|
||||||
if not os.path.exists(target_folder):
|
if not os.path.exists(target_folder):
|
||||||
pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
|
pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])
|
||||||
pmb.helpers.run.root(args, ["cp", temp, target])
|
pmb.helpers.run.root(args, ["cp", temp, target])
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
|
@ -207,7 +207,6 @@ def arguments():
|
||||||
sub.add_parser("shutdown", help="umount, unregister binfmt")
|
sub.add_parser("shutdown", help="umount, unregister binfmt")
|
||||||
sub.add_parser("index", help="re-index all repositories with custom built"
|
sub.add_parser("index", help="re-index all repositories with custom built"
|
||||||
" packages (do this after manually removing package files)")
|
" packages (do this after manually removing package files)")
|
||||||
sub.add_parser("update", help="update all APKINDEX files")
|
|
||||||
arguments_export(sub)
|
arguments_export(sub)
|
||||||
arguments_flasher(sub)
|
arguments_flasher(sub)
|
||||||
arguments_initfs(sub)
|
arguments_initfs(sub)
|
||||||
|
@ -247,6 +246,15 @@ def arguments():
|
||||||
stats = sub.add_parser("stats", help="show ccache stats")
|
stats = sub.add_parser("stats", help="show ccache stats")
|
||||||
stats.add_argument("--arch", default=arch_native, choices=arch_choices)
|
stats.add_argument("--arch", default=arch_native, choices=arch_choices)
|
||||||
|
|
||||||
|
# Action: update
|
||||||
|
update = sub.add_parser("update", help="update all existing APKINDEX"
|
||||||
|
" files")
|
||||||
|
update.add_argument("--arch", default=None, choices=arch_choices,
|
||||||
|
help="only update a specific architecture")
|
||||||
|
update.add_argument("--non-existing", action="store_true", help="do not"
|
||||||
|
" only update the existing APKINDEX files, but all of"
|
||||||
|
" them", dest="non_existing")
|
||||||
|
|
||||||
# Action: build_init / chroot
|
# Action: build_init / chroot
|
||||||
build_init = sub.add_parser("build_init", help="initialize build"
|
build_init = sub.add_parser("build_init", help="initialize build"
|
||||||
" environment (usually you do not need to call this)")
|
" environment (usually you do not need to call this)")
|
||||||
|
|
|
@ -46,7 +46,7 @@ def test_qt_versions(args):
|
||||||
qt5-qtbase version.
|
qt5-qtbase version.
|
||||||
"""
|
"""
|
||||||
# Upstream version
|
# Upstream version
|
||||||
pmb.helpers.repo.update(args)
|
pmb.helpers.repo.update(args, "armhf")
|
||||||
repository = args.mirror_alpine + args.alpine_version + "/community"
|
repository = args.mirror_alpine + args.alpine_version + "/community"
|
||||||
hash = pmb.helpers.repo.hash(repository)
|
hash = pmb.helpers.repo.hash(repository)
|
||||||
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
||||||
|
@ -85,7 +85,7 @@ def test_aportgen_versions(args):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Get Alpine's "main" repository APKINDEX path
|
# Get Alpine's "main" repository APKINDEX path
|
||||||
pmb.helpers.repo.update(args)
|
pmb.helpers.repo.update(args, "armhf")
|
||||||
repository = args.mirror_alpine + args.alpine_version + "/main"
|
repository = args.mirror_alpine + args.alpine_version + "/main"
|
||||||
hash = pmb.helpers.repo.hash(repository)
|
hash = pmb.helpers.repo.hash(repository)
|
||||||
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
|
||||||
|
|
Loading…
Reference in New Issue