From e72afc53fd9cb9e8f9d6dc0c1ee03848a1dc93d0 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Sat, 10 Mar 2018 12:08:02 +0000 Subject: [PATCH] pmbootstrap zap -m: delete outdated packages too (#1306) `-m` is for deleting local compiled packages, for which there is no aport with the same version. Prior to this change, this only worked for packages where no aport exists, or for packages that are newer than the aports. That is, because we used the usual APKINDEX parsing logic, which ignores old packages in the APKINDEX and only returns the one with the highest version (that makes sense during dependency resolution). Changes: * New `pmb.parse.apkindex.parse_blocks()` function that returns a raw list of blocks, instead of the dict with removed duplicates with lower version you get from the usual `.parse()` function. * Renamed each of the zap flags and their descriptions to make clear what they are doing now. ``` short long (old) long (new) -p --packages --pkgs-local -m --mismatch-bins --pkgs-local-mismatch -o, --old-bins --pkgs-online-mismatch ``` --- pmb/chroot/zap.py | 41 ++++++++++++++++++++--------------------- pmb/helpers/frontend.py | 7 ++++--- pmb/parse/apkindex.py | 29 +++++++++++++++++++++++++++++ pmb/parse/arguments.py | 22 +++++++++++++--------- 4 files changed, 66 insertions(+), 33 deletions(-) diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index 2dcdcd58..0913f824 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -23,20 +23,21 @@ import os import pmb.chroot import pmb.helpers.run +import pmb.parse.apkindex -def zap(args, confirm=True, dry=False, packages=False, http=False, - mismatch_bins=False, old_bins=False, distfiles=False): +def zap(args, confirm=True, dry=False, pkgs_local=False, http=False, + pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False): """ Shutdown everything inside the chroots (e.g. distccd, adb), umount everything and then safely remove folders from the work-directory. :param dry: Only show what would be deleted, do not delete for real - :param packages: Remove *all* self-compiled packages (!) + :param pkgs_local: Remove *all* self-compiled packages (!) :param http: Clear the http cache (used e.g. for the initial apk download) - :param mismatch_bins: Remove the packages, that have a different version + :param pkgs_local_mismatch: Remove the packages, that have a different version compared to what is in the aports folder. - :param old_bins: Clean out outdated binary packages downloaded from + :param pkgs_online_mismatch: Clean out outdated binary packages downloaded from mirrors (e.g. from Alpine) :param distfiles: Clear the downloaded files cache @@ -50,12 +51,12 @@ def zap(args, confirm=True, dry=False, packages=False, http=False, size_old = pmb.helpers.other.folder_size(args, args.work) # Delete packages with a different version compared to aports, then re-index - if mismatch_bins: - zap_mismatch_bins(args, confirm, dry) + if pkgs_local_mismatch: + zap_pkgs_local_mismatch(args, confirm, dry) # Delete outdated binary packages - if old_bins: - zap_old_bins(args, confirm, dry) + if pkgs_online_mismatch: + zap_pkgs_online_mismatch(args, confirm, dry) pmb.chroot.shutdown(args) @@ -65,7 +66,7 @@ def zap(args, confirm=True, dry=False, packages=False, http=False, "chroot_buildroot_*", "chroot_rootfs_*", ] - if packages: + if pkgs_local: patterns += ["packages"] if http: patterns += ["cache_http"] @@ -94,7 +95,7 @@ def zap(args, confirm=True, dry=False, packages=False, http=False, logging.info("Cleared up ~" + str(math.ceil(mb)) + " MB of space") -def zap_mismatch_bins(args, confirm=True, dry=False): +def zap_pkgs_local_mismatch(args, confirm=True, dry=False): if not os.path.exists(args.work + "/packages/"): return if confirm and not pmb.helpers.cli.confirm(args, "Remove packages that are newer than" @@ -103,15 +104,13 @@ def zap_mismatch_bins(args, confirm=True, dry=False): reindex = False for apkindex_path in glob.glob(args.work + "/packages/*/APKINDEX.tar.gz"): - apkindex = pmb.parse.apkindex.parse(args, apkindex_path, False) - for pkgname, bin_data in apkindex.items(): - # Only real packages have apks, provided packages do not exist - # (e.g. "so:libtest.so.1.2") - if pkgname != bin_data["pkgname"]: - continue - origin = bin_data["origin"] - version = bin_data["version"] - arch = bin_data["arch"] + # Delete packages without same version in aports + blocks = pmb.parse.apkindex.parse_blocks(args, apkindex_path) + for block in blocks: + pkgname = block["pkgname"] + origin = block["origin"] + version = block["version"] + arch = block["arch"] # Apk path apk_path_short = arch + "/" + pkgname + "-" + version + ".apk" @@ -145,7 +144,7 @@ def zap_mismatch_bins(args, confirm=True, dry=False): pmb.build.other.index_repo(args) -def zap_old_bins(args, confirm=True, dry=False): +def zap_pkgs_online_mismatch(args, confirm=True, dry=False): # Check whether we need to do anything paths = glob.glob(args.work + "/cache_apk_*") if not len(paths): diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index c2d68f05..b5c28c1f 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -324,9 +324,10 @@ def log_distccd(args): def zap(args): - pmb.chroot.zap(args, dry=args.dry, packages=args.packages, http=args.http, - mismatch_bins=args.mismatch_bins, old_bins=args.old_bins, - distfiles=args.distfiles) + pmb.chroot.zap(args, dry=args.dry, http=args.http, + distfiles=args.distfiles, pkgs_local=args.pkgs_local, + pkgs_local_mismatch=args.pkgs_local_mismatch, + pkgs_online_mismatch=args.pkgs_online_mismatch) # Don't write the "Done" message pmb.helpers.logging.disable() diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index a22ba880..38ac5531 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -154,6 +154,8 @@ def parse(args, path, multiple_providers=True): """ Parse an APKINDEX.tar.gz file, and return its content as dictionary. + :param path: path to an APKINDEX.tar.gz file or apk package database + (almost the same format, but not compressed). :param multiple_providers: assume that there are more than one provider for the alias. This makes sense when parsing the APKINDEX files from a repository (#1122), but @@ -220,6 +222,33 @@ def parse(args, path, multiple_providers=True): return ret +def parse_blocks(args, path): + """ + Read all blocks from an APKINDEX.tar.gz into a list. + + :path: full path to the APKINDEX.tar.gz file. + :returns: all blocks in the APKINDEX, without restructuring them by + pkgname or removing duplicates with lower versions (use + parse() if you need these features). Structure: + [block, block, ...] + + NOTE: "block" is the return value from parse_next_block() above. + """ + # Parse all lines + with tarfile.open(path, "r:gz") as tar: + with tar.extractfile(tar.getmember("APKINDEX")) as handle: + lines = handle.readlines() + + # Parse lines into blocks + ret = [] + start = [0] + while True: + block = pmb.parse.apkindex.parse_next_block(args, path, lines, start) + if not block: + return ret + ret.append(block) + + def clear_cache(args, path): """ Clear the APKINDEX parsing cache. diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 65f686d0..ca379f3c 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -230,17 +230,21 @@ def arguments(): zap.add_argument("--dry", action="store_true", help="instead of actually" " deleting anything, print out what would have been" " deleted") - zap.add_argument("-p", "--packages", action="store_true", help="also delete" - " the precious, self-compiled packages") zap.add_argument("-hc", "--http", action="store_true", help="also delete http" - "cache") - zap.add_argument("-m", "--mismatch-bins", action="store_true", help="also delete" - " binary packages that are newer than the corresponding" - " package in aports") - zap.add_argument("-o", "--old-bins", action="store_true", help="also delete outdated" - " binary packages downloaded from mirrors (e.g. from Alpine)") + " cache") zap.add_argument("-d", "--distfiles", action="store_true", help="also delete" - " downloaded files cache") + " downloaded source tarballs") + zap.add_argument("-p", "--pkgs-local", action="store_true", + dest="pkgs_local", + help="also delete *all* locally compiled packages") + zap.add_argument("-m", "--pkgs-local-mismatch", action="store_true", + dest="pkgs_local_mismatch", + help="also delete locally compiled packages without" + " existing aport of same version") + zap.add_argument("-o", "--pkgs-online-mismatch", action="store_true", + dest="pkgs_online_mismatch", + help="also delete outdated packages from online mirrors" + " (that have been downloaded to the apk cache)") # Action: stats stats = sub.add_parser("stats", help="show ccache stats")