pmbootstrap zap -m: delete outdated packages too (#1306)
`-m` is for deleting local compiled packages, for which there is no aport with the same version. Prior to this change, this only worked for packages where no aport exists, or for packages that are newer than the aports. That is, because we used the usual APKINDEX parsing logic, which ignores old packages in the APKINDEX and only returns the one with the highest version (that makes sense during dependency resolution). Changes: * New `pmb.parse.apkindex.parse_blocks()` function that returns a raw list of blocks, instead of the dict with removed duplicates with lower version you get from the usual `.parse()` function. * Renamed each of the zap flags and their descriptions to make clear what they are doing now. ``` short long (old) long (new) -p --packages --pkgs-local -m --mismatch-bins --pkgs-local-mismatch -o, --old-bins --pkgs-online-mismatch ```
This commit is contained in:
Oliver Smith
GitHub
parent
3510a4868f
commit
e72afc53fd
|
|
@ -23,20 +23,21 @@ import os
|
|||
|
||||
import pmb.chroot
|
||||
import pmb.helpers.run
|
||||
import pmb.parse.apkindex
|
||||
|
||||
|
||||
def zap(args, confirm=True, dry=False, packages=False, http=False,
|
||||
mismatch_bins=False, old_bins=False, distfiles=False):
|
||||
def zap(args, confirm=True, dry=False, pkgs_local=False, http=False,
|
||||
pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False):
|
||||
"""
|
||||
Shutdown everything inside the chroots (e.g. distccd, adb), umount
|
||||
everything and then safely remove folders from the work-directory.
|
||||
|
||||
:param dry: Only show what would be deleted, do not delete for real
|
||||
:param packages: Remove *all* self-compiled packages (!)
|
||||
:param pkgs_local: Remove *all* self-compiled packages (!)
|
||||
:param http: Clear the http cache (used e.g. for the initial apk download)
|
||||
:param mismatch_bins: Remove the packages, that have a different version
|
||||
:param pkgs_local_mismatch: Remove the packages, that have a different version
|
||||
compared to what is in the aports folder.
|
||||
:param old_bins: Clean out outdated binary packages downloaded from
|
||||
:param pkgs_online_mismatch: Clean out outdated binary packages downloaded from
|
||||
mirrors (e.g. from Alpine)
|
||||
:param distfiles: Clear the downloaded files cache
|
||||
|
||||
|
|
@ -50,12 +51,12 @@ def zap(args, confirm=True, dry=False, packages=False, http=False,
|
|||
size_old = pmb.helpers.other.folder_size(args, args.work)
|
||||
|
||||
# Delete packages with a different version compared to aports, then re-index
|
||||
if mismatch_bins:
|
||||
zap_mismatch_bins(args, confirm, dry)
|
||||
if pkgs_local_mismatch:
|
||||
zap_pkgs_local_mismatch(args, confirm, dry)
|
||||
|
||||
# Delete outdated binary packages
|
||||
if old_bins:
|
||||
zap_old_bins(args, confirm, dry)
|
||||
if pkgs_online_mismatch:
|
||||
zap_pkgs_online_mismatch(args, confirm, dry)
|
||||
|
||||
pmb.chroot.shutdown(args)
|
||||
|
||||
|
|
@ -65,7 +66,7 @@ def zap(args, confirm=True, dry=False, packages=False, http=False,
|
|||
"chroot_buildroot_*",
|
||||
"chroot_rootfs_*",
|
||||
]
|
||||
if packages:
|
||||
if pkgs_local:
|
||||
patterns += ["packages"]
|
||||
if http:
|
||||
patterns += ["cache_http"]
|
||||
|
|
@ -94,7 +95,7 @@ def zap(args, confirm=True, dry=False, packages=False, http=False,
|
|||
logging.info("Cleared up ~" + str(math.ceil(mb)) + " MB of space")
|
||||
|
||||
|
||||
def zap_mismatch_bins(args, confirm=True, dry=False):
|
||||
def zap_pkgs_local_mismatch(args, confirm=True, dry=False):
|
||||
if not os.path.exists(args.work + "/packages/"):
|
||||
return
|
||||
if confirm and not pmb.helpers.cli.confirm(args, "Remove packages that are newer than"
|
||||
|
|
@ -103,15 +104,13 @@ def zap_mismatch_bins(args, confirm=True, dry=False):
|
|||
|
||||
reindex = False
|
||||
for apkindex_path in glob.glob(args.work + "/packages/*/APKINDEX.tar.gz"):
|
||||
apkindex = pmb.parse.apkindex.parse(args, apkindex_path, False)
|
||||
for pkgname, bin_data in apkindex.items():
|
||||
# Only real packages have apks, provided packages do not exist
|
||||
# (e.g. "so:libtest.so.1.2")
|
||||
if pkgname != bin_data["pkgname"]:
|
||||
continue
|
||||
origin = bin_data["origin"]
|
||||
version = bin_data["version"]
|
||||
arch = bin_data["arch"]
|
||||
# Delete packages without same version in aports
|
||||
blocks = pmb.parse.apkindex.parse_blocks(args, apkindex_path)
|
||||
for block in blocks:
|
||||
pkgname = block["pkgname"]
|
||||
origin = block["origin"]
|
||||
version = block["version"]
|
||||
arch = block["arch"]
|
||||
|
||||
# Apk path
|
||||
apk_path_short = arch + "/" + pkgname + "-" + version + ".apk"
|
||||
|
|
@ -145,7 +144,7 @@ def zap_mismatch_bins(args, confirm=True, dry=False):
|
|||
pmb.build.other.index_repo(args)
|
||||
|
||||
|
||||
def zap_old_bins(args, confirm=True, dry=False):
|
||||
def zap_pkgs_online_mismatch(args, confirm=True, dry=False):
|
||||
# Check whether we need to do anything
|
||||
paths = glob.glob(args.work + "/cache_apk_*")
|
||||
if not len(paths):
|
||||
|
|
|
|||
|
|
@ -324,9 +324,10 @@ def log_distccd(args):
|
|||
|
||||
|
||||
def zap(args):
|
||||
pmb.chroot.zap(args, dry=args.dry, packages=args.packages, http=args.http,
|
||||
mismatch_bins=args.mismatch_bins, old_bins=args.old_bins,
|
||||
distfiles=args.distfiles)
|
||||
pmb.chroot.zap(args, dry=args.dry, http=args.http,
|
||||
distfiles=args.distfiles, pkgs_local=args.pkgs_local,
|
||||
pkgs_local_mismatch=args.pkgs_local_mismatch,
|
||||
pkgs_online_mismatch=args.pkgs_online_mismatch)
|
||||
|
||||
# Don't write the "Done" message
|
||||
pmb.helpers.logging.disable()
|
||||
|
|
|
|||
|
|
@ -154,6 +154,8 @@ def parse(args, path, multiple_providers=True):
|
|||
"""
|
||||
Parse an APKINDEX.tar.gz file, and return its content as dictionary.
|
||||
|
||||
:param path: path to an APKINDEX.tar.gz file or apk package database
|
||||
(almost the same format, but not compressed).
|
||||
:param multiple_providers: assume that there are more than one provider for
|
||||
the alias. This makes sense when parsing the
|
||||
APKINDEX files from a repository (#1122), but
|
||||
|
|
@ -220,6 +222,33 @@ def parse(args, path, multiple_providers=True):
|
|||
return ret
|
||||
|
||||
|
||||
def parse_blocks(args, path):
|
||||
"""
|
||||
Read all blocks from an APKINDEX.tar.gz into a list.
|
||||
|
||||
:path: full path to the APKINDEX.tar.gz file.
|
||||
:returns: all blocks in the APKINDEX, without restructuring them by
|
||||
pkgname or removing duplicates with lower versions (use
|
||||
parse() if you need these features). Structure:
|
||||
[block, block, ...]
|
||||
|
||||
NOTE: "block" is the return value from parse_next_block() above.
|
||||
"""
|
||||
# Parse all lines
|
||||
with tarfile.open(path, "r:gz") as tar:
|
||||
with tar.extractfile(tar.getmember("APKINDEX")) as handle:
|
||||
lines = handle.readlines()
|
||||
|
||||
# Parse lines into blocks
|
||||
ret = []
|
||||
start = [0]
|
||||
while True:
|
||||
block = pmb.parse.apkindex.parse_next_block(args, path, lines, start)
|
||||
if not block:
|
||||
return ret
|
||||
ret.append(block)
|
||||
|
||||
|
||||
def clear_cache(args, path):
|
||||
"""
|
||||
Clear the APKINDEX parsing cache.
|
||||
|
|
|
|||
|
|
@ -230,17 +230,21 @@ def arguments():
|
|||
zap.add_argument("--dry", action="store_true", help="instead of actually"
|
||||
" deleting anything, print out what would have been"
|
||||
" deleted")
|
||||
zap.add_argument("-p", "--packages", action="store_true", help="also delete"
|
||||
" the precious, self-compiled packages")
|
||||
zap.add_argument("-hc", "--http", action="store_true", help="also delete http"
|
||||
"cache")
|
||||
zap.add_argument("-m", "--mismatch-bins", action="store_true", help="also delete"
|
||||
" binary packages that are newer than the corresponding"
|
||||
" package in aports")
|
||||
zap.add_argument("-o", "--old-bins", action="store_true", help="also delete outdated"
|
||||
" binary packages downloaded from mirrors (e.g. from Alpine)")
|
||||
" cache")
|
||||
zap.add_argument("-d", "--distfiles", action="store_true", help="also delete"
|
||||
" downloaded files cache")
|
||||
" downloaded source tarballs")
|
||||
zap.add_argument("-p", "--pkgs-local", action="store_true",
|
||||
dest="pkgs_local",
|
||||
help="also delete *all* locally compiled packages")
|
||||
zap.add_argument("-m", "--pkgs-local-mismatch", action="store_true",
|
||||
dest="pkgs_local_mismatch",
|
||||
help="also delete locally compiled packages without"
|
||||
" existing aport of same version")
|
||||
zap.add_argument("-o", "--pkgs-online-mismatch", action="store_true",
|
||||
dest="pkgs_online_mismatch",
|
||||
help="also delete outdated packages from online mirrors"
|
||||
" (that have been downloaded to the apk cache)")
|
||||
|
||||
# Action: stats
|
||||
stats = sub.add_parser("stats", help="show ccache stats")
|
||||
|
|
|
|||
Loading…
Reference in New Issue