From 28a0e10e56c1642daa71128978aff283ce68e923 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Tue, 27 Jun 2017 17:56:15 +0200 Subject: [PATCH] pmb.chroot.apk.installed(): use apk's internal database ...instead of running apk every time to get the list of installed packages and their versions. The internal package database from apk has the same format, as the extracted APKINDEX file (except that it has more key-value pairs, which we ignore/do not need right now). So the APKINDEX code has been extended to parse both tar-packed APKINDEX files and regular text files in the APKINDEX format. This is required for #108, for a better detection of outdated packages (because the internal package database saves the package's timestamp, too). A nice benefit is, that this is faster than calling apk every time and it doesn't fill up the log as much. I've also used this improved function for determining the apk version (for the outdated version check), and I've deleted pmb.parse.other.package_split(), as it is not needed anymore. --- pmb/build/buildinfo.py | 5 ++-- pmb/build/other.py | 4 ++-- pmb/challenge/build.py | 21 ++++++---------- pmb/chroot/apk.py | 54 +++++++++++++++++++++++------------------- pmb/parse/apkindex.py | 43 ++++++++++++++++++++------------- pmb/parse/other.py | 35 --------------------------- 6 files changed, 67 insertions(+), 95 deletions(-) delete mode 100644 pmb/parse/other.py diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py index 745ddaec..b812a697 100644 --- a/pmb/build/buildinfo.py +++ b/pmb/build/buildinfo.py @@ -67,7 +67,7 @@ def generate(args, apk_path, arch, suffix, apkbuild): "pkgver": apkbuild["pkgver"], "pkgrel": apkbuild["pkgrel"], "arch": arch, - "versions": []} + "versions": {}} # Add makedepends versions installed = pmb.chroot.apk.installed(args, suffix) @@ -75,8 +75,7 @@ def generate(args, apk_path, arch, suffix, apkbuild): get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"])) for pkgname in relevant: if pkgname in installed: - ret["versions"].append(installed[pkgname]["package"]) - ret["versions"].sort() + ret["versions"][pkgname] = installed[pkgname]["version"] return ret diff --git a/pmb/build/other.py b/pmb/build/other.py index 605d7c6c..bffd5e19 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -96,8 +96,8 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None): if pmb.parse.apkindex.compare_version(version_old, version_new) == 1: logging.warning("WARNING: Package '" + package + "' in your aports folder" - " has version " + version_old + ", but the binary package" - " repositories already have version " + version_new + "!") + " has version " + version_new + ", but the binary package" + " repositories already have version " + version_old + "!") return False # b) Aports folder has a newer version diff --git a/pmb/challenge/build.py b/pmb/challenge/build.py index 3fa2bdf9..b4c5586b 100644 --- a/pmb/challenge/build.py +++ b/pmb/challenge/build.py @@ -21,7 +21,6 @@ import json import os import pmb.build import pmb.parse.apkbuild -import pmb.parse.other import pmb.helpers.repo import pmb.challenge @@ -37,23 +36,17 @@ def build(args, apk_path): with open(buildinfo_path) as handle: buildinfo = json.load(handle) - # Parse and install all packages listed in versions - versions = {} - for package in buildinfo["versions"]: - split = pmb.parse.other.package_split(package) - pkgname = split["pkgname"] - versions[pkgname] = split - pmb.chroot.apk.install(args, versions.keys()) + # Install all listed packages + pmb.chroot.apk.install(args, buildinfo["versions"].keys()) # Verify the installed versions installed = pmb.chroot.apk.installed(args) - for pkgname, split in versions.items(): - package_installed = installed[pkgname]["package"] - package_buildinfo = split["package"] - if package_installed != package_buildinfo: + for pkgname, version in buildinfo["versions"].items(): + version_installed = installed[pkgname]["version"] + if version_installed != version: raise RuntimeError("Dependency " + pkgname + " version is different" - " (installed: " + package_installed + "," - " buildinfo: " + package_buildinfo + ")!") + " (installed: " + version_installed + "," + " buildinfo: " + version + ")!") # Build the package repo_before = pmb.helpers.repo.files(args) pmb.build.package(args, buildinfo["pkgname"], buildinfo["arch"], diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index dde27c29..a4547681 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -16,11 +16,11 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ +import os import logging import pmb.chroot import pmb.config import pmb.parse.apkindex -import pmb.parse.other def check_min_version(args, suffix="native"): @@ -28,23 +28,29 @@ def check_min_version(args, suffix="native"): Check the minimum apk version, before running it the first time in the current session (lifetime of one pmbootstrap call). """ + # Skip if we already did this if suffix in args.cache["apk_min_version_checked"]: return - # Read the version from apk and from the config - pkgver = pmb.chroot.root(args, ["apk", "--version"], suffix, - return_stdout=True) - pkgver = pkgver.split(" ")[1].split(",")[0] - pkgver_min = pmb.config.apk_tools_static_min_version.split("-")[0] + # Skip if apk is not installed yet + if not os.path.exists(args.work + "/chroot_" + suffix + "/sbin/apk"): + logging.debug("NOTE: Skipped apk version check for chroot '" + suffix + + "', because it is not installed yet!") + return # Compare - if pmb.parse.apkindex.compare_version(pkgver, pkgver_min) == -1: + version_installed = installed(args, suffix)["apk-tools"]["version"] + version_min = pmb.config.apk_tools_static_min_version + if pmb.parse.apkindex.compare_version( + version_installed, version_min) == -1: raise RuntimeError("You have an outdated version of the 'apk' package" - " manager installed (your version: " + pkgver + - ", expected at least: " + pkgver_min + "). Delete" + " manager installed (your version: " + version_installed + + ", expected at least: " + version_min + "). Delete" " your http cache and zap all chroots, then try again:" " 'pmbootstrap zap -hc'") + + # Mark this suffix as checked args.cache["apk_min_version_checked"].append(suffix) @@ -101,19 +107,19 @@ def upgrade(args, suffix="native", update_index=True): def installed(args, suffix="native"): """ - Get all installed packages and their versions. - :returns: { "hello-world": {"package": "hello-world-1-r2", "pkgrel": "2", - "pkgver": "1", "pkgname": "hello-world"}, ...} + Read the list of installed packages (which has almost the same format, as + an APKINDEX, but with more keys). + :returns: a dictionary with the following structure: + { "postmarketos-mkinitfs": + { + "pkgname": "postmarketos-mkinitfs" + "version": "0.0.4-r10", + "depends": ["busybox-extras", "lddtree", ...], + "provides": ["mkinitfs=0.0.1"] + }, ... + } """ - check_min_version(args, suffix) - list = pmb.chroot.user(args, ["apk", "info", "-vv"], suffix, - return_stdout=True) - # Parse the output into a dictionary - ret = {} - for line in list.split("\n"): - if not line.rstrip(): - continue - package = line.split(" - ")[0] - split = pmb.parse.other.package_split(package) - ret[split["pkgname"]] = split - return ret + path = args.work + "/chroot_" + suffix + "/lib/apk/db/installed" + if not os.path.exists(path): + return {} + return pmb.parse.apkindex.parse(args, path) diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index a19f8a84..984703f9 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -77,7 +77,9 @@ def parse_next_block(args, path, lines, start): for i in range(start[0], len(lines)): # Check for empty line start[0] = i + 1 - line = lines[i].decode() + line = lines[i] + if not isinstance(line, str): + line = line.decode() if line == "\n": end_of_block_found = True break @@ -182,25 +184,32 @@ def parse(args, path, strict=False): if cache["lastmod"] == lastmod: return cache["ret"] - # Parse the whole APKINDEX.tar.gz file + # Read all lines + if tarfile.is_tarfile(path): + with tarfile.open(path, "r:gz") as tar: + with tar.extractfile(tar.getmember("APKINDEX")) as handle: + lines = handle.readlines() + else: + with open(path, "r", encoding="utf-8") as handle: + lines = handle.readlines() + + # Parse the whole APKINDEX file ret = {} start = [0] - with tarfile.open(path, "r:gz") as tar: - with tar.extractfile(tar.getmember("APKINDEX")) as handle: - lines = handle.readlines() - while True: - block = parse_next_block(args, path, lines, start) - if not block: - break + while True: + block = parse_next_block(args, path, lines, start) + if not block: + break + + # Add the next package and all aliases + parse_add_block(path, strict, ret, block) + if "provides" in block: + for alias in block["provides"]: + split = alias.split("=") + if len(split) == 2: + parse_add_block(path, strict, ret, block, split[0], + split[1]) - # Add the next package and all aliases - parse_add_block(path, strict, ret, block) - if "provides" in block: - for alias in block["provides"]: - split = alias.split("=") - if len(split) == 2: - parse_add_block(path, strict, ret, block, - split[0], split[1]) # Update the cache args.cache["apkindex"][path] = {"lastmod": lastmod, "ret": ret} diff --git a/pmb/parse/other.py b/pmb/parse/other.py deleted file mode 100644 index b7bcb8e2..00000000 --- a/pmb/parse/other.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -Copyright 2017 Oliver Smith - -This file is part of pmbootstrap. - -pmbootstrap is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -pmbootstrap is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with pmbootstrap. If not, see . -""" - - -def package_split(package): - """ - Split a full package name (as returned by `apk info -vv` and as found as - apk file name) into its components. - :param package: Example: "heimdall-1.4.2-r1" - """ - split = package.split("-") - pkgrel = split[-1][1:] - pkgver = split[-2] - version = "-" + pkgver + "-r" + pkgrel - pkgname = package[:-1 * len(version)] - return {"pkgname": pkgname, - "pkgrel": pkgrel, - "pkgver": pkgver, - "package": package}