pmb.chroot.apk.installed(): use apk's internal database

...instead of running apk every time to get the list of installed
packages and their versions. The internal package database from
apk has the same format, as the extracted APKINDEX file (except
that it has more key-value pairs, which we ignore/do not need
right now). So the APKINDEX code has been extended to parse both
tar-packed APKINDEX files and regular text files in the APKINDEX
format.

This is required for #108, for a better detection of outdated
packages (because the internal package database  saves the
package's timestamp, too). A nice benefit is, that this is faster
than calling apk every time and it doesn't fill up the log as much.

I've also used this improved function for determining the apk
version (for the outdated version check), and I've deleted
pmb.parse.other.package_split(), as it is not needed anymore.

pmb/build/buildinfo.py

@@ -67,7 +67,7 @@ def generate(args, apk_path, arch, suffix, apkbuild):
            "pkgver": apkbuild["pkgver"],
            "pkgrel": apkbuild["pkgrel"],
            "arch": arch,
-           "versions": []}
+           "versions": {}}
 
     # Add makedepends versions
     installed = pmb.chroot.apk.installed(args, suffix)
@@ -75,8 +75,7 @@ def generate(args, apk_path, arch, suffix, apkbuild):
                 get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"]))
     for pkgname in relevant:
         if pkgname in installed:
-            ret["versions"].append(installed[pkgname]["package"])
+            ret["versions"][pkgname] = installed[pkgname]["version"]
-    ret["versions"].sort()
     return ret
 
 

pmb/build/other.py

@@ -96,8 +96,8 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
     if pmb.parse.apkindex.compare_version(version_old,
                                           version_new) == 1:
         logging.warning("WARNING: Package '" + package + "' in your aports folder"
-                        " has version " + version_old + ", but the binary package"
+                        " has version " + version_new + ", but the binary package"
-                        " repositories already have version " + version_new + "!")
+                        " repositories already have version " + version_old + "!")
         return False
 
     # b) Aports folder has a newer version

pmb/challenge/build.py

@@ -21,7 +21,6 @@ import json
 import os
 import pmb.build
 import pmb.parse.apkbuild
-import pmb.parse.other
 import pmb.helpers.repo
 import pmb.challenge
 
@@ -37,23 +36,17 @@ def build(args, apk_path):
     with open(buildinfo_path) as handle:
         buildinfo = json.load(handle)
 
-    # Parse and install all packages listed in versions
+    # Install all listed packages
-    versions = {}
+    pmb.chroot.apk.install(args, buildinfo["versions"].keys())
-    for package in buildinfo["versions"]:
-        split = pmb.parse.other.package_split(package)
-        pkgname = split["pkgname"]
-        versions[pkgname] = split
-    pmb.chroot.apk.install(args, versions.keys())
 
     # Verify the installed versions
     installed = pmb.chroot.apk.installed(args)
-    for pkgname, split in versions.items():
+    for pkgname, version in buildinfo["versions"].items():
-        package_installed = installed[pkgname]["package"]
+        version_installed = installed[pkgname]["version"]
-        package_buildinfo = split["package"]
+        if version_installed != version:
-        if package_installed != package_buildinfo:
             raise RuntimeError("Dependency " + pkgname + " version is different"
-                               " (installed: " + package_installed + ","
+                               " (installed: " + version_installed + ","
-                               " buildinfo: " + package_buildinfo + ")!")
+                               " buildinfo: " + version + ")!")
     # Build the package
     repo_before = pmb.helpers.repo.files(args)
     pmb.build.package(args, buildinfo["pkgname"], buildinfo["arch"],

pmb/chroot/apk.py

@@ -16,11 +16,11 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
+import os
 import logging
 import pmb.chroot
 import pmb.config
 import pmb.parse.apkindex
-import pmb.parse.other
 
 
 def check_min_version(args, suffix="native"):
@@ -28,23 +28,29 @@ def check_min_version(args, suffix="native"):
     Check the minimum apk version, before running it the first time in the
     current session (lifetime of one pmbootstrap call).
     """
+
     # Skip if we already did this
     if suffix in args.cache["apk_min_version_checked"]:
         return
 
-    # Read the version from apk and from the config
+    # Skip if apk is not installed yet
-    pkgver = pmb.chroot.root(args, ["apk", "--version"], suffix,
+    if not os.path.exists(args.work + "/chroot_" + suffix + "/sbin/apk"):
-                             return_stdout=True)
+        logging.debug("NOTE: Skipped apk version check for chroot '" + suffix +
-    pkgver = pkgver.split(" ")[1].split(",")[0]
+                      "', because it is not installed yet!")
-    pkgver_min = pmb.config.apk_tools_static_min_version.split("-")[0]
+        return
 
     # Compare
-    if pmb.parse.apkindex.compare_version(pkgver, pkgver_min) == -1:
+    version_installed = installed(args, suffix)["apk-tools"]["version"]
+    version_min = pmb.config.apk_tools_static_min_version
+    if pmb.parse.apkindex.compare_version(
+            version_installed, version_min) == -1:
         raise RuntimeError("You have an outdated version of the 'apk' package"
-                           " manager installed (your version: " + pkgver +
+                           " manager installed (your version: " + version_installed +
-                           ", expected at least: " + pkgver_min + "). Delete"
+                           ", expected at least: " + version_min + "). Delete"
                            " your http cache and zap all chroots, then try again:"
                            " 'pmbootstrap zap -hc'")
+
+    # Mark this suffix as checked
     args.cache["apk_min_version_checked"].append(suffix)
 
 
@@ -101,19 +107,19 @@ def upgrade(args, suffix="native", update_index=True):
 
 def installed(args, suffix="native"):
     """
-    Get all installed packages and their versions.
+    Read the list of installed packages (which has almost the same format, as
-    :returns: { "hello-world": {"package": "hello-world-1-r2", "pkgrel": "2",
+    an APKINDEX, but with more keys).
-        "pkgver": "1", "pkgname": "hello-world"}, ...}
+    :returns: a dictionary with the following structure:
+              { "postmarketos-mkinitfs":
+                {
+                  "pkgname": "postmarketos-mkinitfs"
+                  "version": "0.0.4-r10",
+                  "depends": ["busybox-extras", "lddtree", ...],
+                  "provides": ["mkinitfs=0.0.1"]
+                }, ...
+              }
     """
-    check_min_version(args, suffix)
+    path = args.work + "/chroot_" + suffix + "/lib/apk/db/installed"
-    list = pmb.chroot.user(args, ["apk", "info", "-vv"], suffix,
+    if not os.path.exists(path):
-                           return_stdout=True)
+        return {}
-    # Parse the output into a dictionary
+    return pmb.parse.apkindex.parse(args, path)
-    ret = {}
-    for line in list.split("\n"):
-        if not line.rstrip():
-            continue
-        package = line.split(" - ")[0]
-        split = pmb.parse.other.package_split(package)
-        ret[split["pkgname"]] = split
-    return ret

pmb/parse/apkindex.py

@@ -77,7 +77,9 @@ def parse_next_block(args, path, lines, start):
     for i in range(start[0], len(lines)):
         # Check for empty line
         start[0] = i + 1
-        line = lines[i].decode()
+        line = lines[i]
+        if not isinstance(line, str):
+            line = line.decode()
         if line == "\n":
             end_of_block_found = True
             break
@@ -182,25 +184,32 @@ def parse(args, path, strict=False):
         if cache["lastmod"] == lastmod:
             return cache["ret"]
 
-    # Parse the whole APKINDEX.tar.gz file
+    # Read all lines
+    if tarfile.is_tarfile(path):
+        with tarfile.open(path, "r:gz") as tar:
+            with tar.extractfile(tar.getmember("APKINDEX")) as handle:
+                lines = handle.readlines()
+    else:
+        with open(path, "r", encoding="utf-8") as handle:
+            lines = handle.readlines()
+
+    # Parse the whole APKINDEX file
     ret = {}
     start = [0]
-    with tarfile.open(path, "r:gz") as tar:
+    while True:
-        with tar.extractfile(tar.getmember("APKINDEX")) as handle:
+        block = parse_next_block(args, path, lines, start)
-            lines = handle.readlines()
+        if not block:
-            while True:
+            break
-                block = parse_next_block(args, path, lines, start)
+
-                if not block:
+        # Add the next package and all aliases
-                    break
+        parse_add_block(path, strict, ret, block)
+        if "provides" in block:
+            for alias in block["provides"]:
+                split = alias.split("=")
+                if len(split) == 2:
+                    parse_add_block(path, strict, ret, block, split[0],
+                                    split[1])
 
-                # Add the next package and all aliases
-                parse_add_block(path, strict, ret, block)
-                if "provides" in block:
-                    for alias in block["provides"]:
-                        split = alias.split("=")
-                        if len(split) == 2:
-                            parse_add_block(path, strict, ret, block,
-                                            split[0], split[1])
     # Update the cache
     args.cache["apkindex"][path] = {"lastmod": lastmod, "ret": ret}
 

pmb/parse/other.py

@@ -1,35 +0,0 @@
-"""
-Copyright 2017 Oliver Smith
-
-This file is part of pmbootstrap.
-
-pmbootstrap is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-pmbootstrap is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
-"""
-
-
-def package_split(package):
-    """
-    Split a full package name (as returned by `apk info -vv` and as found as
-    apk file name) into its components.
-    :param package: Example: "heimdall-1.4.2-r1"
-    """
-    split = package.split("-")
-    pkgrel = split[-1][1:]
-    pkgver = split[-2]
-    version = "-" + pkgver + "-r" + pkgrel
-    pkgname = package[:-1 * len(version)]
-    return {"pkgname": pkgname,
-            "pkgrel": pkgrel,
-            "pkgver": pkgver,
-            "package": package}