Index parser: support multiple package providers (#1202)

* The APKINDEX parser used to return a dictionary with one package for
  a given package name. This works for the installed packages database,
  because there can only be one provider for a package. But when
  parsing packages from binary repositories, we need to support
  multiple providers for one package. It is now possible to get a
  dictionary with either multiple providers, or just a single provider
  for each package.
* Dependency parsing logic has been adjusted, to support multiple
  providers. For multiple providers, the one with the same package
  name as the package we are looking up is prefered. If there is none
  (eg. "so:libEGL.so.1" is provided by "mesa-egl"), it prefers packages
  that will be installed anyway, and after that packages that are
  already installed. When all else fails, it just picks the first one
  and prints a note in the "pmbootstrap log".
* Added testcases for all functions in pmb.parse.apkindex and
  pmb.parse.depends
* pmbootstrap chroot has a new "--add" parameter to specify packages
  that pmbootstrap should build if neccessary, and install in the
  chroot. This can be used to quickly test the depencency resolution
  of pmbootstrap without doing a full "pmbootstrap install".

Fixes #1122.

pmb/aportgen/busybox_static.py

@@ -26,14 +26,12 @@ import pmb.chroot.apk_static
 
 
 def generate(args, pkgname):
-    # Install busybox-static in chroot (so we have the APKINDEX and verified
+    # Install busybox-static in chroot to get verified apks
-    # apks)
     arch = pkgname.split("-")[2]
-    apkindex = pmb.chroot.apk_static.download(args, "APKINDEX.tar.gz")
     pmb.chroot.apk.install(args, ["busybox-static"], "buildroot_" + arch)
 
     # Parse version from APKINDEX
-    package_data = pmb.parse.apkindex.read(args, "busybox", apkindex)
+    package_data = pmb.parse.apkindex.package(args, "busybox")
     version = package_data["version"]
     pkgver = version.split("-r")[0]
     pkgrel = version.split("-r")[1]

pmb/aportgen/musl.py

@@ -26,13 +26,12 @@ import pmb.chroot.apk_static
 
 
 def generate(args, pkgname):
-    # Install musl in chroot (so we have the APKINDEX and verified musl apks)
+    # Install musl in chroot to get verified apks
     arch = pkgname.split("-")[1]
-    apkindex = pmb.chroot.apk_static.download(args, "APKINDEX.tar.gz")
     pmb.chroot.apk.install(args, ["musl-dev"], "buildroot_" + arch)
 
     # Parse musl version from APKINDEX
-    package_data = pmb.parse.apkindex.read(args, "musl", apkindex)
+    package_data = pmb.parse.apkindex.package(args, "musl")
     version = package_data["version"]
     pkgver = version.split("-r")[0]
     pkgrel = version.split("-r")[1]

pmb/build/_package.py

@@ -63,7 +63,7 @@ def get_apkbuild(args, pkgname, arch):
     aport = pmb.build.find_aport(args, pkgname, False)
     if aport:
         return pmb.parse.apkbuild(args, aport + "/APKBUILD")
-    if pmb.parse.apkindex.read_any_index(args, pkgname, arch):
+    if pmb.parse.apkindex.providers(args, pkgname, arch, False):
         return None
     raise RuntimeError("Package '" + pkgname + "': Could not find aport, and"
                        " could not find this package in any APKINDEX!")
@@ -214,12 +214,7 @@ def get_gcc_version(args, arch):
     <https://linux.die.net/man/1/ccache>
     :returns: a string like "6.4.0-r5"
     """
-    repository = args.mirror_alpine + args.alpine_version + "/main"
+    return pmb.parse.apkindex.package(args, "gcc", arch)["version"]
-    hash = pmb.helpers.repo.hash(repository)
-    index_path = (args.work + "/cache_apk_" + arch + "/APKINDEX." +
-                  hash + ".tar.gz")
-    apkindex = pmb.parse.apkindex.read(args, "gcc", index_path, True)
-    return apkindex["version"]
 
 
 def get_pkgver(original_pkgver, original_source=False, now=None):

pmb/build/other.py

@@ -91,7 +91,7 @@ def copy_to_buildpath(args, package, suffix="native"):
                            "/home/pmos/build"], suffix=suffix)
 
 
-def is_necessary(args, arch, apkbuild, apkindex_path=None):
+def is_necessary(args, arch, apkbuild, indexes=None):
     """
     Check if the package has already been built. Compared to abuild's check,
     this check also works for different architectures, and it recognizes
@@ -100,7 +100,7 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
 
     :param arch: package target architecture
     :param apkbuild: from pmb.parse.apkbuild()
-    :param apkindex_path: override the APKINDEX.tar.gz path
+    :param indexes: list of APKINDEX.tar.gz paths
     :returns: boolean
     """
     # Get package name, version, define start of debug message
@@ -109,11 +109,8 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
     msg = "Build is necessary for package '" + package + "': "
 
     # Get old version from APKINDEX
-    if apkindex_path:
+    index_data = pmb.parse.apkindex.package(args, package, arch, False,
-        index_data = pmb.parse.apkindex.read(
+                                            indexes)
-            args, package, apkindex_path, False)
-    else:
-        index_data = pmb.parse.apkindex.read_any_index(args, package, arch)
     if not index_data:
         logging.debug(msg + "No binary package available")
         return True

pmb/chroot/apk.py

@@ -94,8 +94,7 @@ def check_min_version(args, suffix="native"):
     # Compare
     version_installed = installed(args, suffix)["apk-tools"]["version"]
     version_min = pmb.config.apk_tools_static_min_version
-    if pmb.parse.version.compare(version_installed,
+    if pmb.parse.version.compare(version_installed, version_min) == -1:
-                                 version_min) == -1:
         raise RuntimeError("You have an outdated version of the 'apk' package"
                            " manager installed (your version: " + version_installed +
                            ", expected at least: " + version_min + "). Delete"
@@ -124,7 +123,7 @@ def install_is_necessary(args, build, arch, package, packages_installed):
         return True
 
     # Make sure, that we really have a binary package
-    data_repo = pmb.parse.apkindex.read_any_index(args, package, arch)
+    data_repo = pmb.parse.apkindex.package(args, package, arch, False)
     if not data_repo:
         logging.warning("WARNING: Internal error in pmbootstrap," +
                         " package '" + package + "' for " + arch +
@@ -193,8 +192,7 @@ def install(args, packages, suffix="native", build=True):
 
     # Add depends to packages
     arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
-    packages_with_depends = pmb.parse.depends.recurse(args, packages, arch,
+    packages_with_depends = pmb.parse.depends.recurse(args, packages, suffix)
-                                                      strict=True)
 
     # Filter outdated packages (build them if required)
     packages_installed = installed(args, suffix)
@@ -256,6 +254,4 @@ def installed(args, suffix="native"):
               }
     """
     path = args.work + "/chroot_" + suffix + "/lib/apk/db/installed"
-    if not os.path.exists(path):
+    return pmb.parse.apkindex.parse(args, path, False)
-        return {}
-    return pmb.parse.apkindex.parse(args, path)

pmb/chroot/apk_static.py

@@ -166,7 +166,8 @@ def init(args):
                 pmb.helpers.repo.hash(url) + ".tar.gz")
 
     # Extract and verify the apk-tools-static version
-    index_data = pmb.parse.apkindex.read(args, "apk-tools-static", apkindex)
+    index_data = pmb.parse.apkindex.package(args, "apk-tools-static",
+                                            indexes=[apkindex])
     version = index_data["version"]
     version_min = pmb.config.apk_tools_static_min_version
     apk_name = "apk-tools-static-" + version + ".apk"

pmb/chroot/initfs_hooks.py

@@ -27,7 +27,7 @@ import pmb.chroot.apk
 def list_chroot(args, suffix, remove_prefix=True):
     ret = []
     prefix = pmb.config.initfs_hook_prefix
-    for pkgname in pmb.chroot.apk.installed(args, suffix):
+    for pkgname in pmb.chroot.apk.installed(args, suffix).keys():
         if pkgname.startswith(prefix):
             if remove_prefix:
                 ret.append(pkgname[len(prefix):])

pmb/helpers/frontend.py

@@ -149,6 +149,8 @@ def checksum(args):
 def chroot(args):
     suffix = _parse_suffix(args)
     pmb.chroot.apk.check_min_version(args, suffix)
+    if args.add:
+        pmb.chroot.apk.install(args, args.add.split(","), suffix)
     logging.info("(" + suffix + ") % " + " ".join(args.command))
     pmb.chroot.root(args, args.command, suffix, log=False)
 

pmb/helpers/pkgrel_bump.py

@@ -98,8 +98,8 @@ def auto_apkindex_package(args, pkgname, aport_version, apkindex, arch,
     :returns: True when there was an APKBUILD that needed to be changed.
     """
     # Binary package
-    binary = pmb.parse.apkindex.read(args, pkgname, apkindex,
+    binary = pmb.parse.apkindex.package(args, pkgname, must_exist=False,
-                                     False)
+                                        indexes=[apkindex])
     if not binary:
         return
 
@@ -124,8 +124,9 @@ def auto_apkindex_package(args, pkgname, aport_version, apkindex, arch,
                     ",".join(binary["depends"]))
     missing = []
     for depend in binary["depends"]:
-        if not pmb.parse.apkindex.read_any_index(args, depend,
+        providers = pmb.parse.apkindex.providers(args, depend, arch,
-                                                 arch):
+                                                 must_exist=False)
+        if providers == {}:
             # We're only interested in missing depends starting with "so:"
             # (which means dynamic libraries that the package was linked
             # against) and packages for which no aport exists.

pmb/parse/apkindex.py

@@ -16,6 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
+import collections
 import logging
 import os
 import tarfile
@@ -38,8 +39,7 @@ def parse_next_block(args, path, lines, start):
                 "depends": ["busybox-extras", "lddtree", ... ],
                 "pkgname": "postmarketos-mkinitfs",
                 "provides": ["mkinitfs=0.0.1"],
-                "version": "0.0.4-r10",
+                "version": "0.0.4-r10" }
-              }
     :returns: None, when there are no more blocks
     """
 
@@ -108,58 +108,87 @@ def parse_next_block(args, path, lines, start):
     return None
 
 
-def parse_add_block(path, ret, block, pkgname=None):
+def parse_add_block(ret, block, alias=None, multiple_providers=True):
     """
     Add one block to the return dictionary of parse().
 
-    :param path: to the APKINDEX.tar.gz
     :param ret: dictionary of all packages in the APKINDEX, that is
                 getting built right now. This function will extend it.
     :param block: return value from parse_next_block().
-    :param pkgname: defaults to the real pkgname, could be an alias
+    :param alias: defaults to the pkgname, could be an alias from the
-                    from the "provides" list.
+                  "provides" list.
-    :param version: defaults to the real version, could be a value
+    :param multiple_providers: assume that there are more than one provider for
-                    from the "provides" list.
+                               the alias. This makes sense when parsing the
+                               APKINDEX files from a repository (#1122), but
+                               not when parsing apk's installed packages DB.
     """
 
     # Defaults
-    if not pkgname:
+    pkgname = block["pkgname"]
-        pkgname = block["pkgname"]
+    alias = alias or pkgname
 
-    # Handle duplicate entries
+    # Get an existing block with the same alias
-    if pkgname in ret:
+    block_old = None
-        # Ignore the block, if the block we already have has a higher
+    if multiple_providers and alias in ret and pkgname in ret[alias]:
-        # version
+        block_old = ret[alias][pkgname]
-        version_old = ret[pkgname]["version"]
+    elif not multiple_providers and alias in ret:
+        block_old = ret[alias]
+
+    # Ignore the block, if the block we already have has a higher version
+    if block_old:
+        version_old = block_old["version"]
         version_new = block["version"]
         if pmb.parse.version.compare(version_old, version_new) == 1:
             return
 
     # Add it to the result set
-    ret[pkgname] = block
+    if multiple_providers:
+        if alias not in ret:
+            ret[alias] = {}
+        ret[alias][pkgname] = block
+    else:
+        ret[alias] = block
 
 
-def parse(args, path):
+def parse(args, path, multiple_providers=True):
     """
     Parse an APKINDEX.tar.gz file, and return its content as dictionary.
 
-    :returns: a dictionary with the following structure:
+    :param multiple_providers: assume that there are more than one provider for
-              { "postmarketos-mkinitfs":
+                               the alias. This makes sense when parsing the
-                {
+                               APKINDEX files from a repository (#1122), but
-                  "pkgname": "postmarketos-mkinitfs"
+                               not when parsing apk's installed packages DB.
-                  "version": "0.0.4-r10",
+    :returns: (without multiple_providers)
-                  "depends": ["busybox-extras", "lddtree", ...],
+              generic format:
-                  "provides": ["mkinitfs=0.0.1"]
+              { pkgname: block, ... }
-                }, ...
+
-              }
+              example:
+              { "postmarketos-mkinitfs": block,
+                "so:libGL.so.1": block, ...}
+
+    :returns: (with multiple_providers)
+              generic format:
+              { provide: { pkgname: block, ... }, ... }
+
+              example:
+              { "postmarketos-mkinitfs": {"postmarketos-mkinitfs": block},
+                "so:libGL.so.1": {"mesa-egl": block, "libhybris": block}, ...}
+
+    NOTE: "block" is the return value from parse_next_block() above.
     """
+    # Require the file to exist
+    if not os.path.isfile(path):
+        logging.debug("NOTE: APKINDEX not found, assuming no binary packages"
+                      " exist for that architecture: " + path)
+        return {}
 
     # Try to get a cached result first
     lastmod = os.path.getmtime(path)
+    cache_key = "multiple" if multiple_providers else "single"
     if path in args.cache["apkindex"]:
         cache = args.cache["apkindex"][path]
-        if cache["lastmod"] == lastmod:
+        if cache["lastmod"] == lastmod and cache_key in cache:
-            return cache["ret"]
+            return cache[cache_key]
 
     # Read all lines
     if tarfile.is_tarfile(path):
@@ -171,7 +200,7 @@ def parse(args, path):
             lines = handle.readlines()
 
     # Parse the whole APKINDEX file
-    ret = {}
+    ret = collections.OrderedDict()
     start = [0]
     while True:
         block = parse_next_block(args, path, lines, start)
@@ -179,97 +208,119 @@ def parse(args, path):
             break
 
         # Add the next package and all aliases
-        parse_add_block(path, ret, block)
+        parse_add_block(ret, block, None, multiple_providers)
         if "provides" in block:
             for alias in block["provides"]:
-                parse_add_block(path, ret, block, alias)
+                parse_add_block(ret, block, alias, multiple_providers)
 
     # Update the cache
-    args.cache["apkindex"][path] = {"lastmod": lastmod, "ret": ret}
+    if path not in args.cache["apkindex"]:
-
+        args.cache["apkindex"][path] = {"lastmod": lastmod}
+    args.cache["apkindex"][path][cache_key] = ret
     return ret
 
 
 def clear_cache(args, path):
+    """
+    Clear the APKINDEX parsing cache.
+
+    :returns: True on successful deletion, False otherwise
+    """
     logging.verbose("Clear APKINDEX cache for: " + path)
     if path in args.cache["apkindex"]:
         del args.cache["apkindex"][path]
+        return True
     else:
         logging.verbose("Nothing to do, path was not in cache:" +
                         str(args.cache["apkindex"].keys()))
+        return False
 
 
-def read(args, package, path, must_exist=True):
+def providers(args, package, arch=None, must_exist=True, indexes=None):
     """
-    Get information about a single package from an APKINDEX.tar.gz file.
+    Get all packages, which provide one package.
 
-    :param path: Path to APKINDEX.tar.gz, defaults to $WORK/APKINDEX.tar.gz
+    :param package: of which you want to have the providers
-    :param package: The package of which you want to read the properties.
+    :param arch: defaults to native arch, only relevant for indexes=None
     :param must_exist: When set to true, raise an exception when the package is
-        missing in the index, or the index file was not found.
+                       not provided at all.
-    :returns: {"pkgname": ..., "version": ..., "depends": [...]}
+    :param indexes: list of APKINDEX.tar.gz paths, defaults to all index files
-        When the package appears multiple times in the APKINDEX, this
+                    (depending on arch)
-        function returns the attributes of the latest version.
+    :returns: list of parsed packages. Example for package="so:libGL.so.1":
+                  {"mesa-egl": block, "libhybris": block}
+              block is the return value from parse_next_block() above.
     """
-    # Verify APKINDEX path
-    if not os.path.exists(path):
-        if not must_exist:
-            return None
-        raise RuntimeError("File not found: " + path)
 
-    # Parse the APKINDEX
+    if not indexes:
-    apkindex = parse(args, path)
+        arch = arch or args.arch_native
-    if package not in apkindex:
+        indexes = pmb.helpers.repo.apkindex_files(args, arch)
-        if must_exist:
-            raise RuntimeError("Package '" + package +
-                               "' not found in " + path)
-        else:
-            return None
-    return apkindex[package]
 
-
+    ret = {}
-def read_any_index(args, package, arch=None):
+    for path in indexes:
-    """
+        # Skip indexes not providing the package
-    Get information about a single package from any APKINDEX.tar.gz.
+        index_packages = parse(args, path)
-
+        if package not in index_packages:
-    We iterate through the index files in the order they are listed in
-    /etc/apk/repositories (we write that file in pmbootstrap, so we know the
-    order). That way it is possible to override a package from an upstream
-    binary repository (pmOS or Alpine) with a package built locally with
-    pmbootstrap.
-
-    If a package is in multiple APKINDEX files in multiple versions, then the
-    highest one gets returned (even if it is not in the first APKINDEX we look
-    at).
-
-    :param arch: defaults to native architecture
-    :returns: the same format as read()
-    """
-    if not arch:
-        arch = args.arch_native
-
-    # Iterate over indexes
-    ret = None
-    version_last = None
-    for index in pmb.helpers.repo.apkindex_files(args, arch):
-        # Skip indexes without the package
-        index_data = read(args, package, index, False)
-        if not index_data:
             continue
 
-        # Skip lower versions
+        # Iterate over found providers
-        version = index_data["version"]
+        for provider_pkgname, provider in index_packages[package].items():
-        if ret and pmb.parse.version.compare(version, version_last) == -1:
+            # Skip lower versions of providers we already found
-            logging.verbose(package + ": " + version + " found in " + index +
+            version = provider["version"]
-                            " (but " + version_last + " is bigger)")
+            if provider_pkgname in ret:
-            continue
+                version_last = ret[provider_pkgname]["version"]
+                if pmb.parse.version.compare(version, version_last) == -1:
+                    logging.verbose(package + ": provided by: " +
+                                    provider_pkgname + "-" + version + " in " +
+                                    path + " (but " + version_last + " is"
+                                    " higher)")
+                    continue
 
-        # Save as result
+            # Add the provier to ret
-        logging.verbose(package + ": " + version + " found in " + index)
+            logging.verbose(package + ": provided by: " + provider_pkgname +
-        ret = index_data
+                            "-" + version + " in " + path)
-        version_last = version
+            ret[provider_pkgname] = provider
+
+    if ret == {} and must_exist:
+        logging.debug("Searched in APKINDEX files: " + ", ".join(indexes))
+        raise RuntimeError("Could not find package '" + package + "'!")
 
-    # No result log entry
-    if not ret:
-        logging.verbose(package + ": no match found in any APKINDEX.tar.gz!")
     return ret
+
+
+def package(args, package, arch=None, must_exist=True, indexes=None):
+    """
+    Get a specific package's data from an apkindex.
+
+    :param package: of which you want to have the apkindex data
+    :param arch: defaults to native arch, only relevant for indexes=None
+    :param must_exist: When set to true, raise an exception when the package is
+                       not provided at all.
+    :param indexes: list of APKINDEX.tar.gz paths, defaults to all index files
+                    (depending on arch)
+    :returns: a dictionary with the following structure:
+              { "arch": "noarch",
+                "depends": ["busybox-extras", "lddtree", ... ],
+                "pkgname": "postmarketos-mkinitfs",
+                "provides": ["mkinitfs=0.0.1"],
+                "version": "0.0.4-r10" }
+              or None when the package was not found.
+    """
+    # Provider with the same package
+    package_providers = providers(args, package, arch, must_exist, indexes)
+    if package in package_providers:
+        return package_providers[package]
+
+    # Any provider
+    if package_providers:
+        provider_pkgname = list(package_providers.keys())[0]
+        if len(package_providers) != 1:
+            logging.debug(package + ": provided by multiple packages (" +
+                          ", ".join(package_providers) + "), picked " +
+                          provider_pkgname)
+        return package_providers[provider_pkgname]
+
+    # No provider
+    if must_exist:
+        raise RuntimeError("Package '" + package + "' not found in any"
+                           " APKINDEX.")
+    return None

pmb/parse/arguments.py

@@ -249,6 +249,8 @@ def arguments():
     build_init = sub.add_parser("build_init", help="initialize build"
                                 " environment (usually you do not need to call this)")
     chroot = sub.add_parser("chroot", help="start shell in chroot")
+    chroot.add_argument("--add", help="build/install comma separated list of"
+                        " packages in the chroot before entering it")
     chroot.add_argument("command", default=["sh"], help="command"
                         " to execute inside the chroot. default: sh", nargs='*')
     for action in [build_init, chroot]:

pmb/parse/depends.py

@@ -20,37 +20,119 @@ import logging
 import pmb.chroot
 import pmb.chroot.apk
 import pmb.parse.apkindex
+import pmb.parse.arch
 
 
-def recurse_error_message(pkgname, in_aports, in_apkindexes):
+def package_from_aports(args, pkgname_depend):
-    ret = "Could not find package '" + pkgname + "'"
+    """
-    if in_aports:
+    :returns: None when there is no aport, or a dict with the keys pkgname,
-        ret += " in the aports folder"
+              depends, version. The version is the combined pkgver and pkgrel.
-        if in_apkindexes:
+    """
-            ret += " and could not find it"
+    # Get the aport
-    if in_apkindexes:
+    aport = pmb.build.find_aport(args, pkgname_depend, False)
-        ret += " in any APKINDEX"
+    if not aport:
-    return ret + "."
+        return None
+
+    # Parse its version
+    apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
+    pkgname = apkbuild["pkgname"]
+    version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
+
+    # Return the dict
+    logging.verbose(pkgname_depend + ": provided by: " + pkgname + "-" +
+                    version + " in " + aport)
+    return {"pkgname": pkgname,
+            "depends": apkbuild["depends"],
+            "version": version}
 
 
-def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
+def package_provider(args, pkgname, pkgnames_install, suffix="native"):
-            strict=False):
+    """
+    :param pkgnames_install: packages to be installed
+    :returns: a block from the apkindex: {"pkgname": "...", ...}
+              or None (no provider found)
+    """
+    # Get all providers
+    arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
+    providers = pmb.parse.apkindex.providers(args, pkgname, arch, False)
+
+    # 0. No provider
+    if len(providers) == 0:
+        return None
+
+    # 1. Only one provider
+    logging.verbose(pkgname + ": provided by: " + ", ".join(providers))
+    if len(providers) == 1:
+        return list(providers.values())[0]
+
+    # 2. Provider with the same package name
+    if pkgname in providers:
+        logging.verbose(pkgname + ": choosing package of the same name as"
+                        " provider")
+        return providers[pkgname]
+
+    # 3. Pick a package that will be installed anyway
+    for provider_pkgname, provider in providers.items():
+        if provider_pkgname in pkgnames_install:
+            logging.verbose(pkgname + ": choosing provider '" +
+                            provider_pkgname + "', because it will be"
+                            " installed anyway")
+            return provider
+
+    # 4. Pick a package that is already installed
+    installed = pmb.chroot.apk.installed(args, suffix)
+    for provider_pkgname, provider in providers.items():
+        if provider_pkgname in installed:
+            logging.verbose(pkgname + ": choosing provider '" +
+                            provider_pkgname + "', because it is installed in"
+                            " the '" + suffix + "' chroot already")
+            return provider
+
+    # 5. Pick the first one
+    provider_pkgname = list(providers.keys())[0]
+    logging.debug(pkgname + " has multiple providers (" +
+                  ", ".join(providers) + "), picked: " + provider_pkgname)
+    return providers[provider_pkgname]
+
+
+def package_from_index(args, pkgname_depend, pkgnames_install, package_aport,
+                       suffix="native"):
+    """
+    :returns: None when there is no aport and no binary package, or a dict with
+              the keys pkgname, depends, version from either the aport or the
+              binary package provider.
+    """
+    # No binary package
+    provider = package_provider(args, pkgname_depend, pkgnames_install, suffix)
+    if not provider:
+        return package_aport
+
+    # Binary package outdated
+    if (package_aport and pmb.parse.version.compare(package_aport["version"],
+                                                    provider["version"]) == 1):
+        logging.verbose(pkgname_depend + ": binary package is outdated")
+        return package_aport
+
+    # Binary up to date (#893: overrides aport, so we have sonames in depends)
+    if package_aport:
+        logging.verbose(pkgname_depend + ": binary package is"
+                        " up to date, using binary dependencies"
+                        " instead of the ones from the aport")
+    return provider
+
+
+def recurse(args, pkgnames, suffix="native"):
     """
     Find all dependencies of the given pkgnames.
 
-    :param in_apkindexes: look through all APKINDEX files (with the specified arch)
+    :param suffix: the chroot suffix to resolve dependencies for. If a package
-    :param in_aports: look through the aports folder
+                   has multiple providers, we look at the installed packages in
-    :param strict: raise RuntimeError, when a dependency can not be found.
+                   the chroot to make a decision (see package_provider()).
+    :returns: list of pkgnames: consists of the initial pkgnames plus all
+              depends
     """
-    logging.debug("Calculate depends of packages " + str(pkgnames) +
+    logging.debug("(" + suffix + ") calculate depends of " +
-                  ", arch: " + arch)
+                  ", ".join(pkgnames) + " (pmbootstrap -v for details)")
-    logging.verbose("Search in_aports: " + str(in_aports) + ", in_apkindexes: " +
-                    str(in_apkindexes))
-
-    # Sanity check
-    if not in_apkindexes and not in_aports:
-        raise RuntimeError("Set at least one of in_apkindexes or in_aports to"
-                           " True.")
 
     # Iterate over todo-list until is is empty
     todo = list(pkgnames)
@@ -62,64 +144,28 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
             continue
 
         # Get depends and pkgname from aports
-        depends = None
+        pkgnames_install = list(ret) + todo
-        pkgname = None
+        package = package_from_aports(args, pkgname_depend)
-        version = None
+        package = package_from_index(args, pkgname_depend, pkgnames_install,
-        if in_aports:
+                                     package, suffix)
-            aport = pmb.build.find_aport(args, pkgname_depend, False)
-            if aport:
-                apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
-                depends = apkbuild["depends"]
-                version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
-                logging.verbose(pkgname_depend + ": " + version +
-                                " found in " + aport)
-                if pkgname_depend in apkbuild["subpackages"]:
-                    pkgname = pkgname_depend
-                else:
-                    pkgname = apkbuild["pkgname"]
-
-        # Get depends and pkgname from APKINDEX
-        if in_apkindexes:
-            index_data = pmb.parse.apkindex.read_any_index(args, pkgname_depend,
-                                                           arch)
-            if index_data:
-                # The binary package's depends override the aport's depends in
-                # case it has the same or a higher version. Binary packages have
-                # sonames in their dependencies, which we need to detect
-                # breakage (#893).
-                outdated = (version and pmb.parse.version.compare(version,
-                            index_data["version"]) == 1)
-                if not outdated:
-                    if version:
-                        logging.verbose(pkgname_depend + ": binary package is"
-                                        " up to date, using binary dependencies"
-                                        " instead of the ones from the aport")
-                    depends = index_data["depends"]
-                    pkgname = index_data["pkgname"]
 
         # Nothing found
-        if pkgname is None and strict:
+        if not package:
             logging.info("NOTE: Run 'pmbootstrap pkgrel_bump --auto' to mark"
                          " packages with outdated dependencies for rebuild."
                          " This will most likely fix this issue (soname"
                          " bump?).")
-            logging.info("NOTE: More dependency calculation logging with"
+            raise RuntimeError("Could not find package '" + pkgname_depend +
-                         " 'pmbootstrap -v'.")
+                               "' in any aports folder or APKINDEX.")
-            raise RuntimeError(
-                recurse_error_message(
-                    pkgname_depend,
-                    in_aports,
-                    in_apkindexes))
 
         # Append to todo/ret (unless it is a duplicate)
-        if pkgname != pkgname_depend:
+        pkgname = package["pkgname"]
-            logging.verbose(pkgname_depend + ": provided by '" + pkgname + "'")
         if pkgname in ret:
             logging.verbose(pkgname + ": already found")
         else:
+            depends = package["depends"]
             logging.verbose(pkgname + ": depends on: " + ",".join(depends))
             if depends:
                 todo += depends
             ret.append(pkgname)
-
     return ret

test/test_apk_static.py

@@ -100,9 +100,7 @@ def test_signature_verification(args, tmpdir):
     if os.path.exists(args.work + "/apk.static"):
         os.remove(args.work + "/apk.static")
 
-    apk_index = pmb.chroot.apk_static.download(args, "APKINDEX.tar.gz")
+    version = pmb.parse.apkindex.package(args, "apk-tools-static")["version"]
-    version = pmb.parse.apkindex.read(args, "apk-tools-static",
-                                      apk_index)["version"]
     apk_path = pmb.chroot.apk_static.download(args,
                                               "apk-tools-static-" + version + ".apk")
 

test/test_build_is_necessary.py

@@ -41,7 +41,7 @@ def args(request, tmpdir):
     apkindex_path = str(tmpdir) + "/APKINDEX.tar.gz"
     open(apkindex_path, "a").close()
     lastmod = os.path.getmtime(apkindex_path)
-    args.cache["apkindex"][apkindex_path] = {"lastmod": lastmod, "ret": {}}
+    args.cache["apkindex"][apkindex_path] = {"lastmod": lastmod, "multiple": {}}
     return args
 
 
@@ -53,7 +53,8 @@ def cache_apkindex(args, version):
     """
     apkindex_path = list(args.cache["apkindex"].keys())[0]
 
-    args.cache["apkindex"][apkindex_path]["ret"]["hello-world"]["version"] = version
+    providers = args.cache["apkindex"][apkindex_path]["multiple"]["hello-world"]
+    providers["hello-world"]["version"] = version
 
 
 def test_build_is_necessary(args):
@@ -62,22 +63,23 @@ def test_build_is_necessary(args):
     apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
     apkbuild["pkgver"] = "1"
     apkbuild["pkgrel"] = "2"
-    apkindex_path = list(args.cache["apkindex"].keys())[0]
+    indexes = list(args.cache["apkindex"].keys())
-    args.cache["apkindex"][apkindex_path]["ret"] = {
+    apkindex_path = indexes[0]
-        "hello-world": {"pkgname": "hello-world", "version": "1-r2"}
+    cache = {"hello-world": {"hello-world": {"pkgname": "hello-world",
-    }
+                                             "version": "1-r2"}}}
+    args.cache["apkindex"][apkindex_path]["multiple"] = cache
 
     # Binary repo has a newer version
     cache_apkindex(args, "999-r1")
-    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
+    assert pmb.build.is_necessary(args, None, apkbuild, indexes) is False
 
     # Aports folder has a newer version
     cache_apkindex(args, "0-r0")
-    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
+    assert pmb.build.is_necessary(args, None, apkbuild, indexes) is True
 
     # Same version
     cache_apkindex(args, "1-r2")
-    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
+    assert pmb.build.is_necessary(args, None, apkbuild, indexes) is False
 
 
 def test_build_is_necessary_no_binary_available(args):
@@ -85,7 +87,7 @@ def test_build_is_necessary_no_binary_available(args):
     APKINDEX cache is set up to fake an empty APKINDEX, which means, that the
     hello-world package has not been built yet.
     """
-    apkindex_path = list(args.cache["apkindex"].keys())[0]
+    indexes = list(args.cache["apkindex"].keys())
     aport = pmb.build.other.find_aport(args, "hello-world")
     apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
-    assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
+    assert pmb.build.is_necessary(args, None, apkbuild, indexes) is True

test/test_keys.py

@@ -43,8 +43,7 @@ def args(request):
 def test_keys(args):
     # Get the alpine-keys apk filename
     pmb.chroot.init(args)
-    info = pmb.parse.apkindex.read_any_index(args, "alpine-keys")
+    version = pmb.parse.apkindex.package(args, "alpine-keys")["version"]
-    version = info["version"]
     pattern = (args.work + "/cache_apk_" + args.arch_native + "/alpine-keys-" +
                version + ".*.apk")
     filename = os.path.basename(glob.glob(pattern)[0])

test/test_parse_apkindex.py

@@ -17,6 +17,11 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+"""
+This file tests all functions from pmb.parse.apkindex.
+"""
+
+import collections
 import os
 import pytest
 import sys
@@ -40,20 +45,252 @@ def args(tmpdir, request):
     return args
 
 
-def test_read_any_index_highest_version(args, monkeypatch):
+def test_parse_next_block_exceptions(args):
-    # Return 3 fake "files" for pmb.helpers.repo.apkindex_files()
+    # Mapping of input files (inside the /test/testdata/apkindex) to
-    def return_fake_files(*arguments):
+    # error message substrings
-        return ["0", "1", "2"]
+    mapping = {"key_twice": "specified twice",
-    monkeypatch.setattr(pmb.helpers.repo, "apkindex_files",
+               "key_missing": "Missing required key",
-                        return_fake_files)
+               "new_line_missing": "does not end with a new line!"}
 
-    # Return fake index data for the "files"
+    # Parse the files
-    def return_fake_read(args, package, path, must_exist=True):
+    for file, error_substr in mapping.items():
-        return {"0": {"pkgname": "test", "version": "2"},
+        path = pmb.config.pmb_src + "/test/testdata/apkindex/" + file
-                "1": {"pkgname": "test", "version": "3"},
+        with open(path, "r", encoding="utf-8") as handle:
-                "2": {"pkgname": "test", "version": "1"}}[path]
+            lines = handle.readlines()
-    monkeypatch.setattr(pmb.parse.apkindex, "read", return_fake_read)
+
+        with pytest.raises(RuntimeError) as e:
+            pmb.parse.apkindex.parse_next_block(args, path, lines, [0])
+        assert error_substr in str(e.value)
+
+
+def test_parse_next_block_no_error(args):
+    # Read the file
+    func = pmb.parse.apkindex.parse_next_block
+    path = pmb.config.pmb_src + "/test/testdata/apkindex/no_error"
+    with open(path, "r", encoding="utf-8") as handle:
+        lines = handle.readlines()
+
+    # First block
+    start = [0]
+    block = {'arch': 'x86_64',
+             'depends': [],
+             'origin': 'musl',
+             'pkgname': 'musl',
+             'provides': ['so:libc.musl-x86_64.so.1'],
+             'timestamp': '1515217616',
+             'version': '1.1.18-r5'}
+    assert func(args, path, lines, start) == block
+    assert start == [24]
+
+    # Second block
+    block = {'arch': 'x86_64',
+             'depends': ['ca-certificates',
+                         'so:libc.musl-x86_64.so.1',
+                         'so:libcurl.so.4',
+                         'so:libz.so.1'],
+             'origin': 'curl',
+             'pkgname': 'curl',
+             'provides': ['cmd:curl'],
+             'timestamp': '1512030418',
+             'version': '7.57.0-r0'}
+    assert func(args, path, lines, start) == block
+    assert start == [45]
+
+    # No more blocks
+    assert func(args, path, lines, start) is None
+    assert start == [45]
+
+
+def test_parse_add_block(args):
+    func = pmb.parse.apkindex.parse_add_block
+    multiple_providers = False
+
+    # One package without alias
+    ret = {}
+    block = {"pkgname": "test", "version": "2"}
+    alias = None
+    func(ret, block, alias, multiple_providers)
+    assert ret == {"test": block}
+
+    # Older packages must not overwrite newer ones
+    block_old = {"pkgname": "test", "version": "1"}
+    func(ret, block_old, alias, multiple_providers)
+    assert ret == {"test": block}
+
+    # Newer packages must overwrite older ones
+    block_new = {"pkgname": "test", "version": "3"}
+    func(ret, block_new, alias, multiple_providers)
+    assert ret == {"test": block_new}
+
+    # Add package with alias
+    alias = "test_alias"
+    func(ret, block_new, alias, multiple_providers)
+    assert ret == {"test": block_new, "test_alias": block_new}
+
+
+def test_parse_add_block_multiple_providers(args):
+    func = pmb.parse.apkindex.parse_add_block
+
+    # One package without alias
+    ret = {}
+    block = {"pkgname": "test", "version": "2"}
+    alias = None
+    func(ret, block, alias)
+    assert ret == {"test": {"test": block}}
+
+    # Older packages must not overwrite newer ones
+    block_old = {"pkgname": "test", "version": "1"}
+    func(ret, block_old, alias)
+    assert ret == {"test": {"test": block}}
+
+    # Newer packages must overwrite older ones
+    block_new = {"pkgname": "test", "version": "3"}
+    func(ret, block_new, alias)
+    assert ret == {"test": {"test": block_new}}
+
+    # Add package with alias
+    alias = "test_alias"
+    func(ret, block_new, alias)
+    assert ret == {"test": {"test": block_new},
+                   "test_alias": {"test": block_new}}
+
+    # Add another package with the same alias
+    alias = "test_alias"
+    block_test2 = {"pkgname": "test2", "version": "1"}
+    func(ret, block_test2, alias)
+    assert ret == {"test": {"test": block_new},
+                   "test_alias": {"test": block_new, "test2": block_test2}}
+
+
+def test_parse_invalid_path(args):
+    assert pmb.parse.apkindex.parse(args, "/invalid/path/APKINDEX") == {}
+
+
+def test_parse_cached(args, tmpdir):
+    # Create a real file (cache looks at the last modified date)
+    path = str(tmpdir) + "/APKINDEX"
+    pmb.helpers.run.user(args, ["touch", path])
+    lastmod = os.path.getmtime(path)
+
+    # Fill the cache
+    args.cache["apkindex"][path] = {
+        "lastmod": lastmod,
+        "multiple": "cached_result_multiple",
+        "single": "cached_result_single",
+    }
+
+    # Verify cache usage
+    func = pmb.parse.apkindex.parse
+    assert func(args, path, True) == "cached_result_multiple"
+    assert func(args, path, False) == "cached_result_single"
+
+    # Make cache invalid
+    args.cache["apkindex"][path]["lastmod"] -= 10
+    assert func(args, path, True) == {}
+
+    # Delete the cache (run twice for both code paths)
+    assert pmb.parse.apkindex.clear_cache(args, path) is True
+    assert args.cache["apkindex"] == {}
+    assert pmb.parse.apkindex.clear_cache(args, path) is False
+
+
+def test_parse(args):
+    path = pmb.config.pmb_src + "/test/testdata/apkindex/no_error"
+    block_musl = {'arch': 'x86_64',
+                  'depends': [],
+                  'origin': 'musl',
+                  'pkgname': 'musl',
+                  'provides': ['so:libc.musl-x86_64.so.1'],
+                  'timestamp': '1515217616',
+                  'version': '1.1.18-r5'}
+    block_curl = {'arch': 'x86_64',
+                  'depends': ['ca-certificates',
+                              'so:libc.musl-x86_64.so.1',
+                              'so:libcurl.so.4',
+                              'so:libz.so.1'],
+                  'origin': 'curl',
+                  'pkgname': 'curl',
+                  'provides': ['cmd:curl'],
+                  'timestamp': '1512030418',
+                  'version': '7.57.0-r0'}
+
+    # Test without multiple_providers
+    ret_single = {'cmd:curl': block_curl,
+                  'curl': block_curl,
+                  'musl': block_musl,
+                  'so:libc.musl-x86_64.so.1': block_musl}
+    assert pmb.parse.apkindex.parse(args, path, False) == ret_single
+    assert args.cache["apkindex"][path]["single"] == ret_single
+
+    # Test with multiple_providers
+    ret_multiple = {'cmd:curl': {"curl": block_curl},
+                    'curl': {"curl": block_curl},
+                    'musl': {"musl": block_musl},
+                    'so:libc.musl-x86_64.so.1': {"musl": block_musl}}
+    assert pmb.parse.apkindex.parse(args, path, True) == ret_multiple
+    assert args.cache["apkindex"][path]["multiple"] == ret_multiple
+
+
+def test_providers_invalid_package(args, tmpdir):
+    # Create empty APKINDEX
+    path = str(tmpdir) + "/APKINDEX"
+    pmb.helpers.run.user(args, ["touch", path])
+
+    # Test with must_exist=False
+    func = pmb.parse.apkindex.providers
+    package = "test"
+    indexes = [path]
+    assert func(args, package, None, False, indexes) == {}
+
+    # Test with must_exist=True
+    with pytest.raises(RuntimeError) as e:
+        func(args, package, None, True, indexes)
+    assert str(e.value).startswith("Could not find package")
+
+
+def test_providers_highest_version(args, monkeypatch):
+    """
+    In this test, we simulate 3 APKINDEX files ("i0", "i1", "i2" instead of
+    full paths to real APKINDEX.tar.gz files), and each of them has a different
+    version of the same package. The highest version must win, no matter in
+    which order the APKINDEX files are processed.
+    """
+    # Fake parse function
+    def return_fake_parse(args, path):
+        version_mapping = {"i0": "2", "i1": "3", "i2": "1"}
+        package_block = {"pkgname": "test", "version": version_mapping[path]}
+        return {"test": {"test": package_block}}
+    monkeypatch.setattr(pmb.parse.apkindex, "parse", return_fake_parse)
 
     # Verify that it picks the highest version
-    func = pmb.parse.apkindex.read_any_index
+    func = pmb.parse.apkindex.providers
-    assert func(args, "test")["version"] == "3"
+    providers = func(args, "test", indexes=["i0", "i1", "i2"])
+    assert providers["test"]["version"] == "3"
+
+
+def test_package(args, monkeypatch):
+    # Override pmb.parse.apkindex.providers()
+    providers = collections.OrderedDict()
+
+    def return_providers(*args, **kwargs):
+        return providers
+    monkeypatch.setattr(pmb.parse.apkindex, "providers", return_providers)
+
+    # Provider with the same pkgname
+    func = pmb.parse.apkindex.package
+    pkgname = "test"
+    providers = {"test2": {"pkgname": "test2"}, "test": {"pkgname": "test"}}
+    assert func(args, pkgname) == {"pkgname": "test"}
+
+    # First provider
+    providers = {"test2": {"pkgname": "test2"}, "test3": {"pkgname": "test3"}}
+    assert func(args, pkgname) == {"pkgname": "test2"}
+
+    # No provider (with must_exist)
+    providers = {}
+    with pytest.raises(RuntimeError) as e:
+        func(args, pkgname)
+    assert "not found in any APKINDEX" in str(e.value)
+
+    # No provider (without must_exist)
+    assert func(args, pkgname, must_exist=False) is None

test/test_parse_depends.py

@@ -0,0 +1,177 @@
+"""
+Copyright 2018 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+"""
+This file tests all functions from pmb.parse.depends.
+"""
+
+import collections
+import os
+import pytest
+import sys
+
+# Import from parent directory
+sys.path.append(os.path.realpath(
+    os.path.join(os.path.dirname(__file__) + "/..")))
+import pmb.config
+import pmb.config.init
+import pmb.helpers.logging
+import pmb.parse.depends
+
+
+@pytest.fixture
+def args(tmpdir, request):
+    import pmb.parse
+    sys.argv = ["pmbootstrap", "init"]
+    args = pmb.parse.arguments()
+    args.log = args.work + "/log_testsuite.txt"
+    pmb.helpers.logging.init(args)
+    request.addfinalizer(args.logfd.close)
+    return args
+
+
+def test_package_from_aports(args):
+    func = pmb.parse.depends.package_from_aports
+    assert func(args, "invalid-package") is None
+    assert func(args, "hello-world") == {"pkgname": "hello-world",
+                                         "depends": [],
+                                         "version": "1-r4"}
+
+
+def test_package_provider(args, monkeypatch):
+    # Override pmb.parse.apkindex.providers()
+    providers = collections.OrderedDict()
+
+    def return_providers(*args, **kwargs):
+        return providers
+    monkeypatch.setattr(pmb.parse.apkindex, "providers", return_providers)
+
+    # Override pmb.chroot.apk.installed()
+    installed = {}
+
+    def return_installed(*args, **kwards):
+        return installed
+    monkeypatch.setattr(pmb.chroot.apk, "installed", return_installed)
+
+    # 0. No provider
+    pkgname = "test"
+    pkgnames_install = []
+    func = pmb.parse.depends.package_provider
+    assert func(args, pkgname, pkgnames_install) is None
+
+    # 1. Only one provider
+    package = {"pkgname": "test", "version": "1234"}
+    providers = {"test": package}
+    assert func(args, pkgname, pkgnames_install) == package
+
+    # 2. Provider with the same package name
+    package_two = {"pkgname": "test-two", "provides": ["test"]}
+    providers = {"test-two": package_two, "test": package}
+    assert func(args, pkgname, pkgnames_install) == package
+
+    # 3. Pick a package, that will be installed anyway
+    providers = {"test_": package, "test-two": package_two}
+    installed = {"test_": package}
+    pkgnames_install = ["test-two"]
+    assert func(args, pkgname, pkgnames_install) == package_two
+
+    # 4. Pick a package, that is already installed
+    pkgnames_install = []
+    assert func(args, pkgname, pkgnames_install) == package
+
+    # 5. Pick the first one
+    installed = {}
+    assert func(args, pkgname, pkgnames_install) == package
+
+
+def test_package_from_index(args, monkeypatch):
+    # Override pmb.parse.depends.package_provider()
+    provider = None
+
+    def return_provider(*args, **kwargs):
+        return provider
+    monkeypatch.setattr(pmb.parse.depends, "package_provider",
+                        return_provider)
+
+    func = pmb.parse.depends.package_from_index
+    aport = {"pkgname": "test", "version": "2"}
+    pkgname = "test"
+    pkgnames_install = []
+
+    # No binary package providers
+    assert func(args, pkgname, pkgnames_install, aport) is aport
+
+    # Binary package outdated
+    provider = {"pkgname": "test", "version": "1"}
+    assert func(args, pkgname, pkgnames_install, aport) is aport
+
+    # Binary package up-to-date
+    for version in ["2", "3"]:
+        provider = {"pkgname": "test", "version": version}
+        assert func(args, pkgname, pkgnames_install, aport) is provider
+
+
+def test_recurse_invalid(args, monkeypatch):
+    func = pmb.parse.depends.recurse
+
+    # Invalid package
+    with pytest.raises(RuntimeError) as e:
+        func(args, ["invalid-pkgname"])
+    assert str(e.value).startswith("Could not find package")
+
+
+def return_none(*args, **kwargs):
+    return None
+
+
+def test_recurse(args, monkeypatch):
+    """
+    Test recursing through the following dependencies:
+
+    test:
+        libtest
+        so:libtest.so.1
+    libtest:
+        libtest_depend
+    libtest_depend:
+    so:libtest.so.1:
+        libtest_depend
+    """
+    # Override finding the package in aports: always no result
+    monkeypatch.setattr(pmb.parse.depends, "package_from_aports",
+                        return_none)
+
+    # Override depends returned from APKINDEX
+    depends = {
+        "test": ["libtest", "so:libtest.so.1"],
+        "libtest": ["libtest_depend"],
+        "libtest_depend": [],
+        "so:libtest.so.1": ["libtest_depend"],
+    }
+
+    def package_from_index(args, pkgname, install, aport, suffix):
+        return {"pkgname": pkgname, "depends": depends[pkgname]}
+    monkeypatch.setattr(pmb.parse.depends, "package_from_index",
+                        package_from_index)
+
+    # Run
+    func = pmb.parse.depends.recurse
+    pkgnames = ["test", "so:libtest.so.1"]
+    result = ["test", "so:libtest.so.1", "libtest", "libtest_depend"]
+    assert func(args, pkgnames) == result

test/test_upstream_compatibility.py

@@ -51,7 +51,8 @@ def test_qt_versions(args):
     hash = pmb.helpers.repo.hash(repository)
     index_path = (args.work + "/cache_apk_armhf/APKINDEX." + hash +
                   ".tar.gz")
-    index_data = pmb.parse.apkindex.read(args, "qt5-qtbase", index_path)
+    index_data = pmb.parse.apkindex.package(args, "qt5-qtbase",
+                                            indexes=[index_path])
     pkgver_upstream = index_data["version"].split("-r")[0]
 
     # Iterate over our packages
@@ -101,7 +102,8 @@ def test_aportgen_versions(args):
     generated = "# Automatically generated aport, do not edit!"
     for pkgname, pattern in map.items():
         # Upstream version
-        index_data = pmb.parse.apkindex.read(args, pkgname, index_path)
+        index_data = pmb.parse.apkindex.package(args, pkgname,
+                                                indexes=[index_path])
         version_upstream = index_data["version"]
 
         # Iterate over our packages

test/testdata/apkindex/key_missing

@@ -0,0 +1,23 @@
+C:Q1gKkFdQUwKAmcUpGY8VaErq0uHNo=
+P:musl
+A:x86_64
+S:357094
+I:581632
+T:the musl c library (libc) implementation
+U:http://www.musl-libc.org/
+L:MIT
+o:musl
+m:Timo Ter  s <timo.teras@iki.fi>
+t:1515217616
+c:6cc1d4e6ac35607dd09003e4d013a0d9c4800c49
+p:so:libc.musl-x86_64.so.1=1
+F:lib
+R:libc.musl-x86_64.so.1
+a:0:0:777
+Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI=
+R:ld-musl-x86_64.so.1
+a:0:0:755
+Z:Q1DadJ0cqdT+ImyeY5FgTdZWaLnyQ=
+F:usr
+F:usr/lib
+

test/testdata/apkindex/key_twice

@@ -0,0 +1,25 @@
+C:Q1gKkFdQUwKAmcUpGY8VaErq0uHNo=
+P:musl
+V:1.1.18-r5
+V:1.1.18-r5
+A:x86_64
+S:357094
+I:581632
+T:the musl c library (libc) implementation
+U:http://www.musl-libc.org/
+L:MIT
+o:musl
+m:Timo Ter  s <timo.teras@iki.fi>
+t:1515217616
+c:6cc1d4e6ac35607dd09003e4d013a0d9c4800c49
+p:so:libc.musl-x86_64.so.1=1
+F:lib
+R:libc.musl-x86_64.so.1
+a:0:0:777
+Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI=
+R:ld-musl-x86_64.so.1
+a:0:0:755
+Z:Q1DadJ0cqdT+ImyeY5FgTdZWaLnyQ=
+F:usr
+F:usr/lib
+

test/testdata/apkindex/new_line_missing

@@ -0,0 +1,23 @@
+C:Q1gKkFdQUwKAmcUpGY8VaErq0uHNo=
+P:musl
+V:1.1.18-r5
+A:x86_64
+S:357094
+I:581632
+T:the musl c library (libc) implementation
+U:http://www.musl-libc.org/
+L:MIT
+o:musl
+m:Timo Ter  s <timo.teras@iki.fi>
+t:1515217616
+c:6cc1d4e6ac35607dd09003e4d013a0d9c4800c49
+p:so:libc.musl-x86_64.so.1=1
+F:lib
+R:libc.musl-x86_64.so.1
+a:0:0:777
+Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI=
+R:ld-musl-x86_64.so.1
+a:0:0:755
+Z:Q1DadJ0cqdT+ImyeY5FgTdZWaLnyQ=
+F:usr
+F:usr/lib

test/testdata/apkindex/no_error

@@ -0,0 +1,45 @@
+C:Q1gKkFdQUwKAmcUpGY8VaErq0uHNo=
+P:musl
+V:1.1.18-r5
+A:x86_64
+S:357094
+I:581632
+T:the musl c library (libc) implementation
+U:http://www.musl-libc.org/
+L:MIT
+o:musl
+m:Timo Ter  s <timo.teras@iki.fi>
+t:1515217616
+c:6cc1d4e6ac35607dd09003e4d013a0d9c4800c49
+p:so:libc.musl-x86_64.so.1=1
+F:lib
+R:libc.musl-x86_64.so.1
+a:0:0:777
+Z:Q17yJ3JFNypA4mxhJJr0ou6CzsJVI=
+R:ld-musl-x86_64.so.1
+a:0:0:755
+Z:Q1DadJ0cqdT+ImyeY5FgTdZWaLnyQ=
+F:usr
+F:usr/lib
+
+C:Q1iundrWyXyQtSTZ9h2qqh44cZcYA=
+P:curl
+V:7.57.0-r0
+A:x86_64
+S:118233
+I:217088
+T:An URL retrival utility and library
+U:http://curl.haxx.se
+L:MIT
+o:curl
+m:Natanael Copa <ncopa@alpinelinux.org>
+t:1512030418
+c:d19c5b26c70a3055c5d6c7d2f15587f62a33a1fe
+D:ca-certificates so:libc.musl-x86_64.so.1 so:libcurl.so.4 so:libz.so.1
+p:cmd:curl
+F:usr
+F:usr/bin
+R:curl
+a:0:0:755
+Z:Q1tlqDmZcIJJXo+ScFT6Nd31EPrBM=
+