Fix #731: Create symlinks for noarch-subpackages (#740)

* apkindex:
  * Also parse the architecture field
* symlink_noarch_package:
  * Renamed to symlink_noarch_packages
  * Always work on all packages (so we don't need to guess which
    subpackages have been generated after a certain build)
  * Get invoked when running 'pmbootstrap index'
  * Use 'apk index' to generate one index, where the architecture
    does not get rewritten (abuild does that by default, due to
    Alpine's repos not having a 'noarch' folder and diverging from
    that doesn't make things easier for us). That goes super fast,
    and then we know which packages are noarch packages and can
    create the symlinks.
* Made output less verbose:
  * Use -q for 'apk index' when calling it directly (when it gets
    called by abuild we can't control that)
  * Output that the APKINDEXes get reindexed only to the 'pmbootstrap
    log'.

aports/main/postmarketos-base/APKBUILD

@@ -1,6 +1,6 @@
 pkgname=postmarketos-base
 pkgver=3
-pkgrel=7
+pkgrel=8
 pkgdesc="Meta package for minimal postmarketOS base"
 url="https://github.com/postmarketOS"
 arch="noarch"

pmb/build/__init__.py

@@ -20,6 +20,6 @@ along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 from pmb.build.init import init
 from pmb.build.checksum import checksum
 from pmb.build.other import copy_to_buildpath, is_necessary, \
-    symlink_noarch_package, find_aport, ccache_stats, index_repo
+    symlink_noarch_packages, find_aport, ccache_stats, index_repo
 from pmb.build.package import package
 from pmb.build.menuconfig import menuconfig

pmb/build/other.py

@@ -129,7 +129,8 @@ def aports_files_out_of_sync_with_git(args, package=None):
     ret = []
     if git_root and os.path.exists(git_root):
         # Find all out of sync files
-        tracked = pmb.helpers.git.find_out_of_sync_files_tracked(args, git_root)
+        tracked = pmb.helpers.git.find_out_of_sync_files_tracked(
+            args, git_root)
         untracked = pmb.helpers.run.user(
             args, ["git", "ls-files", "--others", "--exclude-standard"],
             working_dir=git_root, return_stdout=True)
@@ -247,9 +248,9 @@ def index_repo(args, arch=None):
     for path in paths:
         path_arch = os.path.basename(path)
         path_repo_chroot = "/home/user/packages/user/" + path_arch
-        logging.info("(native) index " + path_arch + " repository")
+        logging.debug("(native) index " + path_arch + " repository")
         commands = [
-            ["apk", "index", "--output", "APKINDEX.tar.gz_",
+            ["apk", "-q", "index", "--output", "APKINDEX.tar.gz_",
              "--rewrite-arch", path_arch, "*.apk"],
             ["abuild-sign", "APKINDEX.tar.gz_"],
             ["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
@@ -260,21 +261,44 @@ def index_repo(args, arch=None):
                                        "/APKINDEX.tar.gz")
 
 
-def symlink_noarch_package(args, arch_apk):
+def symlink_noarch_packages(args):
     """
-    :param arch_apk: for example: x86_64/mypackage-1.2.3-r0.apk
+    All noarch packages from the native architecture folder (x86_64 usually)
+    get symlinked to all other architectures.
     """
-
-    for arch in pmb.config.build_device_architectures:
-        # Create the arch folder
+    # Create the arch folders
+    architectures = pmb.config.build_device_architectures
+    logging.debug("Symlink noarch-packages to " + ", ".join(architectures))
+    for arch in architectures:
         arch_folder = "/home/user/packages/user/" + arch
         arch_folder_outside = args.work + "/packages/" + arch
         if not os.path.exists(arch_folder_outside):
             pmb.chroot.user(args, ["mkdir", "-p", arch_folder])
 
-        # Add symlink, rewrite index
-        pmb.chroot.user(args, ["ln", "-sf", "../" + arch_apk, "."],
-                        working_dir=arch_folder)
+    # Create an APKINDEX *without* replaced architectures (that is much
+    # faster than reading each apk file with Python!)
+    index = "/tmp/APKINDEX_without_replaced_archs"
+    index_outside = args.work + "/chroot_native" + index
+    pmb.chroot.user(args, ["apk", "-q", "index", "--output", index, "*.apk"],
+                    working_dir="/home/user/packages/user/" + args.arch_native)
+
+    # Iterate over noarch packages
+    for package, data in pmb.parse.apkindex.parse(args, index_outside).items():
+        if data["arch"] != "noarch":
+            continue
+
+        # Create missing symlinks
+        apk_file = data["pkgname"] + "-" + data["version"] + ".apk"
+        for arch in architectures:
+            if os.path.exists(args.work + "/packages/" + arch + "/" + apk_file):
+                continue
+            arch_folder = "/home/user/packages/user/" + arch
+            source = "../" + args.arch_native + "/" + apk_file
+            pmb.chroot.user(args, ["ln", "-sf", source, "."],
+                            working_dir=arch_folder)
+
+    # Rewrite indexes
+    for arch in architectures:
         index_repo(args, arch)
 
 

pmb/build/package.py

@@ -124,9 +124,9 @@ def package(args, pkgname, carch, force=False, buildinfo=False, strict=False):
         pmb.build.buildinfo.write(args, output, carch_buildenv, suffix,
                                   apkbuild)
 
-    # Symlink noarch packages
+    # Symlink noarch package (and subpackages)
     if "noarch" in apkbuild["arch"]:
-        pmb.build.symlink_noarch_package(args, output)
+        pmb.build.symlink_noarch_packages(args)
 
     # Clean up (APKINDEX cache, depends when strict)
     pmb.parse.apkindex.clear_cache(args, args.work + "/packages/" +

pmb/helpers/frontend.py

@@ -129,7 +129,8 @@ def config(args):
 
 
 def index(args):
-    pmb.build.index_repo(args)
+    pmb.build.index_repo(args, args.arch_native)
+    pmb.build.symlink_noarch_packages(args)
 
 
 def initfs(args):

pmb/parse/apkindex.py

@@ -34,10 +34,11 @@ def parse_next_block(args, path, lines, start):
                   "by reference". Example: [5]
     :param lines: all lines from the "APKINDEX" file inside the archive
     :returns: a dictionary with the following structure:
-              { "pkgname": "postmarketos-mkinitfs",
-                "version": "0.0.4-r10",
+              { "arch": "noarch",
                 "depends": ["busybox-extras", "lddtree", ... ],
+                "pkgname": "postmarketos-mkinitfs",
                 "provides": ["mkinitfs=0.0.1"],
+                "version": "0.0.4-r10",
               }
     :returns: None, when there are no more blocks
     """
@@ -45,6 +46,7 @@ def parse_next_block(args, path, lines, start):
     # Parse until we hit an empty line or end of file
     ret = {}
     mapping = {
+        "A": "arch",
         "P": "pkgname",
         "V": "version",
         "D": "depends",