WIP #64: "lazy reproducible builds"

pmb/build/__init__.py

@@ -23,3 +23,4 @@ from pmb.build.other import copy_to_buildpath, is_necessary, \
     symlink_noarch_package, find_aport, ccache_stats, index_repo
 from pmb.build.package import package
 from pmb.build.menuconfig import menuconfig
+from pmb.build.challenge import challenge

pmb/build/buildinfo.py

@@ -0,0 +1,100 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import json
+import pmb.chroot
+import pmb.chroot.apk
+import pmb.parse.apkindex
+
+
+def get_depends_recursively(args, pkgnames, arch=None):
+    """
+    :param pkgnames: List of pkgnames, for which the dependencies shall be
+                     retrieved.
+    """
+    todo = list(pkgnames)
+    ret = []
+    seen = []
+    while len(todo):
+        pkgname = todo.pop(0)
+        index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch)
+        if not index_data:
+            raise RuntimeError("Could not find dependency " + pkgname +
+                               " of packages " + str(pkgnames) + " in any APKINDEX")
+        pkgname = index_data["pkgname"]
+        if pkgname not in pkgnames and pkgname not in ret:
+            ret.append(pkgname)
+        for depend in index_data["depends"]:
+            if depend not in ret:
+                if depend.startswith("!"):
+                    continue
+                for operator in [">", "="]:
+                    if operator in depend:
+                        depend = depend.split(operator)[0]
+                if depend not in seen:
+                    todo.append(depend)
+                    seen.append(depend)
+    return ret
+
+
+def generate(args, apk_path, carch, suffix, apkbuild):
+    """
+    :param apk_path: Path to the .apk file, relative to the packages cache.
+    :param carch: Architecture, that the package has been built for.
+    :apkbuild: Return from pmb.parse.apkbuild().
+    """
+    ret = {"pkgname": apkbuild["pkgname"],
+           "pkgver": apkbuild["pkgver"],
+           "pkgrel": apkbuild["pkgrel"],
+           "carch": carch,
+           "versions": []}
+
+    # Add makedepends versions
+    installed = pmb.chroot.apk.installed(args, suffix)
+    relevant = (apkbuild["makedepends"] +
+                get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"]))
+    for pkgname in relevant:
+        if pkgname in installed:
+            ret["versions"].append(installed[pkgname]["package"])
+    ret["versions"].sort()
+    return ret
+
+
+def write(args, apk_path, carch, suffix, apkbuild):
+    """
+    Write a .buildinfo.json file for a package, right after building it.
+    It stores all information required to rebuild the package, very similar
+    to how they do it in Debian (but as JSON file, so it's easier to parse in
+    Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles
+
+    :param apk_path: Path to the .apk file, relative to the packages cache.
+    :param carch: Architecture, that the package has been built for.
+    :apkbuild: Return from pmb.parse.apkbuild().
+    """
+    # Write to temp
+    if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"):
+        pmb.chroot.root(args, ["rm", "/tmp/buildinfo"])
+    buildinfo = generate(args, apk_path, carch, suffix, apkbuild)
+    with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle:
+        handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n")
+
+    # Move to packages
+    pmb.chroot.root(args, ["chown", "user:user", "/tmp/buildinfo"])
+    pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/user/packages/user/" +
+                           apk_path + ".buildinfo.json"])

pmb/build/challenge.py

@@ -0,0 +1,109 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import logging
+import json
+import os
+import tarfile
+import tempfile
+import filecmp
+import shutil
+import pmb.build
+import pmb.parse.apkbuild
+
+
+def diff(args, apk_a, apk_b):
+    logging.info("Challenge " + apk_a)
+    with tarfile.open(apk_a, "r:gz") as tar_a:
+        with tarfile.open(apk_b, "r:gz") as tar_b:
+            # List of files must be the same
+            list_a = sorted(tar_a.getnames())
+            list_b = tar_b.getnames()
+            list_b.sort()
+            if list_a != list_b:
+                raise RuntimeError(
+                    "Both APKs do not contain the same file names!")
+
+            # Iterate through the list
+            for name in list_a:
+                logging.debug("Compare: " + name)
+                if name == ".PKGINFO" or name.startswith(".SIGN.RSA."):
+                    logging.debug(
+                        "=> Skipping, this is expected to be different")
+                    continue
+                temp_files = []
+
+                # Extract
+                for tar in [tar_a, tar_b]:
+                    member = tar.getmember(name)
+                    if member.isdir():
+                        continue
+                    handle, path = tempfile.mkstemp("pmbootstrap")
+                    handle = open(handle, "wb")
+                    shutil.copyfileobj(tar.extractfile(member), handle)
+                    handle.close()
+                    temp_files.append(path)
+                if not len(temp_files):
+                    logging.debug("=> Skipping, this is a directory")
+                    continue
+
+                # Compare and delete
+                equal = filecmp.cmp(
+                    temp_files[0], temp_files[1], shallow=False)
+                for temp_file in temp_files:
+                    os.remove(temp_file)
+                if equal:
+                    logging.debug("=> Equal")
+                else:
+                    raise RuntimeError("File '" + name + "' is different!")
+
+
+def challenge(args, apk_path):
+    # Parse buildinfo
+    buildinfo_path = apk_path + ".buildinfo.json"
+    if not os.path.exists(buildinfo_path):
+        logging.info("NOTE: To create a .buildinfo.json file, use the"
+                     " --buildinfo command while building: 'pmbootstrap build"
+                     " --buildinfo <pkgname>'")
+        raise RuntimeError("Missing file: " + buildinfo_path)
+    with open(buildinfo_path) as handle:
+        buildinfo = json.load(handle)
+
+    # Parse and install all packages listed in versions
+    versions = {}
+    for package in buildinfo["versions"]:
+        split = pmb.chroot.apk.package_split(package)
+        pkgname = split["pkgname"]
+        versions[pkgname] = split
+    pmb.chroot.apk.install(args, versions.keys())
+
+    # Verify the installed versions
+    installed = pmb.chroot.apk.installed(args)
+    for pkgname, split in versions.items():
+        package_installed = installed[pkgname]["package"]
+        package_buildinfo = split["package"]
+        if package_installed != package_buildinfo:
+            raise RuntimeError("Dependency " + pkgname + " version is different"
+                               " (installed: " + package_installed + ","
+                               " buildinfo: " + package_buildinfo + ")!")
+    # Build the package
+    output = pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"],
+                               force=True)
+
+    # Diff the apk contents
+    diff(args, apk_path, args.work + "/packages/" + output)

pmb/build/package.py

@@ -21,6 +21,7 @@ import logging
 
 import pmb.build
 import pmb.build.autodetect
+import pmb.build.buildinfo
 import pmb.build.crosscompiler
 import pmb.chroot
 import pmb.chroot.apk
@@ -29,11 +30,12 @@ import pmb.parse
 import pmb.parse.arch
 
 
-def package(args, pkgname, carch, force=False, recurse=True):
+def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False):
     """
     Build a package with Alpine Linux' abuild.
 
     :param force: even build, if not necessary
+    :returns: output path relative to the packages folder
     """
     # Get aport, skip upstream only packages
     aport = pmb.build.find_aport(args, pkgname, False)
@@ -109,6 +111,14 @@ def package(args, pkgname, carch, force=False, recurse=True):
     if not os.path.exists(path):
         raise RuntimeError("Package not found after build: " + path)
 
+    # Create .buildinfo.json file
+    if buildinfo:
+        logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
+        pmb.build.buildinfo.write(args, output, carch_buildenv, suffix,
+                                  apkbuild)
+
     # Symlink noarch packages
     if "noarch" in apkbuild["arch"]:
         pmb.build.symlink_noarch_package(args, output)
+
+    return output

pmb/chroot/apk.py

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import logging
-import os
 import pmb.chroot
 import pmb.parse.apkindex
 
@@ -58,19 +57,43 @@ def install(args, packages, suffix="native", build=True):
     pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo,
                     suffix)
 
-# Update all packages installed in a chroot
-
 
 def update(args, suffix="native"):
+    """
+    Update all packages installed in a chroot
+    """
     pmb.chroot.init(args, suffix)
     pmb.chroot.root(args, ["apk", "update"], suffix)
 
-# Get all explicitly installed packages
+
+def package_split(package):
+    """
+    FIXME: move to pmb.parse
+    """
+    split = package.split("-")
+    pkgrel = split[-1][1:]
+    pkgver = split[-2]
+    version = "-" + pkgver + "-r" + pkgrel
+    pkgname = package[:-1 * len(version)]
+    return {"pkgname": pkgname,
+            "pkgrel": pkgrel,
+            "pkgver": pkgver,
+            "package": package}
 
 
 def installed(args, suffix="native"):
-    world = args.work + "/chroot_" + suffix + "/etc/apk/world"
-    if not os.path.exists(world):
-        return []
-    with open(world, encoding="utf-8") as handle:
-        return handle.read().splitlines()
+    """
+    Get all installed packages and their versions.
+    :returns: { "hello-world": {"package": "hello-world-1-r2", "pkgrel": "2",
+        "pkgver": "1", "pkgname": "hello-world"}, ...}
+    """
+    ret = {}
+    list = pmb.chroot.user(args, ["apk", "info", "-vv"], suffix,
+                           return_stdout=True)
+    for line in list.split("\n"):
+        if not line.rstrip():
+            continue
+        package = line.split(" - ")[0]
+        split = package_split(package)
+        ret[split["pkgname"]] = split
+    return ret

pmb/helpers/run.py.orig

@@ -0,0 +1,76 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import subprocess
+import logging
+
+
+def core(args, cmd, log_message, log, return_stdout, check=True):
+    logging.debug(log_message)
+    """
+    Run the command and write the output to the log.
+
+    :param check: raise an exception, when the command fails
+    """
+
+    try:
+        ret = None
+        if log:
+            if return_stdout:
+                ret = subprocess.check_output(cmd).decode("utf-8")
+                args.logfd.write(ret)
+            else:
+                subprocess.check_call(cmd, stdout=args.logfd,
+                                      stderr=args.logfd)
+            args.logfd.flush()
+        else:
+            logging.debug("*** output passed to pmbootstrap stdout, not" +
+                          " to this log ***")
+            subprocess.check_call(cmd)
+
+    except subprocess.CalledProcessError as exc:
+        if check:
+            if log:
+                logging.debug("^" * 70)
+                logging.info("NOTE: The failed command's output is above"
+                             " the ^^^ line in the logfile: " + args.log)
+            raise RuntimeError("Command failed: " + log_message) from exc
+        else:
+            pass
+    return ret
+
+
+def user(args, cmd, log=True, working_dir=None, return_stdout=False,
+         check=True):
+    """
+    :param working_dir: defaults to args.work
+    """
+    if not working_dir:
+        working_dir = args.work
+
+    # TODO: maintain and check against a whitelist
+    return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check)
+
+
+def root(args, cmd, log=True, working_dir=None, return_stdout=False,
+         check=True):
+    """
+    :param working_dir: defaults to args.work
+    """
+    cmd = ["sudo"] + cmd
+    return user(args, cmd, log, working_dir, return_stdout, check)

pmb/parse/apkbuild.py

@@ -37,7 +37,7 @@ def replace_variables(apkbuild):
         replaced.append(subpackage.replace("$pkgname", ret["pkgname"]))
     ret["subpackages"] = replaced
 
-    # makedepend: $makedepends_host, $makedepends_build, $_llvmver
+    # makedepends: $makedepends_host, $makedepends_build, $_llvmver
     replaced = []
     for makedepend in ret["makedepends"]:
         if makedepend.startswith("$"):

pmb/parse/apkindex.py

@@ -74,6 +74,17 @@ def read(args, package, path, must_exist=True):
                         if not ret or compare_version(current["version"],
                                                       ret["version"]) == 1:
                             ret = current
+                    if "provides" in current:
+                        for alias in current["provides"]:
+                            split = alias.split("=")
+                            if len(split) == 1:
+                                continue
+                            name = split[0]
+                            version = split[1]
+                            if name == package:
+                                if not ret or compare_version(current["version"],
+                                                              version) == 1:
+                                    ret = current
                     current = {}
                 if line.startswith("P:"):  # package
                     current["pkgname"] = line[2:-1]
@@ -85,8 +96,17 @@ def read(args, package, path, must_exist=True):
                         current["depends"] = depends.split(" ")
                     else:
                         current["depends"] = []
+                if line.startswith("p:"):  # provides
+                    provides = line[2:-1]
+                    current["provides"] = provides.split(" ")
     if not ret and must_exist:
         raise RuntimeError("Package " + package + " not found in " + path)
+
+    if ret:
+        for key in ["depends", "provides"]:
+            if key not in ret:
+                ret[key] = []
+
     return ret
 
 

pmb/parse/arguments.py

@@ -120,9 +120,15 @@ def arguments():
                            " specific architecture")
     build.add_argument("--arch")
     build.add_argument("--force", action="store_true")
+    build.add_argument("--buildinfo", action="store_true")
     for action in [checksum, build, menuconfig, parse_apkbuild, aportgen]:
         action.add_argument("package")
 
+    # Action: challenge
+    challenge = sub.add_parser("challenge",
+                               help="rebuild a package and diff its contents")
+    challenge.add_argument("apk")
+
     # Use defaults from the user's config file
     args = parser.parse_args()
     cfg = pmb.config.load(args)

pmbootstrap.py

@@ -55,9 +55,12 @@ def main():
         if args.action == "aportgen":
             pmb.aportgen.generate(args, args.package)
         elif args.action == "build":
-            pmb.build.package(args, args.package, args.arch, args.force, False)
+            pmb.build.package(args, args.package, args.arch, args.force, False,
+                              args.buildinfo)
         elif args.action == "build_init":
             pmb.build.init(args, args.suffix)
+        elif args.action == "challenge":
+            pmb.build.challenge(args, args.apk)
         elif args.action == "checksum":
             pmb.build.checksum(args, args.package)
         elif args.action == "chroot":