From 63ac1f5f6c389db402d4265cd4a57cbe386ac36a Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Mon, 5 Jun 2017 03:58:45 +0200 Subject: [PATCH 1/2] WIP #64: "lazy reproducible builds" --- pmb/build/__init__.py | 1 + pmb/build/buildinfo.py | 100 ++++++++++++++++++++++++++++++++++++ pmb/build/challenge.py | 109 ++++++++++++++++++++++++++++++++++++++++ pmb/build/package.py | 12 ++++- pmb/chroot/apk.py | 41 +++++++++++---- pmb/helpers/run.py.orig | 76 ++++++++++++++++++++++++++++ pmb/parse/apkbuild.py | 2 +- pmb/parse/apkindex.py | 20 ++++++++ pmb/parse/arguments.py | 6 +++ pmbootstrap.py | 5 +- 10 files changed, 360 insertions(+), 12 deletions(-) create mode 100644 pmb/build/buildinfo.py create mode 100644 pmb/build/challenge.py create mode 100644 pmb/helpers/run.py.orig diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index c89d01c7..5b6baa8b 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -23,3 +23,4 @@ from pmb.build.other import copy_to_buildpath, is_necessary, \ symlink_noarch_package, find_aport, ccache_stats, index_repo from pmb.build.package import package from pmb.build.menuconfig import menuconfig +from pmb.build.challenge import challenge diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py new file mode 100644 index 00000000..1ab3ca74 --- /dev/null +++ b/pmb/build/buildinfo.py @@ -0,0 +1,100 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import json +import pmb.chroot +import pmb.chroot.apk +import pmb.parse.apkindex + + +def get_depends_recursively(args, pkgnames, arch=None): + """ + :param pkgnames: List of pkgnames, for which the dependencies shall be + retrieved. + """ + todo = list(pkgnames) + ret = [] + seen = [] + while len(todo): + pkgname = todo.pop(0) + index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch) + if not index_data: + raise RuntimeError("Could not find dependency " + pkgname + + " of packages " + str(pkgnames) + " in any APKINDEX") + pkgname = index_data["pkgname"] + if pkgname not in pkgnames and pkgname not in ret: + ret.append(pkgname) + for depend in index_data["depends"]: + if depend not in ret: + if depend.startswith("!"): + continue + for operator in [">", "="]: + if operator in depend: + depend = depend.split(operator)[0] + if depend not in seen: + todo.append(depend) + seen.append(depend) + return ret + + +def generate(args, apk_path, carch, suffix, apkbuild): + """ + :param apk_path: Path to the .apk file, relative to the packages cache. + :param carch: Architecture, that the package has been built for. + :apkbuild: Return from pmb.parse.apkbuild(). + """ + ret = {"pkgname": apkbuild["pkgname"], + "pkgver": apkbuild["pkgver"], + "pkgrel": apkbuild["pkgrel"], + "carch": carch, + "versions": []} + + # Add makedepends versions + installed = pmb.chroot.apk.installed(args, suffix) + relevant = (apkbuild["makedepends"] + + get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"])) + for pkgname in relevant: + if pkgname in installed: + ret["versions"].append(installed[pkgname]["package"]) + ret["versions"].sort() + return ret + + +def write(args, apk_path, carch, suffix, apkbuild): + """ + Write a .buildinfo.json file for a package, right after building it. + It stores all information required to rebuild the package, very similar + to how they do it in Debian (but as JSON file, so it's easier to parse in + Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles + + :param apk_path: Path to the .apk file, relative to the packages cache. + :param carch: Architecture, that the package has been built for. + :apkbuild: Return from pmb.parse.apkbuild(). + """ + # Write to temp + if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"): + pmb.chroot.root(args, ["rm", "/tmp/buildinfo"]) + buildinfo = generate(args, apk_path, carch, suffix, apkbuild) + with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle: + handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n") + + # Move to packages + pmb.chroot.root(args, ["chown", "user:user", "/tmp/buildinfo"]) + pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/user/packages/user/" + + apk_path + ".buildinfo.json"]) diff --git a/pmb/build/challenge.py b/pmb/build/challenge.py new file mode 100644 index 00000000..ed5bc5be --- /dev/null +++ b/pmb/build/challenge.py @@ -0,0 +1,109 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import json +import os +import tarfile +import tempfile +import filecmp +import shutil +import pmb.build +import pmb.parse.apkbuild + + +def diff(args, apk_a, apk_b): + logging.info("Challenge " + apk_a) + with tarfile.open(apk_a, "r:gz") as tar_a: + with tarfile.open(apk_b, "r:gz") as tar_b: + # List of files must be the same + list_a = sorted(tar_a.getnames()) + list_b = tar_b.getnames() + list_b.sort() + if list_a != list_b: + raise RuntimeError( + "Both APKs do not contain the same file names!") + + # Iterate through the list + for name in list_a: + logging.debug("Compare: " + name) + if name == ".PKGINFO" or name.startswith(".SIGN.RSA."): + logging.debug( + "=> Skipping, this is expected to be different") + continue + temp_files = [] + + # Extract + for tar in [tar_a, tar_b]: + member = tar.getmember(name) + if member.isdir(): + continue + handle, path = tempfile.mkstemp("pmbootstrap") + handle = open(handle, "wb") + shutil.copyfileobj(tar.extractfile(member), handle) + handle.close() + temp_files.append(path) + if not len(temp_files): + logging.debug("=> Skipping, this is a directory") + continue + + # Compare and delete + equal = filecmp.cmp( + temp_files[0], temp_files[1], shallow=False) + for temp_file in temp_files: + os.remove(temp_file) + if equal: + logging.debug("=> Equal") + else: + raise RuntimeError("File '" + name + "' is different!") + + +def challenge(args, apk_path): + # Parse buildinfo + buildinfo_path = apk_path + ".buildinfo.json" + if not os.path.exists(buildinfo_path): + logging.info("NOTE: To create a .buildinfo.json file, use the" + " --buildinfo command while building: 'pmbootstrap build" + " --buildinfo '") + raise RuntimeError("Missing file: " + buildinfo_path) + with open(buildinfo_path) as handle: + buildinfo = json.load(handle) + + # Parse and install all packages listed in versions + versions = {} + for package in buildinfo["versions"]: + split = pmb.chroot.apk.package_split(package) + pkgname = split["pkgname"] + versions[pkgname] = split + pmb.chroot.apk.install(args, versions.keys()) + + # Verify the installed versions + installed = pmb.chroot.apk.installed(args) + for pkgname, split in versions.items(): + package_installed = installed[pkgname]["package"] + package_buildinfo = split["package"] + if package_installed != package_buildinfo: + raise RuntimeError("Dependency " + pkgname + " version is different" + " (installed: " + package_installed + "," + " buildinfo: " + package_buildinfo + ")!") + # Build the package + output = pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"], + force=True) + + # Diff the apk contents + diff(args, apk_path, args.work + "/packages/" + output) diff --git a/pmb/build/package.py b/pmb/build/package.py index 6b8de703..7f600227 100644 --- a/pmb/build/package.py +++ b/pmb/build/package.py @@ -21,6 +21,7 @@ import logging import pmb.build import pmb.build.autodetect +import pmb.build.buildinfo import pmb.build.crosscompiler import pmb.chroot import pmb.chroot.apk @@ -29,11 +30,12 @@ import pmb.parse import pmb.parse.arch -def package(args, pkgname, carch, force=False, recurse=True): +def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False): """ Build a package with Alpine Linux' abuild. :param force: even build, if not necessary + :returns: output path relative to the packages folder """ # Get aport, skip upstream only packages aport = pmb.build.find_aport(args, pkgname, False) @@ -109,6 +111,14 @@ def package(args, pkgname, carch, force=False, recurse=True): if not os.path.exists(path): raise RuntimeError("Package not found after build: " + path) + # Create .buildinfo.json file + if buildinfo: + logging.info("(" + suffix + ") generate " + output + ".buildinfo.json") + pmb.build.buildinfo.write(args, output, carch_buildenv, suffix, + apkbuild) + # Symlink noarch packages if "noarch" in apkbuild["arch"]: pmb.build.symlink_noarch_package(args, output) + + return output diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index 879a3e15..f4b19429 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ import logging -import os import pmb.chroot import pmb.parse.apkindex @@ -58,19 +57,43 @@ def install(args, packages, suffix="native", build=True): pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo, suffix) -# Update all packages installed in a chroot - def update(args, suffix="native"): + """ + Update all packages installed in a chroot + """ pmb.chroot.init(args, suffix) pmb.chroot.root(args, ["apk", "update"], suffix) -# Get all explicitly installed packages + +def package_split(package): + """ + FIXME: move to pmb.parse + """ + split = package.split("-") + pkgrel = split[-1][1:] + pkgver = split[-2] + version = "-" + pkgver + "-r" + pkgrel + pkgname = package[:-1 * len(version)] + return {"pkgname": pkgname, + "pkgrel": pkgrel, + "pkgver": pkgver, + "package": package} def installed(args, suffix="native"): - world = args.work + "/chroot_" + suffix + "/etc/apk/world" - if not os.path.exists(world): - return [] - with open(world, encoding="utf-8") as handle: - return handle.read().splitlines() + """ + Get all installed packages and their versions. + :returns: { "hello-world": {"package": "hello-world-1-r2", "pkgrel": "2", + "pkgver": "1", "pkgname": "hello-world"}, ...} + """ + ret = {} + list = pmb.chroot.user(args, ["apk", "info", "-vv"], suffix, + return_stdout=True) + for line in list.split("\n"): + if not line.rstrip(): + continue + package = line.split(" - ")[0] + split = package_split(package) + ret[split["pkgname"]] = split + return ret diff --git a/pmb/helpers/run.py.orig b/pmb/helpers/run.py.orig new file mode 100644 index 00000000..e9b419a9 --- /dev/null +++ b/pmb/helpers/run.py.orig @@ -0,0 +1,76 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import subprocess +import logging + + +def core(args, cmd, log_message, log, return_stdout, check=True): + logging.debug(log_message) + """ + Run the command and write the output to the log. + + :param check: raise an exception, when the command fails + """ + + try: + ret = None + if log: + if return_stdout: + ret = subprocess.check_output(cmd).decode("utf-8") + args.logfd.write(ret) + else: + subprocess.check_call(cmd, stdout=args.logfd, + stderr=args.logfd) + args.logfd.flush() + else: + logging.debug("*** output passed to pmbootstrap stdout, not" + + " to this log ***") + subprocess.check_call(cmd) + + except subprocess.CalledProcessError as exc: + if check: + if log: + logging.debug("^" * 70) + logging.info("NOTE: The failed command's output is above" + " the ^^^ line in the logfile: " + args.log) + raise RuntimeError("Command failed: " + log_message) from exc + else: + pass + return ret + + +def user(args, cmd, log=True, working_dir=None, return_stdout=False, + check=True): + """ + :param working_dir: defaults to args.work + """ + if not working_dir: + working_dir = args.work + + # TODO: maintain and check against a whitelist + return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check) + + +def root(args, cmd, log=True, working_dir=None, return_stdout=False, + check=True): + """ + :param working_dir: defaults to args.work + """ + cmd = ["sudo"] + cmd + return user(args, cmd, log, working_dir, return_stdout, check) diff --git a/pmb/parse/apkbuild.py b/pmb/parse/apkbuild.py index 2e0488e8..9af6fe89 100644 --- a/pmb/parse/apkbuild.py +++ b/pmb/parse/apkbuild.py @@ -37,7 +37,7 @@ def replace_variables(apkbuild): replaced.append(subpackage.replace("$pkgname", ret["pkgname"])) ret["subpackages"] = replaced - # makedepend: $makedepends_host, $makedepends_build, $_llvmver + # makedepends: $makedepends_host, $makedepends_build, $_llvmver replaced = [] for makedepend in ret["makedepends"]: if makedepend.startswith("$"): diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index ead1a85d..6281d3a0 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -74,6 +74,17 @@ def read(args, package, path, must_exist=True): if not ret or compare_version(current["version"], ret["version"]) == 1: ret = current + if "provides" in current: + for alias in current["provides"]: + split = alias.split("=") + if len(split) == 1: + continue + name = split[0] + version = split[1] + if name == package: + if not ret or compare_version(current["version"], + version) == 1: + ret = current current = {} if line.startswith("P:"): # package current["pkgname"] = line[2:-1] @@ -85,8 +96,17 @@ def read(args, package, path, must_exist=True): current["depends"] = depends.split(" ") else: current["depends"] = [] + if line.startswith("p:"): # provides + provides = line[2:-1] + current["provides"] = provides.split(" ") if not ret and must_exist: raise RuntimeError("Package " + package + " not found in " + path) + + if ret: + for key in ["depends", "provides"]: + if key not in ret: + ret[key] = [] + return ret diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index f6c98ad5..37638550 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -120,9 +120,15 @@ def arguments(): " specific architecture") build.add_argument("--arch") build.add_argument("--force", action="store_true") + build.add_argument("--buildinfo", action="store_true") for action in [checksum, build, menuconfig, parse_apkbuild, aportgen]: action.add_argument("package") + # Action: challenge + challenge = sub.add_parser("challenge", + help="rebuild a package and diff its contents") + challenge.add_argument("apk") + # Use defaults from the user's config file args = parser.parse_args() cfg = pmb.config.load(args) diff --git a/pmbootstrap.py b/pmbootstrap.py index 1d1c7e48..bd140ae5 100755 --- a/pmbootstrap.py +++ b/pmbootstrap.py @@ -55,9 +55,12 @@ def main(): if args.action == "aportgen": pmb.aportgen.generate(args, args.package) elif args.action == "build": - pmb.build.package(args, args.package, args.arch, args.force, False) + pmb.build.package(args, args.package, args.arch, args.force, False, + args.buildinfo) elif args.action == "build_init": pmb.build.init(args, args.suffix) + elif args.action == "challenge": + pmb.build.challenge(args, args.apk) elif args.action == "checksum": pmb.build.checksum(args, args.package) elif args.action == "chroot": From 31b276eeb90e104a7fa2c2763d64ac76e17e0a85 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Tue, 6 Jun 2017 22:21:59 +0200 Subject: [PATCH 2/2] WIP #64: make gcc-armhf lazy-reproducible, properly compare symlinks --- aports/binutils-armhf/APKBUILD | 1 + pmb/aportgen/binutils.py | 1 + pmb/build/buildinfo.py | 8 +++- pmb/build/challenge.py | 82 ++++++++++++++++++++++------------ pmb/helpers/http.py | 2 +- 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/aports/binutils-armhf/APKBUILD b/aports/binutils-armhf/APKBUILD index 0b445795..4b703e74 100644 --- a/aports/binutils-armhf/APKBUILD +++ b/aports/binutils-armhf/APKBUILD @@ -44,6 +44,7 @@ build() { --enable-ld=default \ --enable-gold=yes \ --enable-plugins \ + --enable-deterministic-archives \ --disable-multilib \ --disable-werror \ --disable-nls \ diff --git a/pmb/aportgen/binutils.py b/pmb/aportgen/binutils.py index 55db5c04..c591a94e 100644 --- a/pmb/aportgen/binutils.py +++ b/pmb/aportgen/binutils.py @@ -50,6 +50,7 @@ def generate(args, pkgname): --enable-ld=default \\ --enable-gold=yes \\ --enable-plugins \\ + --enable-deterministic-archives \\ --disable-multilib \\ --disable-werror \\ --disable-nls \\ diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py index 1ab3ca74..1ea5853c 100644 --- a/pmb/build/buildinfo.py +++ b/pmb/build/buildinfo.py @@ -18,6 +18,7 @@ along with pmbootstrap. If not, see . """ import os import json +import logging import pmb.chroot import pmb.chroot.apk import pmb.parse.apkindex @@ -35,8 +36,11 @@ def get_depends_recursively(args, pkgnames, arch=None): pkgname = todo.pop(0) index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch) if not index_data: - raise RuntimeError("Could not find dependency " + pkgname + - " of packages " + str(pkgnames) + " in any APKINDEX") + logging.debug( + "NOTE: Could not find dependency " + + pkgname + + " in any APKINDEX.") + continue pkgname = index_data["pkgname"] if pkgname not in pkgnames and pkgname not in ret: ret.append(pkgname) diff --git a/pmb/build/challenge.py b/pmb/build/challenge.py index ed5bc5be..a6c64150 100644 --- a/pmb/build/challenge.py +++ b/pmb/build/challenge.py @@ -27,6 +27,28 @@ import pmb.build import pmb.parse.apkbuild +def diff_files(tar_a, tar_b, member_a, member_b, name): + # Extract both files + tars = [tar_a, tar_b] + members = [member_a, member_b] + temp_files = [] + for i in range(2): + handle, path = tempfile.mkstemp("pmbootstrap") + handle = open(handle, "wb") + shutil.copyfileobj(tars[i].extractfile(members[i]), handle) + handle.close() + temp_files.append(path) + + # Compare and delete + equal = filecmp.cmp(temp_files[0], temp_files[1], shallow=False) + for temp_file in temp_files: + os.remove(temp_file) + if equal: + logging.debug("=> File has the same content") + else: + raise RuntimeError("File '" + name + "' is different!") + + def diff(args, apk_a, apk_b): logging.info("Challenge " + apk_a) with tarfile.open(apk_a, "r:gz") as tar_a: @@ -40,37 +62,41 @@ def diff(args, apk_a, apk_b): "Both APKs do not contain the same file names!") # Iterate through the list + success = True for name in list_a: - logging.debug("Compare: " + name) - if name == ".PKGINFO" or name.startswith(".SIGN.RSA."): - logging.debug( - "=> Skipping, this is expected to be different") - continue - temp_files = [] - - # Extract - for tar in [tar_a, tar_b]: - member = tar.getmember(name) - if member.isdir(): + try: + logging.debug("Compare: " + name) + if name == ".PKGINFO" or name.startswith(".SIGN.RSA."): + logging.debug( + "=> Skipping, this is expected to be different") continue - handle, path = tempfile.mkstemp("pmbootstrap") - handle = open(handle, "wb") - shutil.copyfileobj(tar.extractfile(member), handle) - handle.close() - temp_files.append(path) - if not len(temp_files): - logging.debug("=> Skipping, this is a directory") - continue - # Compare and delete - equal = filecmp.cmp( - temp_files[0], temp_files[1], shallow=False) - for temp_file in temp_files: - os.remove(temp_file) - if equal: - logging.debug("=> Equal") - else: - raise RuntimeError("File '" + name + "' is different!") + # Get members + member_a = tar_a.getmember(name) + member_b = tar_b.getmember(name) + if member_a.type != member_b.type: + raise RuntimeError( + "Entry '" + name + "' has a different type!") + + if member_a.isdir(): + logging.debug("=> Skipping, this is directory") + elif member_a.isfile(): + diff_files(tar_a, tar_b, member_a, member_b, name) + elif member_a.issym() or member_a.islnk(): + if member_a.linkname == member_b.linkname: + logging.debug( + "=> Both link to " + member_a.linkname) + else: + raise RuntimeError( + "Link " + name + " has a different target!") + else: + raise RuntimeError( + "Can't diff '" + name + "', unsupported type!") + except Exception as e: + logging.info("CHALLENGE FAILED for " + name + ":" + str(e)) + success = False + if not success: + raise RuntimeError("Challenge failed (see errors above)") def challenge(args, apk_path): diff --git a/pmb/helpers/http.py b/pmb/helpers/http.py index 2abf0a21..c628a8c7 100644 --- a/pmb/helpers/http.py +++ b/pmb/helpers/http.py @@ -35,7 +35,7 @@ def download(args, url, prefix, cache=True): # Check if file exists in cache prefix = prefix.replace("/", "_") path = (args.work + "/cache_http/" + prefix + "_" + - hashlib.sha512(url.encode("utf-8")).hexdigest()) + hashlib.sha256(url.encode("utf-8")).hexdigest()) if os.path.exists(path): if cache: return path