Merge branch 'lazy-reproducible-builds'
We have "lazy reproducible builds" now. What I mean by that is, that the resulting "apk" archive is not fully reproducible, but all binaries inside it are. This is necessary to kick-off the binary repo, which is in turn required to get the testsuite going on Travis. Read #64 for more information. Usage: ``` pmbootstrap build hello-world --buildinfo pmbootstrap challenge /tmp/path/to/hello-world-1-r2.apk ``` The "--buildinfo" parameter generates a "buildinfo.json", which contains the versions of all dependencies. It is not very optimizied, so this is a performance bottleneck and takes 10 seconds (which is quite much considering that the hello-world package builds in less than a second). This can be improved in the future, and then the buildinfo parameter may become the default.
This commit is contained in:
Oliver Smith
commit
3a3dd8063f
|
|
@ -44,6 +44,7 @@ build() {
|
|||
--enable-ld=default \
|
||||
--enable-gold=yes \
|
||||
--enable-plugins \
|
||||
--enable-deterministic-archives \
|
||||
--disable-multilib \
|
||||
--disable-werror \
|
||||
--disable-nls \
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ def generate(args, pkgname):
|
|||
--enable-ld=default \\
|
||||
--enable-gold=yes \\
|
||||
--enable-plugins \\
|
||||
--enable-deterministic-archives \\
|
||||
--disable-multilib \\
|
||||
--disable-werror \\
|
||||
--disable-nls \\
|
||||
|
|
|
|||
|
|
@ -23,3 +23,4 @@ from pmb.build.other import copy_to_buildpath, is_necessary, \
|
|||
symlink_noarch_package, find_aport, ccache_stats, index_repo
|
||||
from pmb.build.package import package
|
||||
from pmb.build.menuconfig import menuconfig
|
||||
from pmb.build.challenge import challenge
|
||||
|
|
|
|||
|
|
@ -0,0 +1,104 @@
|
|||
"""
|
||||
Copyright 2017 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
import pmb.chroot
|
||||
import pmb.chroot.apk
|
||||
import pmb.parse.apkindex
|
||||
|
||||
|
||||
def get_depends_recursively(args, pkgnames, arch=None):
|
||||
"""
|
||||
:param pkgnames: List of pkgnames, for which the dependencies shall be
|
||||
retrieved.
|
||||
"""
|
||||
todo = list(pkgnames)
|
||||
ret = []
|
||||
seen = []
|
||||
while len(todo):
|
||||
pkgname = todo.pop(0)
|
||||
index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch)
|
||||
if not index_data:
|
||||
logging.debug(
|
||||
"NOTE: Could not find dependency " +
|
||||
pkgname +
|
||||
" in any APKINDEX.")
|
||||
continue
|
||||
pkgname = index_data["pkgname"]
|
||||
if pkgname not in pkgnames and pkgname not in ret:
|
||||
ret.append(pkgname)
|
||||
for depend in index_data["depends"]:
|
||||
if depend not in ret:
|
||||
if depend.startswith("!"):
|
||||
continue
|
||||
for operator in [">", "="]:
|
||||
if operator in depend:
|
||||
depend = depend.split(operator)[0]
|
||||
if depend not in seen:
|
||||
todo.append(depend)
|
||||
seen.append(depend)
|
||||
return ret
|
||||
|
||||
|
||||
def generate(args, apk_path, carch, suffix, apkbuild):
|
||||
"""
|
||||
:param apk_path: Path to the .apk file, relative to the packages cache.
|
||||
:param carch: Architecture, that the package has been built for.
|
||||
:apkbuild: Return from pmb.parse.apkbuild().
|
||||
"""
|
||||
ret = {"pkgname": apkbuild["pkgname"],
|
||||
"pkgver": apkbuild["pkgver"],
|
||||
"pkgrel": apkbuild["pkgrel"],
|
||||
"carch": carch,
|
||||
"versions": []}
|
||||
|
||||
# Add makedepends versions
|
||||
installed = pmb.chroot.apk.installed(args, suffix)
|
||||
relevant = (apkbuild["makedepends"] +
|
||||
get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"]))
|
||||
for pkgname in relevant:
|
||||
if pkgname in installed:
|
||||
ret["versions"].append(installed[pkgname]["package"])
|
||||
ret["versions"].sort()
|
||||
return ret
|
||||
|
||||
|
||||
def write(args, apk_path, carch, suffix, apkbuild):
|
||||
"""
|
||||
Write a .buildinfo.json file for a package, right after building it.
|
||||
It stores all information required to rebuild the package, very similar
|
||||
to how they do it in Debian (but as JSON file, so it's easier to parse in
|
||||
Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles
|
||||
|
||||
:param apk_path: Path to the .apk file, relative to the packages cache.
|
||||
:param carch: Architecture, that the package has been built for.
|
||||
:apkbuild: Return from pmb.parse.apkbuild().
|
||||
"""
|
||||
# Write to temp
|
||||
if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"):
|
||||
pmb.chroot.root(args, ["rm", "/tmp/buildinfo"])
|
||||
buildinfo = generate(args, apk_path, carch, suffix, apkbuild)
|
||||
with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle:
|
||||
handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n")
|
||||
|
||||
# Move to packages
|
||||
pmb.chroot.root(args, ["chown", "user:user", "/tmp/buildinfo"])
|
||||
pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/user/packages/user/" +
|
||||
apk_path + ".buildinfo.json"])
|
||||
|
|
@ -0,0 +1,134 @@
|
|||
"""
|
||||
Copyright 2017 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
import filecmp
|
||||
import shutil
|
||||
import pmb.build
|
||||
import pmb.parse.apkbuild
|
||||
|
||||
|
||||
def diff_files(tar_a, tar_b, member_a, member_b, name):
|
||||
# Extract both files
|
||||
tars = [tar_a, tar_b]
|
||||
members = [member_a, member_b]
|
||||
temp_files = []
|
||||
for i in range(2):
|
||||
handle, path = tempfile.mkstemp("pmbootstrap")
|
||||
handle = open(handle, "wb")
|
||||
shutil.copyfileobj(tars[i].extractfile(members[i]), handle)
|
||||
handle.close()
|
||||
temp_files.append(path)
|
||||
|
||||
# Compare and delete
|
||||
equal = filecmp.cmp(temp_files[0], temp_files[1], shallow=False)
|
||||
for temp_file in temp_files:
|
||||
os.remove(temp_file)
|
||||
if equal:
|
||||
logging.debug("=> File has the same content")
|
||||
else:
|
||||
raise RuntimeError("File '" + name + "' is different!")
|
||||
|
||||
|
||||
def diff(args, apk_a, apk_b):
|
||||
logging.info("Challenge " + apk_a)
|
||||
with tarfile.open(apk_a, "r:gz") as tar_a:
|
||||
with tarfile.open(apk_b, "r:gz") as tar_b:
|
||||
# List of files must be the same
|
||||
list_a = sorted(tar_a.getnames())
|
||||
list_b = sorted(tar_b.getnames())
|
||||
if list_a != list_b:
|
||||
raise RuntimeError(
|
||||
"Both APKs do not contain the same file names!")
|
||||
|
||||
# Iterate through the list
|
||||
success = True
|
||||
for name in list_a:
|
||||
try:
|
||||
logging.debug("Compare: " + name)
|
||||
if name == ".PKGINFO" or name.startswith(".SIGN.RSA."):
|
||||
logging.debug(
|
||||
"=> Skipping, this is expected to be different")
|
||||
continue
|
||||
|
||||
# Get members
|
||||
member_a = tar_a.getmember(name)
|
||||
member_b = tar_b.getmember(name)
|
||||
if member_a.type != member_b.type:
|
||||
raise RuntimeError(
|
||||
"Entry '" + name + "' has a different type!")
|
||||
|
||||
if member_a.isdir():
|
||||
logging.debug("=> Skipping, this is directory")
|
||||
elif member_a.isfile():
|
||||
diff_files(tar_a, tar_b, member_a, member_b, name)
|
||||
elif member_a.issym() or member_a.islnk():
|
||||
if member_a.linkname == member_b.linkname:
|
||||
logging.debug(
|
||||
"=> Both link to " + member_a.linkname)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Link " + name + " has a different target!")
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"Can't diff '" + name + "', unsupported type!")
|
||||
except Exception as e:
|
||||
logging.info("CHALLENGE FAILED for " + name + ":" + str(e))
|
||||
success = False
|
||||
if not success:
|
||||
raise RuntimeError("Challenge failed (see errors above)")
|
||||
|
||||
|
||||
def challenge(args, apk_path):
|
||||
# Parse buildinfo
|
||||
buildinfo_path = apk_path + ".buildinfo.json"
|
||||
if not os.path.exists(buildinfo_path):
|
||||
logging.info("NOTE: To create a .buildinfo.json file, use the"
|
||||
" --buildinfo command while building: 'pmbootstrap build"
|
||||
" --buildinfo <pkgname>'")
|
||||
raise RuntimeError("Missing file: " + buildinfo_path)
|
||||
with open(buildinfo_path) as handle:
|
||||
buildinfo = json.load(handle)
|
||||
|
||||
# Parse and install all packages listed in versions
|
||||
versions = {}
|
||||
for package in buildinfo["versions"]:
|
||||
split = pmb.chroot.apk.package_split(package)
|
||||
pkgname = split["pkgname"]
|
||||
versions[pkgname] = split
|
||||
pmb.chroot.apk.install(args, versions.keys())
|
||||
|
||||
# Verify the installed versions
|
||||
installed = pmb.chroot.apk.installed(args)
|
||||
for pkgname, split in versions.items():
|
||||
package_installed = installed[pkgname]["package"]
|
||||
package_buildinfo = split["package"]
|
||||
if package_installed != package_buildinfo:
|
||||
raise RuntimeError("Dependency " + pkgname + " version is different"
|
||||
" (installed: " + package_installed + ","
|
||||
" buildinfo: " + package_buildinfo + ")!")
|
||||
# Build the package
|
||||
output = pmb.build.package(args, buildinfo["pkgname"], buildinfo["carch"],
|
||||
force=True)
|
||||
|
||||
# Diff the apk contents
|
||||
diff(args, apk_path, args.work + "/packages/" + output)
|
||||
|
|
@ -21,6 +21,7 @@ import logging
|
|||
|
||||
import pmb.build
|
||||
import pmb.build.autodetect
|
||||
import pmb.build.buildinfo
|
||||
import pmb.build.crosscompiler
|
||||
import pmb.chroot
|
||||
import pmb.chroot.apk
|
||||
|
|
@ -29,11 +30,12 @@ import pmb.parse
|
|||
import pmb.parse.arch
|
||||
|
||||
|
||||
def package(args, pkgname, carch, force=False, recurse=True):
|
||||
def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False):
|
||||
"""
|
||||
Build a package with Alpine Linux' abuild.
|
||||
|
||||
:param force: even build, if not necessary
|
||||
:returns: output path relative to the packages folder
|
||||
"""
|
||||
# Get aport, skip upstream only packages
|
||||
aport = pmb.build.find_aport(args, pkgname, False)
|
||||
|
|
@ -109,6 +111,14 @@ def package(args, pkgname, carch, force=False, recurse=True):
|
|||
if not os.path.exists(path):
|
||||
raise RuntimeError("Package not found after build: " + path)
|
||||
|
||||
# Create .buildinfo.json file
|
||||
if buildinfo:
|
||||
logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
|
||||
pmb.build.buildinfo.write(args, output, carch_buildenv, suffix,
|
||||
apkbuild)
|
||||
|
||||
# Symlink noarch packages
|
||||
if "noarch" in apkbuild["arch"]:
|
||||
pmb.build.symlink_noarch_package(args, output)
|
||||
|
||||
return output
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ def replace_variables(apkbuild):
|
|||
replaced.append(subpackage.replace("$pkgname", ret["pkgname"]))
|
||||
ret["subpackages"] = replaced
|
||||
|
||||
# makedepend: $makedepends_host, $makedepends_build, $_llvmver
|
||||
# makedepends: $makedepends_host, $makedepends_build, $_llvmver
|
||||
replaced = []
|
||||
for makedepend in ret["makedepends"]:
|
||||
if makedepend.startswith("$"):
|
||||
|
|
|
|||
|
|
@ -74,6 +74,17 @@ def read(args, package, path, must_exist=True):
|
|||
if not ret or compare_version(current["version"],
|
||||
ret["version"]) == 1:
|
||||
ret = current
|
||||
if "provides" in current:
|
||||
for alias in current["provides"]:
|
||||
split = alias.split("=")
|
||||
if len(split) == 1:
|
||||
continue
|
||||
name = split[0]
|
||||
version = split[1]
|
||||
if name == package:
|
||||
if not ret or compare_version(current["version"],
|
||||
version) == 1:
|
||||
ret = current
|
||||
current = {}
|
||||
if line.startswith("P:"): # package
|
||||
current["pkgname"] = line[2:-1]
|
||||
|
|
@ -85,8 +96,17 @@ def read(args, package, path, must_exist=True):
|
|||
current["depends"] = depends.split(" ")
|
||||
else:
|
||||
current["depends"] = []
|
||||
if line.startswith("p:"): # provides
|
||||
provides = line[2:-1]
|
||||
current["provides"] = provides.split(" ")
|
||||
if not ret and must_exist:
|
||||
raise RuntimeError("Package " + package + " not found in " + path)
|
||||
|
||||
if ret:
|
||||
for key in ["depends", "provides"]:
|
||||
if key not in ret:
|
||||
ret[key] = []
|
||||
|
||||
return ret
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -61,7 +61,9 @@ def arguments_initfs(subparser):
|
|||
# ls, build, extract
|
||||
ls = sub.add_parser("ls", help="list initramfs contents")
|
||||
build = sub.add_parser("build", help="(re)build the initramfs")
|
||||
extract = sub.add_parser("extract", help="extract the initramfs to a temporary folder")
|
||||
extract = sub.add_parser(
|
||||
"extract",
|
||||
help="extract the initramfs to a temporary folder")
|
||||
for action in [ls, build, extract]:
|
||||
action.add_argument(
|
||||
"--flavor",
|
||||
|
|
@ -164,9 +166,15 @@ def arguments():
|
|||
" specific architecture")
|
||||
build.add_argument("--arch")
|
||||
build.add_argument("--force", action="store_true")
|
||||
build.add_argument("--buildinfo", action="store_true")
|
||||
for action in [checksum, build, menuconfig, parse_apkbuild, aportgen]:
|
||||
action.add_argument("package")
|
||||
|
||||
# Action: challenge
|
||||
challenge = sub.add_parser("challenge",
|
||||
help="rebuild a package and diff its contents")
|
||||
challenge.add_argument("apk")
|
||||
|
||||
# Use defaults from the user's config file
|
||||
args = parser.parse_args()
|
||||
cfg = pmb.config.load(args)
|
||||
|
|
|
|||
|
|
@ -56,9 +56,12 @@ def main():
|
|||
if args.action == "aportgen":
|
||||
pmb.aportgen.generate(args, args.package)
|
||||
elif args.action == "build":
|
||||
pmb.build.package(args, args.package, args.arch, args.force, False)
|
||||
pmb.build.package(args, args.package, args.arch, args.force, False,
|
||||
args.buildinfo)
|
||||
elif args.action == "build_init":
|
||||
pmb.build.init(args, args.suffix)
|
||||
elif args.action == "challenge":
|
||||
pmb.build.challenge(args, args.apk)
|
||||
elif args.action == "checksum":
|
||||
pmb.build.checksum(args, args.package)
|
||||
elif args.action == "chroot":
|
||||
|
|
|
|||
Loading…
Reference in New Issue