diff --git a/.gitignore b/.gitignore index 7bbc71c0..e9292a77 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ __pycache__/ # Distribution / packaging .Python env/ -build/ develop-eggs/ dist/ downloads/ diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py new file mode 100644 index 00000000..c89d01c7 --- /dev/null +++ b/pmb/build/__init__.py @@ -0,0 +1,25 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +# Exported functions +from pmb.build.init import init +from pmb.build.checksum import checksum +from pmb.build.other import copy_to_buildpath, is_necessary, \ + symlink_noarch_package, find_aport, ccache_stats, index_repo +from pmb.build.package import package +from pmb.build.menuconfig import menuconfig diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py new file mode 100644 index 00000000..96b6c437 --- /dev/null +++ b/pmb/build/autodetect.py @@ -0,0 +1,64 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import fnmatch +import pmb.config +import pmb.chroot.apk + + +def carch(args, apkbuild, carch): + if "noarch" in apkbuild["arch"]: + return args.arch_native + if carch: + return carch + if ("all" in apkbuild["arch"] or + args.arch_native in apkbuild["arch"]): + return args.arch_native + return apkbuild["arch"][0] + + +def suffix(args, apkbuild, carch): + if carch == args.arch_native: + return "native" + if "noarch" in apkbuild["arch"]: + return "native" + + pkgname = apkbuild["pkgname"] + if pkgname.endswith("-repack"): + return "native" + if args.cross and apkbuild["_pmb_build_in_native_chroot"] != "false": + for pattern in pmb.config.build_cross_native: + if fnmatch.fnmatch(pkgname, pattern): + return "native" + + return "buildroot_" + carch + + +def crosscompile(args, apkbuild, carch, suffix): + """ + :returns: None, "native" or "distcc" + """ + if not args.cross: + return None + if apkbuild["pkgname"].endswith("-repack"): + return None + if carch == args.arch_native: + return None + if suffix == "native": + return "native" + return "distcc" diff --git a/pmb/build/checksum.py b/pmb/build/checksum.py new file mode 100644 index 00000000..a7397cb8 --- /dev/null +++ b/pmb/build/checksum.py @@ -0,0 +1,36 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging + +import pmb.chroot +import pmb.build +import pmb.helpers.run + + +def checksum(args, pkgname): + pmb.build.init(args) + pmb.build.copy_to_buildpath(args, pkgname) + logging.info("(native) generate checksums for " + pkgname) + pmb.chroot.user(args, ["abuild", "checksum"], + working_dir="/home/user/build") + + # Copy modified APKBUILD back + source = args.work + "/chroot_native/home/user/build/APKBUILD" + target = args.aports + "/" + pkgname + "/" + pmb.helpers.run.user(args, ["cp", source, target]) diff --git a/pmb/build/crosscompiler.py b/pmb/build/crosscompiler.py new file mode 100644 index 00000000..2897c941 --- /dev/null +++ b/pmb/build/crosscompiler.py @@ -0,0 +1,32 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import pmb.config +import fnmatch + + +def init(args, arch): + packages = ["gcc-" + arch, "ccache-cross-symlinks"] + pmb.chroot.apk.install(args, packages) + + +def native_chroot(args, pkgname): + for pattern in pmb.config.crosscompile_supported: + if fnmatch.fnmatch(pkgname, pattern): + return True + return False diff --git a/pmb/build/init.py b/pmb/build/init.py new file mode 100644 index 00000000..96ff1281 --- /dev/null +++ b/pmb/build/init.py @@ -0,0 +1,78 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + +import pmb.config +import pmb.chroot +import pmb.chroot.apk +import pmb.helpers.run + + +def init(args, suffix="native"): + # Check if already initialized + marker = "/var/local/pmbootstrap_chroot_build_init_done" + if os.path.exists(args.work + "/chroot_" + suffix + marker): + return + + # Initialize chroot, install packages + pmb.chroot.apk.install(args, pmb.config.build_packages, suffix) + + # Fix permissions + pmb.chroot.root(args, ["chmod", "-R", "a+rw", + "/var/cache/distfiles"], suffix) + + # Generate package signing keys + chroot = args.work + "/chroot_" + suffix + if not os.path.exists(chroot + "/home/user/.abuild/abuild.conf"): + logging.info("(" + suffix + ") generate abuild keys") + pmb.chroot.user(args, ["abuild-keygen", "-n", "-q", "-a"], + suffix) + + # Copy package signing key to /etc/apk/keys + pmb.chroot.root(args, ["cp", "/home/user/.abuild/*.pub", + "/etc/apk/keys/"], suffix) + + # Add gzip wrapper, that converts '-9' to '-1' + if not os.path.exists(chroot + "/usr/local/bin/gzip"): + with open(chroot + "/tmp/gzip_wrapper.sh", "w") as handle: + content = """ + #!/bin/sh + # Simple wrapper, that converts -9 flag for gzip to -1 for speed + # improvement with abuild. FIXME: upstream to abuild with a flag! + args="" + for arg in "$@"; do + [ "$arg" == "-9" ] && arg="-1" + args="$args $arg" + done + /bin/gzip $args + """ + lines = content.split("\n")[1:] + for i in range(len(lines)): + lines[i] = lines[i][16:] + handle.write("\n".join(lines)) + pmb.chroot.root(args, ["cp", "/tmp/gzip_wrapper.sh", "/usr/local/bin/gzip"], + suffix) + pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix) + + # Add user to group abuild + pmb.chroot.root(args, ["adduser", "user", "abuild"], suffix) + + # Mark the chroot as initialized + pmb.chroot.root(args, ["touch", marker], suffix) diff --git a/pmb/build/menuconfig.py b/pmb/build/menuconfig.py new file mode 100644 index 00000000..b5e63d29 --- /dev/null +++ b/pmb/build/menuconfig.py @@ -0,0 +1,68 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + +import pmb.build +import pmb.build.autodetect +import pmb.build.checksum +import pmb.chroot +import pmb.chroot.apk +import pmb.helpers.run +import pmb.parse + + +def menuconfig(args, pkgname, arch): + # Read apkbuild + aport = pmb.build.find_aport(args, pkgname, False) + if not aport: + raise RuntimeError("Package " + pkgname + ": Could not find aport!") + apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD") + + # Set up build tools and makedepends + pmb.build.init(args) + depends = apkbuild["makedepends"] + ["ncurses-dev"] + pmb.chroot.apk.install(args, depends, build=False) + + # Patch and extract sources + pmb.build.copy_to_buildpath(args, pkgname) + logging.info("(native) extract kernel source") + pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/user/build") + logging.info("(native) apply patches") + pmb.chroot.user(args, ["abuild", "prepare"], "native", "/home/user/build", + log=False) + + # Run abuild menuconfig + cmd = [] + environment = {"CARCH": arch, "TERM": "xterm"} + for key, value in environment.items(): + cmd += [key + "=" + value] + cmd += ["abuild", "-d", "menuconfig"] + logging.info("(native) run menuconfig") + pmb.chroot.user(args, cmd, "native", "/home/user/build", log=False) + + # Update config + checksums + logging.info("copy kernel config back to aport-folder") + source = args.work + "/chroot_native/home/user/build/src/build/.config" + if not os.path.exists(source): + raise RuntimeError("No kernel config generated!") + target = (args.aports + "/" + pkgname + "/config-" + apkbuild["_flavor"] + + "." + arch) + pmb.helpers.run.user(args, ["cp", source, target]) + pmb.build.checksum(args, pkgname) diff --git a/pmb/build/other.py b/pmb/build/other.py new file mode 100644 index 00000000..e5a3c282 --- /dev/null +++ b/pmb/build/other.py @@ -0,0 +1,166 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging +import glob + +import pmb.chroot +import pmb.helpers.run +import pmb.parse.apkindex + + +def find_aport(args, package, must_exist=True): + """ + Find the aport, that provides a certain subpackage. + + :param must_exist: Raise an exception, when not found + :returns: the full path to the aport folder + """ + path = args.aports + "/" + package + if os.path.exists(path): + return path + + for path_current in glob.glob(args.aports + "/*/APKBUILD"): + apkbuild = pmb.parse.apkbuild(path_current) + if package in apkbuild["subpackages"]: + return os.path.dirname(path_current) + if must_exist: + raise RuntimeError("Could not find aport for package: " + + package) + return None + + +def copy_to_buildpath(args, package, suffix="native"): + # Sanity check + aport = args.aports + "/" + package + if not os.path.exists(aport + "/APKBUILD"): + raise ValueError("Path does not contain an APKBUILD file:" + + aport) + + # Clean up folder + build = args.work + "/chroot_" + suffix + "/home/user/build" + if os.path.exists(build): + pmb.chroot.root(args, ["rm", "-rf", "/home/user/build"], + suffix=suffix) + + # Copy aport contents + pmb.helpers.run.root(args, ["cp", "-r", aport + "/", build]) + pmb.chroot.root(args, ["chown", "-R", "user:user", + "/home/user/build"], suffix=suffix) + + +def is_necessary(args, suffix, carch, apkbuild): + """ + Check if the package has already been built (because abuild's check + only works, if it is the same architecture!) + + :param apkbuild: From pmb.parse.apkbuild() + :returns: Boolean + """ + + # Get new version from APKBUILD + package = apkbuild["pkgname"] + version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + + # Get old version from APKINDEX + version_old = None + index_data = pmb.parse.apkindex.read(args, package, + args.work + "/packages/" + carch + "/APKINDEX.tar.gz", False) + if index_data: + version_old = index_data["version"] + + if version_new == version_old: + return False + if pmb.parse.apkindex.compare_version(version_old, + version_new) == 1: + logging.warning("WARNING: Package " + package + "-" + version_old + + " in your binary repository is higher than the version defined" + + " in the APKBUILD. Consider cleaning your package cache" + + " (pmbootstrap zap -p) or removing that file and running" + + " 'pmbootstrap index'!") + return False + return True + +# When arch is not defined, reindex all repos + + +def index_repo(args, arch=None): + if arch: + paths = [args.work + "/packages/" + arch] + else: + paths = glob.glob(args.work + "/packages/*") + + for path in paths: + path_arch = os.path.basename(path) + path_repo_chroot = "/home/user/packages/user/" + path_arch + logging.info("(native) index " + path_arch + " repository") + commands = [ + ["apk", "index", "--output", "APKINDEX.tar.gz_", + "--rewrite-arch", path_arch, "*.apk"], + ["abuild-sign", "APKINDEX.tar.gz_"], + ["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"] + ] + for command in commands: + pmb.chroot.user(args, command, working_dir=path_repo_chroot) + + +def symlink_noarch_package(args, arch_apk): + """ + :param arch_apk: for example: x86_64/mypackage-1.2.3-r0.apk + """ + + # Create the arch folder + device_arch = args.deviceinfo["arch"] + device_repo = args.work + "/packages/" + device_arch + if not os.path.exists(device_repo): + pmb.chroot.user(args, ["mkdir", "-p", "/home/user/packages/user/" + + device_arch]) + + # Add symlink, rewrite index + device_repo_chroot = "/home/user/packages/user/" + device_arch + pmb.chroot.user(args, ["ln", "-sf", "../" + arch_apk, "."], + working_dir=device_repo_chroot) + index_repo(args, device_arch) + + +def ccache_stats(args, arch): + suffix = "native" + if args.arch: + suffix = "buildroot_" + arch + pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False) + + +# set the correct JOBS count in abuild.conf +def configure_abuild(args, suffix, verify=False): + path = args.work + "/chroot_" + suffix + "/etc/abuild.conf" + prefix = "export JOBS=" + with open(path, encoding="utf-8") as handle: + for line in handle: + if not line.startswith(prefix): + continue + if line != (prefix + args.jobs + "\n"): + if verify: + raise RuntimeError("Failed to configure abuild: " + path + + "\nTry to delete the file (or zap the chroot).") + pmb.chroot.root(args, ["sed", "-i", "-e", + "s/^" + prefix + ".*/" + prefix + args.jobs + "/", + "/etc/abuild.conf"], suffix) + configure_abuild(args, suffix, True) + return + raise RuntimeError("Could not find " + prefix + " line in " + path) diff --git a/pmb/build/package.py b/pmb/build/package.py new file mode 100644 index 00000000..6b8de703 --- /dev/null +++ b/pmb/build/package.py @@ -0,0 +1,114 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import logging + +import pmb.build +import pmb.build.autodetect +import pmb.build.crosscompiler +import pmb.chroot +import pmb.chroot.apk +import pmb.chroot.distccd +import pmb.parse +import pmb.parse.arch + + +def package(args, pkgname, carch, force=False, recurse=True): + """ + Build a package with Alpine Linux' abuild. + + :param force: even build, if not necessary + """ + # Get aport, skip upstream only packages + aport = pmb.build.find_aport(args, pkgname, False) + if not aport: + if pmb.parse.apkindex.read_any_index(args, pkgname, carch): + return + raise RuntimeError("Package " + pkgname + ": Could not find aport," + " and could not find this package in any APKINDEX!") + + # Autodetect the build environment + apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD") + pkgname = apkbuild["pkgname"] + carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch) + suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv) + cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv, + suffix) + + # Skip already built versions + if not force and not pmb.build.is_necessary(args, suffix, + carch_buildenv, apkbuild): + return + + # Build dependencies first + if recurse: + for depend in apkbuild["depends"]: + package(args, depend, carch) + + # Install build tools and makedepends + pmb.build.init(args, suffix) + if len(apkbuild["makedepends"]): + pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix) + if cross: + pmb.chroot.apk.install(args, ["gcc-" + carch_buildenv, + "ccache-cross-symlinks"]) + if cross == "distcc": + pmb.chroot.apk.install(args, ["distcc"], suffix=suffix) + pmb.chroot.distccd.start(args) + + # Configure abuild.conf + pmb.build.other.configure_abuild(args, suffix) + + # Generate output name, log build message + output = (carch_buildenv + "/" + apkbuild["pkgname"] + "-" + + apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + ".apk") + logging.info("(" + suffix + ") build " + output) + + # Sanity check + if cross == "native" and "!tracedeps" not in apkbuild["options"]: + logging.info("WARNING: Option !tracedeps is not set, but we're" + " cross-compiling in the native chroot. This will probably" + " fail!") + + # Run abuild with ignored dependencies + pmb.build.copy_to_buildpath(args, pkgname, suffix) + cmd = [] + env = {"CARCH": carch_buildenv} + if cross == "native": + hostspec = pmb.parse.arch.alpine_to_hostspec(carch_buildenv) + env["CROSS_COMPILE"] = hostspec + "-" + env["CC"] = hostspec + "-gcc" + if cross == "distcc": + env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path + env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd + for key, value in env.items(): + cmd += [key + "=" + value] + cmd += ["abuild", "-d"] + if force: + cmd += ["-f"] + pmb.chroot.user(args, cmd, suffix, "/home/user/build") + + # Verify output file + path = args.work + "/packages/" + output + if not os.path.exists(path): + raise RuntimeError("Package not found after build: " + path) + + # Symlink noarch packages + if "noarch" in apkbuild["arch"]: + pmb.build.symlink_noarch_package(args, output)