Add missing "build" folder (was gitignored before)

This commit is contained in:
Oliver Smith 2017-05-26 22:35:21 +02:00
parent ed9fc7f8b2
commit bf387f0ef8
No known key found for this signature in database
GPG Key ID: 5AE7F5513E0885CB
9 changed files with 583 additions and 1 deletions

1
.gitignore vendored
View File

@ -9,7 +9,6 @@ __pycache__/
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/

25
pmb/build/__init__.py Normal file
View File

@ -0,0 +1,25 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
# Exported functions
from pmb.build.init import init
from pmb.build.checksum import checksum
from pmb.build.other import copy_to_buildpath, is_necessary, \
symlink_noarch_package, find_aport, ccache_stats, index_repo
from pmb.build.package import package
from pmb.build.menuconfig import menuconfig

64
pmb/build/autodetect.py Normal file
View File

@ -0,0 +1,64 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import fnmatch
import pmb.config
import pmb.chroot.apk
def carch(args, apkbuild, carch):
if "noarch" in apkbuild["arch"]:
return args.arch_native
if carch:
return carch
if ("all" in apkbuild["arch"] or
args.arch_native in apkbuild["arch"]):
return args.arch_native
return apkbuild["arch"][0]
def suffix(args, apkbuild, carch):
if carch == args.arch_native:
return "native"
if "noarch" in apkbuild["arch"]:
return "native"
pkgname = apkbuild["pkgname"]
if pkgname.endswith("-repack"):
return "native"
if args.cross and apkbuild["_pmb_build_in_native_chroot"] != "false":
for pattern in pmb.config.build_cross_native:
if fnmatch.fnmatch(pkgname, pattern):
return "native"
return "buildroot_" + carch
def crosscompile(args, apkbuild, carch, suffix):
"""
:returns: None, "native" or "distcc"
"""
if not args.cross:
return None
if apkbuild["pkgname"].endswith("-repack"):
return None
if carch == args.arch_native:
return None
if suffix == "native":
return "native"
return "distcc"

36
pmb/build/checksum.py Normal file
View File

@ -0,0 +1,36 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import logging
import pmb.chroot
import pmb.build
import pmb.helpers.run
def checksum(args, pkgname):
pmb.build.init(args)
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) generate checksums for " + pkgname)
pmb.chroot.user(args, ["abuild", "checksum"],
working_dir="/home/user/build")
# Copy modified APKBUILD back
source = args.work + "/chroot_native/home/user/build/APKBUILD"
target = args.aports + "/" + pkgname + "/"
pmb.helpers.run.user(args, ["cp", source, target])

View File

@ -0,0 +1,32 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import pmb.config
import fnmatch
def init(args, arch):
packages = ["gcc-" + arch, "ccache-cross-symlinks"]
pmb.chroot.apk.install(args, packages)
def native_chroot(args, pkgname):
for pattern in pmb.config.crosscompile_supported:
if fnmatch.fnmatch(pkgname, pattern):
return True
return False

78
pmb/build/init.py Normal file
View File

@ -0,0 +1,78 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import pmb.config
import pmb.chroot
import pmb.chroot.apk
import pmb.helpers.run
def init(args, suffix="native"):
# Check if already initialized
marker = "/var/local/pmbootstrap_chroot_build_init_done"
if os.path.exists(args.work + "/chroot_" + suffix + marker):
return
# Initialize chroot, install packages
pmb.chroot.apk.install(args, pmb.config.build_packages, suffix)
# Fix permissions
pmb.chroot.root(args, ["chmod", "-R", "a+rw",
"/var/cache/distfiles"], suffix)
# Generate package signing keys
chroot = args.work + "/chroot_" + suffix
if not os.path.exists(chroot + "/home/user/.abuild/abuild.conf"):
logging.info("(" + suffix + ") generate abuild keys")
pmb.chroot.user(args, ["abuild-keygen", "-n", "-q", "-a"],
suffix)
# Copy package signing key to /etc/apk/keys
pmb.chroot.root(args, ["cp", "/home/user/.abuild/*.pub",
"/etc/apk/keys/"], suffix)
# Add gzip wrapper, that converts '-9' to '-1'
if not os.path.exists(chroot + "/usr/local/bin/gzip"):
with open(chroot + "/tmp/gzip_wrapper.sh", "w") as handle:
content = """
#!/bin/sh
# Simple wrapper, that converts -9 flag for gzip to -1 for speed
# improvement with abuild. FIXME: upstream to abuild with a flag!
args=""
for arg in "$@"; do
[ "$arg" == "-9" ] && arg="-1"
args="$args $arg"
done
/bin/gzip $args
"""
lines = content.split("\n")[1:]
for i in range(len(lines)):
lines[i] = lines[i][16:]
handle.write("\n".join(lines))
pmb.chroot.root(args, ["cp", "/tmp/gzip_wrapper.sh", "/usr/local/bin/gzip"],
suffix)
pmb.chroot.root(args, ["chmod", "+x", "/usr/local/bin/gzip"], suffix)
# Add user to group abuild
pmb.chroot.root(args, ["adduser", "user", "abuild"], suffix)
# Mark the chroot as initialized
pmb.chroot.root(args, ["touch", marker], suffix)

68
pmb/build/menuconfig.py Normal file
View File

@ -0,0 +1,68 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import pmb.build
import pmb.build.autodetect
import pmb.build.checksum
import pmb.chroot
import pmb.chroot.apk
import pmb.helpers.run
import pmb.parse
def menuconfig(args, pkgname, arch):
# Read apkbuild
aport = pmb.build.find_aport(args, pkgname, False)
if not aport:
raise RuntimeError("Package " + pkgname + ": Could not find aport!")
apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD")
# Set up build tools and makedepends
pmb.build.init(args)
depends = apkbuild["makedepends"] + ["ncurses-dev"]
pmb.chroot.apk.install(args, depends, build=False)
# Patch and extract sources
pmb.build.copy_to_buildpath(args, pkgname)
logging.info("(native) extract kernel source")
pmb.chroot.user(args, ["abuild", "unpack"], "native", "/home/user/build")
logging.info("(native) apply patches")
pmb.chroot.user(args, ["abuild", "prepare"], "native", "/home/user/build",
log=False)
# Run abuild menuconfig
cmd = []
environment = {"CARCH": arch, "TERM": "xterm"}
for key, value in environment.items():
cmd += [key + "=" + value]
cmd += ["abuild", "-d", "menuconfig"]
logging.info("(native) run menuconfig")
pmb.chroot.user(args, cmd, "native", "/home/user/build", log=False)
# Update config + checksums
logging.info("copy kernel config back to aport-folder")
source = args.work + "/chroot_native/home/user/build/src/build/.config"
if not os.path.exists(source):
raise RuntimeError("No kernel config generated!")
target = (args.aports + "/" + pkgname + "/config-" + apkbuild["_flavor"] +
"." + arch)
pmb.helpers.run.user(args, ["cp", source, target])
pmb.build.checksum(args, pkgname)

166
pmb/build/other.py Normal file
View File

@ -0,0 +1,166 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import glob
import pmb.chroot
import pmb.helpers.run
import pmb.parse.apkindex
def find_aport(args, package, must_exist=True):
"""
Find the aport, that provides a certain subpackage.
:param must_exist: Raise an exception, when not found
:returns: the full path to the aport folder
"""
path = args.aports + "/" + package
if os.path.exists(path):
return path
for path_current in glob.glob(args.aports + "/*/APKBUILD"):
apkbuild = pmb.parse.apkbuild(path_current)
if package in apkbuild["subpackages"]:
return os.path.dirname(path_current)
if must_exist:
raise RuntimeError("Could not find aport for package: " +
package)
return None
def copy_to_buildpath(args, package, suffix="native"):
# Sanity check
aport = args.aports + "/" + package
if not os.path.exists(aport + "/APKBUILD"):
raise ValueError("Path does not contain an APKBUILD file:" +
aport)
# Clean up folder
build = args.work + "/chroot_" + suffix + "/home/user/build"
if os.path.exists(build):
pmb.chroot.root(args, ["rm", "-rf", "/home/user/build"],
suffix=suffix)
# Copy aport contents
pmb.helpers.run.root(args, ["cp", "-r", aport + "/", build])
pmb.chroot.root(args, ["chown", "-R", "user:user",
"/home/user/build"], suffix=suffix)
def is_necessary(args, suffix, carch, apkbuild):
"""
Check if the package has already been built (because abuild's check
only works, if it is the same architecture!)
:param apkbuild: From pmb.parse.apkbuild()
:returns: Boolean
"""
# Get new version from APKBUILD
package = apkbuild["pkgname"]
version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
# Get old version from APKINDEX
version_old = None
index_data = pmb.parse.apkindex.read(args, package,
args.work + "/packages/" + carch + "/APKINDEX.tar.gz", False)
if index_data:
version_old = index_data["version"]
if version_new == version_old:
return False
if pmb.parse.apkindex.compare_version(version_old,
version_new) == 1:
logging.warning("WARNING: Package " + package + "-" + version_old +
" in your binary repository is higher than the version defined" +
" in the APKBUILD. Consider cleaning your package cache" +
" (pmbootstrap zap -p) or removing that file and running" +
" 'pmbootstrap index'!")
return False
return True
# When arch is not defined, reindex all repos
def index_repo(args, arch=None):
if arch:
paths = [args.work + "/packages/" + arch]
else:
paths = glob.glob(args.work + "/packages/*")
for path in paths:
path_arch = os.path.basename(path)
path_repo_chroot = "/home/user/packages/user/" + path_arch
logging.info("(native) index " + path_arch + " repository")
commands = [
["apk", "index", "--output", "APKINDEX.tar.gz_",
"--rewrite-arch", path_arch, "*.apk"],
["abuild-sign", "APKINDEX.tar.gz_"],
["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
]
for command in commands:
pmb.chroot.user(args, command, working_dir=path_repo_chroot)
def symlink_noarch_package(args, arch_apk):
"""
:param arch_apk: for example: x86_64/mypackage-1.2.3-r0.apk
"""
# Create the arch folder
device_arch = args.deviceinfo["arch"]
device_repo = args.work + "/packages/" + device_arch
if not os.path.exists(device_repo):
pmb.chroot.user(args, ["mkdir", "-p", "/home/user/packages/user/" +
device_arch])
# Add symlink, rewrite index
device_repo_chroot = "/home/user/packages/user/" + device_arch
pmb.chroot.user(args, ["ln", "-sf", "../" + arch_apk, "."],
working_dir=device_repo_chroot)
index_repo(args, device_arch)
def ccache_stats(args, arch):
suffix = "native"
if args.arch:
suffix = "buildroot_" + arch
pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False)
# set the correct JOBS count in abuild.conf
def configure_abuild(args, suffix, verify=False):
path = args.work + "/chroot_" + suffix + "/etc/abuild.conf"
prefix = "export JOBS="
with open(path, encoding="utf-8") as handle:
for line in handle:
if not line.startswith(prefix):
continue
if line != (prefix + args.jobs + "\n"):
if verify:
raise RuntimeError("Failed to configure abuild: " + path +
"\nTry to delete the file (or zap the chroot).")
pmb.chroot.root(args, ["sed", "-i", "-e",
"s/^" + prefix + ".*/" + prefix + args.jobs + "/",
"/etc/abuild.conf"], suffix)
configure_abuild(args, suffix, True)
return
raise RuntimeError("Could not find " + prefix + " line in " + path)

114
pmb/build/package.py Normal file
View File

@ -0,0 +1,114 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import pmb.build
import pmb.build.autodetect
import pmb.build.crosscompiler
import pmb.chroot
import pmb.chroot.apk
import pmb.chroot.distccd
import pmb.parse
import pmb.parse.arch
def package(args, pkgname, carch, force=False, recurse=True):
"""
Build a package with Alpine Linux' abuild.
:param force: even build, if not necessary
"""
# Get aport, skip upstream only packages
aport = pmb.build.find_aport(args, pkgname, False)
if not aport:
if pmb.parse.apkindex.read_any_index(args, pkgname, carch):
return
raise RuntimeError("Package " + pkgname + ": Could not find aport,"
" and could not find this package in any APKINDEX!")
# Autodetect the build environment
apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD")
pkgname = apkbuild["pkgname"]
carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch)
suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv)
cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv,
suffix)
# Skip already built versions
if not force and not pmb.build.is_necessary(args, suffix,
carch_buildenv, apkbuild):
return
# Build dependencies first
if recurse:
for depend in apkbuild["depends"]:
package(args, depend, carch)
# Install build tools and makedepends
pmb.build.init(args, suffix)
if len(apkbuild["makedepends"]):
pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix)
if cross:
pmb.chroot.apk.install(args, ["gcc-" + carch_buildenv,
"ccache-cross-symlinks"])
if cross == "distcc":
pmb.chroot.apk.install(args, ["distcc"], suffix=suffix)
pmb.chroot.distccd.start(args)
# Configure abuild.conf
pmb.build.other.configure_abuild(args, suffix)
# Generate output name, log build message
output = (carch_buildenv + "/" + apkbuild["pkgname"] + "-" +
apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + ".apk")
logging.info("(" + suffix + ") build " + output)
# Sanity check
if cross == "native" and "!tracedeps" not in apkbuild["options"]:
logging.info("WARNING: Option !tracedeps is not set, but we're"
" cross-compiling in the native chroot. This will probably"
" fail!")
# Run abuild with ignored dependencies
pmb.build.copy_to_buildpath(args, pkgname, suffix)
cmd = []
env = {"CARCH": carch_buildenv}
if cross == "native":
hostspec = pmb.parse.arch.alpine_to_hostspec(carch_buildenv)
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "distcc":
env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path
env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
for key, value in env.items():
cmd += [key + "=" + value]
cmd += ["abuild", "-d"]
if force:
cmd += ["-f"]
pmb.chroot.user(args, cmd, suffix, "/home/user/build")
# Verify output file
path = args.work + "/packages/" + output
if not os.path.exists(path):
raise RuntimeError("Package not found after build: " + path)
# Symlink noarch packages
if "noarch" in apkbuild["arch"]:
pmb.build.symlink_noarch_package(args, output)