pmbootstrap build --src: override source for any package (#1210)
* New "pmbootstrap build --src=/local/source/path hello-world" syntax * The local source path gets mounted inside the chroot * From there, a copy of the source code gets created with rsync (so we can write into the source folder if necessary, for better compatibility with all kinds of APKBUILDs) * After the aport gets copied into the chroot before building (as usually), we extend the APKBUILD with overrides to make it use mountpoint's source instead of downloading the package's source from the web as usually * The package built with the local source gets _pYYYYMMDDHHMMSS appended to the pkgver * linux-postmarketos-mainline: use $builddir, fix patch checksum
This commit is contained in:
Oliver Smith
GitHub
parent
42ed5dcb0a
commit
0f371e426f
|
|
@ -9,7 +9,7 @@ _kernver=${pkgver%_rc*}
|
|||
_mainver=${_kernver%.*}
|
||||
_patchlevel=${_kernver/$_mainver./}
|
||||
_basever=${_mainver}.$((_patchlevel-1))
|
||||
pkgrel=3
|
||||
pkgrel=4
|
||||
|
||||
arch="x86_64 armhf aarch64"
|
||||
pkgdesc="Linux for pmOS supported chipsets (mainline, more bleeding-edge than stable)"
|
||||
|
|
@ -43,11 +43,11 @@ esac
|
|||
HOSTCC="${CC:-gcc}"
|
||||
HOSTCC="${HOSTCC#${CROSS_COMPILE}}"
|
||||
|
||||
ksrcdir="$srcdir/linux-$_basever"
|
||||
builddir="$srcdir/linux-$_basever"
|
||||
|
||||
prepare() {
|
||||
local _patch_failed=
|
||||
cd "$ksrcdir"
|
||||
cd "$builddir"
|
||||
# first apply patches in specified order
|
||||
for i in $source; do
|
||||
case $i in
|
||||
|
|
@ -70,7 +70,7 @@ prepare() {
|
|||
|
||||
mkdir -p "$srcdir"/build
|
||||
cp -v "$srcdir"/$_config "$srcdir"/build/.config
|
||||
make -C "$ksrcdir" O="$srcdir"/build ARCH="$_carch" HOSTCC="$HOSTCC" \
|
||||
make -C "$builddir" O="$srcdir"/build ARCH="$_carch" HOSTCC="$HOSTCC" \
|
||||
olddefconfig
|
||||
}
|
||||
|
||||
|
|
@ -136,7 +136,7 @@ dev() {
|
|||
# external modules, and create the scripts
|
||||
mkdir -p "$dir"
|
||||
cp "$srcdir"/$_config "$dir"/.config
|
||||
make -j1 -C "$ksrcdir" O="$dir" ARCH="$_carch" HOSTCC="$HOSTCC" \
|
||||
make -j1 -C "$builddir" O="$dir" ARCH="$_carch" HOSTCC="$HOSTCC" \
|
||||
olddefconfig prepare modules_prepare scripts
|
||||
|
||||
# needed for 3rd party modules
|
||||
|
|
@ -154,7 +154,7 @@ dev() {
|
|||
# this is taken from ubuntu kernel build script
|
||||
# http://kernel.ubuntu.com/git/ubuntu/ubuntu-zesty.git/tree/debian/rules.d/3-binary-indep.mk
|
||||
|
||||
cd "$ksrcdir"
|
||||
cd "$builddir"
|
||||
find . -path './include/*' -prune \
|
||||
-o -path './scripts/*' -prune -o -type f \
|
||||
\( -name 'Makefile*' -o -name 'Kconfig*' -o -name 'Kbuild*' -o \
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ GNU General Public License for more details.
|
|||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import datetime
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
|
|
@ -155,7 +156,7 @@ def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built):
|
|||
|
||||
|
||||
def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
|
||||
suffix="native", skip_init_buildenv=False):
|
||||
suffix="native", skip_init_buildenv=False, src=None):
|
||||
"""
|
||||
Build all dependencies, check if we need to build at all (otherwise we've
|
||||
just initialized the build environment for nothing) and then setup the
|
||||
|
|
@ -166,6 +167,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
|
|||
build environment. Use this when building
|
||||
something during initialization of the build
|
||||
environment (e.g. qemu aarch64 bug workaround)
|
||||
:param src: override source used to build the package with a local folder
|
||||
:returns: True when the build is necessary (otherwise False)
|
||||
"""
|
||||
# Build dependencies (package arch)
|
||||
|
|
@ -182,6 +184,8 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
|
|||
pmb.build.other.configure_ccache(args, suffix)
|
||||
if not strict and len(depends):
|
||||
pmb.chroot.apk.install(args, depends, suffix)
|
||||
if src:
|
||||
pmb.chroot.apk.install(args, ["rsync"], suffix)
|
||||
|
||||
# Cross-compiler init
|
||||
if cross:
|
||||
|
|
@ -218,14 +222,106 @@ def get_gcc_version(args, arch):
|
|||
return apkindex["version"]
|
||||
|
||||
|
||||
def get_pkgver(original_pkgver, original_source=False, now=None):
|
||||
"""
|
||||
Get the original pkgver when using the original source. Otherwise, get the
|
||||
pkgver with an appended suffix of current date and time. For example:
|
||||
_p20180218550502
|
||||
When appending the suffix, an existing suffix (e.g. _git20171231) gets
|
||||
replaced.
|
||||
|
||||
:param original_pkgver: unmodified pkgver from the package's APKBUILD.
|
||||
:param original_source: the original source is used instead of overriding
|
||||
it with --src.
|
||||
:param now: use a specific date instead of current date (for test cases)
|
||||
"""
|
||||
if original_source:
|
||||
return original_pkgver
|
||||
|
||||
# Append current date
|
||||
no_suffix = original_pkgver.split("_", 1)[0]
|
||||
now = now if now else datetime.datetime.now()
|
||||
new_suffix = "_p" + now.strftime("%Y%m%d%H%M%S")
|
||||
return no_suffix + new_suffix
|
||||
|
||||
|
||||
def override_source(args, apkbuild, pkgver, src, suffix="native"):
|
||||
"""
|
||||
Mount local source inside chroot and append new functions (prepare() etc.)
|
||||
to the APKBUILD to make it use the local source.
|
||||
"""
|
||||
if not src:
|
||||
return
|
||||
|
||||
# Mount source in chroot
|
||||
mount_path = "/mnt/pmbootstrap-source-override/"
|
||||
mount_path_outside = args.work + "/chroot_" + suffix + mount_path
|
||||
pmb.helpers.mount.bind(args, src, mount_path_outside, umount=True)
|
||||
|
||||
# Delete existing append file
|
||||
append_path = "/tmp/APKBUILD.append"
|
||||
append_path_outside = args.work + "/chroot_" + suffix + append_path
|
||||
if os.path.exists(append_path_outside):
|
||||
pmb.chroot.root(args, ["rm", append_path])
|
||||
|
||||
# Add src path to pkgdesc, cut it off after max length
|
||||
pkgdesc = ("[" + src + "] " + apkbuild["pkgdesc"])[:127]
|
||||
|
||||
# Appended content
|
||||
append = """
|
||||
# ** Overrides below appended by pmbootstrap for --src **
|
||||
|
||||
pkgver=\"""" + pkgver + """\"
|
||||
pkgdesc=\"""" + pkgdesc + """\"
|
||||
_pmb_src_copy="/tmp/pmbootstrap-local-source-copy"
|
||||
|
||||
# Empty $source avoids patching in prepare()
|
||||
_pmb_source_original="$source"
|
||||
source=""
|
||||
sha512sums=""
|
||||
|
||||
fetch() {
|
||||
# Update source copy
|
||||
msg "Copying source from host system: """ + src + """\"
|
||||
rsync -a --exclude=".git/" --delete --ignore-errors --force \\
|
||||
\"""" + mount_path + """\" "$_pmb_src_copy" || true
|
||||
|
||||
# Link local source files (e.g. kernel config)
|
||||
mkdir "$srcdir"
|
||||
local s
|
||||
for s in $_pmb_source_original; do
|
||||
is_remote "$s" || ln -sf "$startdir/$s" "$srcdir/"
|
||||
done
|
||||
}
|
||||
|
||||
unpack() {
|
||||
ln -sv "$_pmb_src_copy" "$builddir"
|
||||
}
|
||||
"""
|
||||
|
||||
# Write and log append file
|
||||
with open(append_path_outside, "w", encoding="utf-8") as handle:
|
||||
for line in append.split("\n"):
|
||||
handle.write(line[13:].replace(" " * 4, "\t") + "\n")
|
||||
pmb.chroot.user(args, ["cat", append_path])
|
||||
|
||||
# Append it to the APKBUILD
|
||||
apkbuild_path = "/home/pmos/build/APKBUILD"
|
||||
shell_cmd = ("cat " + apkbuild_path + " " + append_path + " > " +
|
||||
append_path + "_")
|
||||
pmb.chroot.user(args, ["sh", "-c", shlex.quote(shell_cmd)])
|
||||
pmb.chroot.user(args, ["mv", append_path + "_", apkbuild_path])
|
||||
|
||||
|
||||
def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
|
||||
suffix="native"):
|
||||
suffix="native", src=None):
|
||||
"""
|
||||
Set up all environment variables and construct the abuild command (all
|
||||
depending on the cross-compiler method and target architecture), copy
|
||||
the aport to the chroot and execute abuild.
|
||||
|
||||
:param cross: None, "native" or "distcc"
|
||||
:param src: override source used to build the package with a local folder
|
||||
:returns: (output, cmd, env), output is the destination apk path relative
|
||||
to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
|
||||
used by the test case, and they are the full abuild command and
|
||||
|
|
@ -238,9 +334,13 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
|
|||
" probably fail!")
|
||||
|
||||
# Pretty log message
|
||||
output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +
|
||||
pkgver = get_pkgver(apkbuild["pkgver"], src is None)
|
||||
output = (arch + "/" + apkbuild["pkgname"] + "-" + pkgver +
|
||||
"-r" + apkbuild["pkgrel"] + ".apk")
|
||||
logging.info("(" + suffix + ") build " + output)
|
||||
message = "(" + suffix + ") build " + output
|
||||
if src:
|
||||
message += " (source: " + src + ")"
|
||||
logging.info(message)
|
||||
|
||||
# Environment variables
|
||||
env = {"CARCH": arch,
|
||||
|
|
@ -269,6 +369,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
|
|||
|
||||
# Copy the aport to the chroot and build it
|
||||
pmb.build.copy_to_buildpath(args, apkbuild["pkgname"], suffix)
|
||||
override_source(args, apkbuild, pkgver, src, suffix)
|
||||
pmb.chroot.user(args, cmd, suffix, "/home/pmos/build")
|
||||
return (output, cmd, env)
|
||||
|
||||
|
|
@ -296,7 +397,7 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
|
|||
|
||||
|
||||
def package(args, pkgname, arch=None, force=False, strict=False,
|
||||
skip_init_buildenv=False):
|
||||
skip_init_buildenv=False, src=None):
|
||||
"""
|
||||
Build a package and its dependencies with Alpine Linux' abuild.
|
||||
|
||||
|
|
@ -309,6 +410,7 @@ def package(args, pkgname, arch=None, force=False, strict=False,
|
|||
build environment. Use this when building
|
||||
something during initialization of the build
|
||||
environment (e.g. qemu aarch64 bug workaround)
|
||||
:param src: override source used to build the package with a local folder
|
||||
:returns: None if the build was not necessary
|
||||
output path relative to the packages folder ("armhf/ab-1-r2.apk")
|
||||
"""
|
||||
|
|
@ -327,11 +429,11 @@ def package(args, pkgname, arch=None, force=False, strict=False,
|
|||
suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
|
||||
cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
|
||||
if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
|
||||
skip_init_buildenv):
|
||||
skip_init_buildenv, src):
|
||||
return
|
||||
|
||||
# Build and finish up
|
||||
(output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
|
||||
suffix)
|
||||
suffix, src)
|
||||
finish(args, apkbuild, arch, output, strict, suffix)
|
||||
return output
|
||||
|
|
|
|||
|
|
@ -120,11 +120,17 @@ def build(args):
|
|||
for package in args.packages:
|
||||
_build_device_depends_note(args, package)
|
||||
|
||||
# Set src and force
|
||||
src = os.path.realpath(os.path.expanduser(args.src[0])) if args.src else None
|
||||
force = True if src else args.force
|
||||
if src and not os.path.exists(src):
|
||||
raise RuntimeError("Invalid path specified for --src: " + src)
|
||||
|
||||
# Build all packages
|
||||
for package in args.packages:
|
||||
arch_package = args.arch or pmb.build.autodetect.arch(args, package)
|
||||
if not pmb.build.package(args, package, arch_package, args.force,
|
||||
args.strict):
|
||||
if not pmb.build.package(args, package, arch_package, force,
|
||||
args.strict, src=src):
|
||||
logging.info("NOTE: Package '" + package + "' is up to date. Use"
|
||||
" 'pmbootstrap build " + package + " --force'"
|
||||
" if needed.")
|
||||
|
|
|
|||
|
|
@ -36,12 +36,17 @@ def ismount(folder):
|
|||
return False
|
||||
|
||||
|
||||
def bind(args, source, destination, create_folders=True):
|
||||
def bind(args, source, destination, create_folders=True, umount=False):
|
||||
"""
|
||||
Mount --bind a folder and create necessary directory structure.
|
||||
:param umount: when destination is already a mount point, umount it first.
|
||||
"""
|
||||
# Check/umount destination
|
||||
if ismount(destination):
|
||||
return
|
||||
if umount:
|
||||
umount_all(args, destination)
|
||||
else:
|
||||
return
|
||||
|
||||
# Check/create folders
|
||||
for path in [source, destination]:
|
||||
|
|
|
|||
|
|
@ -314,6 +314,11 @@ def arguments():
|
|||
" necessary")
|
||||
build.add_argument("--strict", action="store_true", help="(slower) zap and install only"
|
||||
" required depends when building, to detect dependency errors")
|
||||
build.add_argument("--src", help="override source used to build the"
|
||||
" package with a local folder (the APKBUILD must"
|
||||
" expect the source to be in $builddir, so you might"
|
||||
" need to adjust it)",
|
||||
nargs=1)
|
||||
build.add_argument("-i", "--ignore-depends", action="store_true",
|
||||
help="only build and install makedepends from an"
|
||||
" APKBUILD, ignore the depends (old behavior). This is"
|
||||
|
|
|
|||
|
|
@ -21,8 +21,11 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|||
This file tests all functions from pmb.build._package.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import glob
|
||||
import os
|
||||
import pytest
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
# Import from parent directory
|
||||
|
|
@ -201,6 +204,17 @@ def test_init_buildenv(args, monkeypatch):
|
|||
assert func(args, apkbuild, "armhf") is False
|
||||
|
||||
|
||||
def test_get_pkgver(monkeypatch):
|
||||
# With original source
|
||||
func = pmb.build._package.get_pkgver
|
||||
assert func("1.0", True) == "1.0"
|
||||
|
||||
# Without original source
|
||||
now = datetime.date(2018, 1, 1)
|
||||
assert func("1.0", False, now) == "1.0_p20180101000000"
|
||||
assert func("1.0_git20170101", False, now) == "1.0_p20180101000000"
|
||||
|
||||
|
||||
def test_run_abuild(args, monkeypatch):
|
||||
# Disable effects of functions we don't want to test here
|
||||
monkeypatch.setattr(pmb.build, "copy_to_buildpath", return_none)
|
||||
|
|
@ -311,3 +325,61 @@ def test_build_depends_high_level(args, monkeypatch):
|
|||
# instead.
|
||||
assert pmb.build.package(args, "hello-world-wrapper") is None
|
||||
assert os.path.exists(output_hello_outside)
|
||||
|
||||
|
||||
def test_build_local_source_high_level(args, tmpdir):
|
||||
"""
|
||||
Test building a package with overriding the source code:
|
||||
pmbootstrap build --src=/some/path hello-world
|
||||
|
||||
We use a copy of the hello-world APKBUILD here, that doesn't have the
|
||||
source files it needs to build included. And we use the original aport
|
||||
folder as local source folder, so pmbootstrap should take the source files
|
||||
from there and the build should succeed.
|
||||
"""
|
||||
# aports: Add deviceinfo (required by pmbootstrap to start)
|
||||
tmpdir = str(tmpdir)
|
||||
aports = tmpdir + "/aports"
|
||||
aport = aports + "/device/device-" + args.device
|
||||
os.makedirs(aport)
|
||||
shutil.copy(args.aports + "/device/device-" + args.device + "/deviceinfo",
|
||||
aport)
|
||||
|
||||
# aports: Add modified hello-world aport (source="", uses $builddir)
|
||||
aport = aports + "/main/hello-world"
|
||||
os.makedirs(aport)
|
||||
shutil.copy(pmb.config.pmb_src + "/test/testdata/build_local_src/APKBUILD",
|
||||
aport)
|
||||
|
||||
# src: Copy hello-world source files
|
||||
src = tmpdir + "/src"
|
||||
os.makedirs(src)
|
||||
shutil.copy(args.aports + "/main/hello-world/Makefile", src)
|
||||
shutil.copy(args.aports + "/main/hello-world/main.c", src)
|
||||
|
||||
# src: Create unreadable file (rsync should skip it)
|
||||
unreadable = src + "/_unreadable_file"
|
||||
shutil.copy(args.aports + "/main/hello-world/main.c", unreadable)
|
||||
pmb.helpers.run.root(args, ["chown", "root:root", unreadable])
|
||||
pmb.helpers.run.root(args, ["chmod", "500", unreadable])
|
||||
|
||||
# Delete all hello-world --src packages
|
||||
pattern = (args.work + "/packages/" + args.arch_native +
|
||||
"/hello-world-*_p*.apk")
|
||||
for path in glob.glob(pattern):
|
||||
pmb.helpers.run.root(args, ["rm", path])
|
||||
assert len(glob.glob(pattern)) == 0
|
||||
|
||||
# Build hello-world --src package
|
||||
pmb.helpers.run.user(args, [pmb.config.pmb_src + "/pmbootstrap.py",
|
||||
"--aports", aports, "build", "--src", src,
|
||||
"hello-world"])
|
||||
|
||||
# Verify that the package has been built
|
||||
paths = glob.glob(pattern)
|
||||
assert len(paths) == 1
|
||||
|
||||
# Clean up: delete package and tempfolder, update index
|
||||
pmb.helpers.run.root(args, ["rm", paths[0]])
|
||||
pmb.build.index_repo(args, args.arch_native)
|
||||
pmb.helpers.run.root(args, ["rm", "-r", tmpdir])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
|
||||
# Import from parent directory
|
||||
sys.path.append(os.path.realpath(
|
||||
os.path.join(os.path.dirname(__file__) + "/..")))
|
||||
import pmb.config
|
||||
import pmb.parse
|
||||
import pmb.helpers.frontend
|
||||
import pmb.helpers.logging
|
||||
|
||||
|
||||
def test_build_src_invalid_path():
|
||||
sys.argv = ["pmbootstrap.py", "build", "--src=/invalidpath", "hello-world"]
|
||||
args = pmb.parse.arguments()
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
pmb.helpers.frontend.build(args)
|
||||
assert str(e.value).startswith("Invalid path specified for --src:")
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
pkgname=hello-world
|
||||
pkgver=1
|
||||
pkgrel=0
|
||||
pkgdesc="hello world program for testing 'pmbootstrap build --src'"
|
||||
url="https://en.wikipedia.org/wiki/%22Hello,_World!%22_program"
|
||||
arch="all"
|
||||
license="MIT"
|
||||
depends=""
|
||||
makedepends=""
|
||||
subpackages=""
|
||||
source="non-existing-file.c" # this will be overridden by --src
|
||||
options=""
|
||||
|
||||
build() {
|
||||
cd "$builddir"
|
||||
make
|
||||
}
|
||||
|
||||
check() {
|
||||
cd "$builddir"
|
||||
printf 'hello, world!\n' > expected
|
||||
./hello-world > real
|
||||
diff -q expected real
|
||||
}
|
||||
|
||||
package() {
|
||||
install -D -m755 "$builddir"/hello-world \
|
||||
"$pkgdir"/usr/bin/hello-world
|
||||
}
|
||||
# These will be overridden as well
|
||||
sha512sums="d5ad91600d9be3e53be4cb6e5846b0757786c947b2c0d10f612f67262fc91c148e8d73621623e259ca9dcd5e2c8ec7069cebec44165e203ea8c0133669d3382d invalid-file.c"
|
||||
Loading…
Reference in New Issue