synit
/
pmbootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix #824: Refactor pmb/build/package.py (make depends work like in abuild) (#935) 

* Rename pmb/build/package.py to pmb/build/_package.py, so we can
  access the functions it contains in testcases, and still use
  pmb.build.package()
* Refactor the entire file. Instead of one big function that does
  too many things, we have many small ones now, that are tested
  in the testsuite and easier to modify
* Whenever building a package, pmbootstrap does not only build and
  install the "makedepends" (like we did before), now it does the
  same for the "depends". That's required to be compatible with
  abuild. The old behavior can still be used with 'pmbootstrap
  build --ignore-depends'.
* Because of that change, noarch packages can no longer be built in
  the native chroot if we need them for a foreign chroot. A device-
  package depending on a kernel would pull in the same kernel for
  the native architecture otherwise.
* Running 'pmbootstrap build device-...' without '--ignore-depends'
  and without a matching '--arch' displays a note that explains
  this change to the user and tells how to use it instead.
* Noarch packages no longer get symlinked. That was only
  implemented for packages built in the native chroot, and now that
  is not always the case anymore. Symlinking these packages creates
  packages with broken dependencies anyway (e.g.
  device-samsung-i9100 can't be installed in x86_64, because
  linux-samsung-i9100 is armhf only).
* Rename "carch" to "arch" wherever used. Naming it "carch"
  sometimes is confusing with no benefit.
* Add a testcase for the aarch64 qemu workaround (because it failed
  first and I needed to know for sure if it is working again).
* Improved some verbose logging, which helped with development of
  this feature.
* Removed the old "build" test case (which was disabled in
  testcases_fast.sh) as the new "build_package" test case covers its
  functionallity.
* Only build indexes if the packages folder exists for that arch (Travis
  couldn't run a test case otherwise)
This commit is contained in:
Oliver Smith 2017-11-26 14:32:02 +00:00 committed by GitHub
parent eb1f4a4002
commit d3c77c39ac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 755 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pmb/__init__.py
View File

@ -16,8 +16,6 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import sys
import logging
import os

6
pmb/build/__init__.py
View File

@ -19,8 +19,8 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
# Exported functions
from pmb.build.init import init
from pmb.build.checksum import checksum
from pmb.build.other import copy_to_buildpath, is_necessary, \
symlink_noarch_packages, find_aport, ccache_stats, index_repo
from pmb.build.package import package
from pmb.build.menuconfig import menuconfig
from pmb.build.other import copy_to_buildpath, is_necessary, \
find_aport, ccache_stats, index_repo
from pmb.build._package import package
from pmb.build.qemu_workaround_aarch64 import qemu_workaround_aarch64

289
pmb/build/_package.py Normal file
View File

@ -0,0 +1,289 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import pmb.build
import pmb.build.autodetect
import pmb.build.buildinfo
import pmb.chroot
import pmb.chroot.apk
import pmb.chroot.distccd
import pmb.helpers.repo
import pmb.parse
import pmb.parse.arch
def get_apkbuild(args, pkgname, arch):
"""
Find the APKBUILD path for pkgname. When there is none, try to find it in
the binary package APKINDEX files or raise an exception.
:param pkgname: package name to be built, as specified in the APKBUILD
:returns: None or full path to APKBUILD
"""
# Get existing binary package indexes
pmb.helpers.repo.update(args)
# Get aport, skip upstream only packages
aport = pmb.build.find_aport(args, pkgname, False)
if aport:
return pmb.parse.apkbuild(args, aport + "/APKBUILD")
if pmb.parse.apkindex.read_any_index(args, pkgname, arch):
return None
raise RuntimeError("Package '" + pkgname + "': Could not find aport, and"
" could not find this package in any APKINDEX!")
def check_arch(args, apkbuild, arch):
"""
Check if the APKBUILD can be built for a specific architecture and abort
with a helpful message if it is not the case.
"""
for value in [arch, "all", "noarch"]:
if value in apkbuild["arch"]:
return
pkgname = apkbuild["pkgname"]
logging.info("NOTE: You can edit the 'arch=' line inside the APKBUILD")
if args.action == "build":
logging.info("NOTE: Alternatively, use --arch to build for another"
"architecture ('pmbootstrap build --arch=armhf " +
pkgname + "')")
raise RuntimeError("Can't build '" + pkgname + "' for architecture " +
arch)
def get_depends(args, apkbuild):
"""
Alpine's abuild always builds/installs the "depends" and "makedepends"
of a package before building it. We used to only care about "makedepends"
and it's still possible to ignore the depends with --ignore-depends.
:returns: list of dependency pkgnames (eg. ["sdl2", "sdl2_net"])
"""
ret = list(apkbuild["makedepends"])
if "ignore_depends" not in args or not args.ignore_depends:
ret += apkbuild["depends"]
return sorted(set(ret))
def build_depends(args, apkbuild, arch, strict):
"""
Get and build dependencies with verbose logging messages.
:returns: (depends, depends_built)
"""
# Get dependencies
pkgname = apkbuild["pkgname"]
depends = get_depends(args, apkbuild)
logging.verbose(pkgname + ": build/install dependencies: " +
", ".join(depends))
# Build them
depends_built = []
for depend in depends:
if package(args, depend, arch, strict=strict):
depends_built += [depend]
logging.verbose(pkgname + ": build dependencies: done, built: " +
", ".join(depends_built))
return (depends, depends_built)
def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built):
"""
Check if a build is necessary, and warn if it is not, but there were
dependencies built.
:returns: True or False
"""
pkgname = apkbuild["pkgname"]
ret = True if force else pmb.build.is_necessary(args, arch, apkbuild)
if not ret and len(depends_built):
# Warn of potentially outdated package
logging.warning("WARNING: " + pkgname + " depends on rebuilt" +
" package(s) " + ",".join(depends_built) + " (use" +
" 'pmbootstrap build " + pkgname + " --force' if" +
" necessary!)")
logging.verbose(pkgname + ": build necessary: " + str(ret))
return ret
def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
suffix="native", skip_init_buildenv=False):
"""
Build all dependencies, check if we need to build at all (otherwise we've
just initialized the build environment for nothing) and then setup the
whole build environment (abuild, gcc, dependencies, cross-compiler).
:param cross: None, "native" or "distcc"
:param skip_init_buildenv: can be set to False to avoid initializing the
build environment. Use this when building
something during initialization of the build
environment (e.g. qemu aarch64 bug workaround)
:returns: True when the build is necessary (otherwise False)
"""
# Build dependencies
depends, built = build_depends(args, apkbuild, arch, strict)
# Check if build is necessary
if not is_necessary_warn_depends(args, apkbuild, arch, force, built):
return False
# Install and configure abuild, gcc, dependencies
if not skip_init_buildenv:
pmb.build.init(args, suffix)
pmb.build.other.configure_abuild(args, suffix)
if not strict and len(depends):
pmb.chroot.apk.install(args, depends, suffix)
# Cross-compiler init
if cross:
pmb.chroot.apk.install(args, ["gcc-" + arch, "g++-" + arch,
"ccache-cross-symlinks"])
if cross == "distcc":
pmb.chroot.apk.install(args, ["distcc"], suffix=suffix,
build=False)
pmb.chroot.distccd.start(args, arch)
return True
def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
suffix="native"):
"""
Set up all environment variables and construct the abuild command (all
depending on the cross-compiler method and target architecture), copy
the aport to the chroot and execute abuild.
:param cross: None, "native" or "distcc"
:returns: (output, cmd, env), output is the destination apk path relative
to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
used by the test case, and they are the full abuild command and
the environment variables dict generated in this function.
"""
# Sanity check
if cross == "native" and "!tracedeps" not in apkbuild["options"]:
logging.info("WARNING: Option !tracedeps is not set, but we're"
" cross-compiling in the native chroot. This will"
" probably fail!")
# Pretty log message
output = (arch + "/" + apkbuild["pkgname"] + "-" + apkbuild["pkgver"] +
"-r" + apkbuild["pkgrel"] + ".apk")
logging.info("(" + suffix + ") build " + output)
# Environment variables
env = {"CARCH": arch}
if cross == "native":
hostspec = pmb.parse.arch.alpine_to_hostspec(arch)
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "distcc":
env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path
env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
# Build the abuild command
cmd = []
for key, value in env.items():
cmd += [key + "=" + value]
cmd += ["abuild"]
if strict:
cmd += ["-r"] # install depends with abuild
else:
cmd += ["-d"] # do not install depends with abuild
if force:
cmd += ["-f"]
# Copy the aport to the chroot and build it
pmb.build.copy_to_buildpath(args, apkbuild["pkgname"], suffix)
pmb.chroot.user(args, cmd, suffix, "/home/pmos/build")
return (output, cmd, env)
def finish(args, apkbuild, arch, output, strict=False, suffix="native",
buildinfo=False):
"""
Various finishing tasks that need to be done after a build.
"""
# Verify output file
path = args.work + "/packages/" + output
if not os.path.exists(path):
raise RuntimeError("Package not found after build: " + path)
# Create .buildinfo.json file (captures the build environment, from the
# reproducible builds approach in #64 that we aren't using anymore, but it
# might still be useful)
if buildinfo:
logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
pmb.build.buildinfo.write(args, output, arch, suffix, apkbuild)
# Clear APKINDEX cache (we only parse APKINDEX files once per session and
# cache the result for faster dependency resolving, but after we built a
# package we need to parse it again)
pmb.parse.apkindex.clear_cache(args, args.work + "/packages/" +
arch + "/APKINDEX.tar.gz")
# Uninstall build dependencies (strict mode)
if strict:
logging.info("(" + suffix + ") uninstall build dependencies")
pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build")
def package(args, pkgname, arch=None, force=False, buildinfo=False,
strict=False, skip_init_buildenv=False):
"""
Build a package and its dependencies with Alpine Linux' abuild.
:param pkgname: package name to be built, as specified in the APKBUILD
:param arch: architecture we're building for (default: native)
:param force: even build, if not necessary
:param buildinfo: record the build environment in a .buildinfo.json file
:param strict: avoid building with irrelevant dependencies installed by
letting abuild install and uninstall all dependencies.
:param skip_init_buildenv: can be set to False to avoid initializing the
build environment. Use this when building
something during initialization of the build
environment (e.g. qemu aarch64 bug workaround)
:returns: None if the build was not necessary
output path relative to the packages folder ("armhf/ab-1-r2.apk")
"""
# Only build when APKBUILD exists
arch = arch or args.arch_native
apkbuild = get_apkbuild(args, pkgname, arch)
if not apkbuild:
return
# Detect the build environment (skip unnecessary builds)
check_arch(args, apkbuild, arch)
suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
skip_init_buildenv):
return
# Build and finish up
(output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
suffix)
finish(args, apkbuild, arch, output, strict, suffix, buildinfo)
return output

30
pmb/build/autodetect.py
View File

@ -22,28 +22,8 @@ import pmb.chroot.apk
import pmb.parse.arch
def carch(args, apkbuild, carch, strict=False):
if "noarch" in apkbuild["arch"]:
if "noarch_arch" in args and args.noarch_arch:
return args.noarch_arch
if strict:
return args.deviceinfo["arch"]
return args.arch_native
if carch:
if "all" not in apkbuild["arch"] and carch not in apkbuild["arch"]:
raise RuntimeError("Architecture '" + carch + "' is not supported"
" for this package. Please add it to the"
" 'arch=' line inside the APKBUILD and try"
" again: " + apkbuild["pkgname"])
return carch
if ("all" in apkbuild["arch"] or
args.arch_native in apkbuild["arch"]):
return args.arch_native
return apkbuild["arch"][0]
def suffix(args, apkbuild, carch):
if carch == args.arch_native:
def suffix(args, apkbuild, arch):
if arch == args.arch_native:
return "native"
pkgname = apkbuild["pkgname"]
@ -54,10 +34,10 @@ def suffix(args, apkbuild, carch):
if fnmatch.fnmatch(pkgname, pattern):
return "native"
return "buildroot_" + carch
return "buildroot_" + arch
def crosscompile(args, apkbuild, carch, suffix):
def crosscompile(args, apkbuild, arch, suffix):
"""
:returns: None, "native" or "distcc"
"""
@ -65,7 +45,7 @@ def crosscompile(args, apkbuild, carch, suffix):
return None
if apkbuild["pkgname"].endswith("-repack"):
return None
if not pmb.parse.arch.cpu_emulation_required(args, carch):
if not pmb.parse.arch.cpu_emulation_required(args, arch):
return None
if suffix == "native":
return "native"

70
pmb/build/other.py
View File

@ -246,60 +246,22 @@ def index_repo(args, arch=None):
paths = glob.glob(args.work + "/packages/*")
for path in paths:
path_arch = os.path.basename(path)
path_repo_chroot = "/home/pmos/packages/pmos/" + path_arch
logging.debug("(native) index " + path_arch + " repository")
commands = [
["apk", "-q", "index", "--output", "APKINDEX.tar.gz_",
"--rewrite-arch", path_arch, "*.apk"],
["abuild-sign", "APKINDEX.tar.gz_"],
["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
]
for command in commands:
pmb.chroot.user(args, command, working_dir=path_repo_chroot)
pmb.parse.apkindex.clear_cache(args, args.work + path +
"/APKINDEX.tar.gz")
def symlink_noarch_packages(args):
"""
All noarch packages from the native architecture folder (x86_64 usually)
get symlinked to all other architectures.
"""
# Create the arch folders
architectures = pmb.config.build_device_architectures
logging.debug("Symlink noarch-packages to " + ", ".join(architectures))
for arch in architectures:
arch_folder = "/mnt/pmbootstrap-packages/" + arch
arch_folder_outside = args.work + "/packages/" + arch
if not os.path.exists(arch_folder_outside):
pmb.chroot.user(args, ["mkdir", "-p", arch_folder])
# Create an APKINDEX *without* replaced architectures (that is much
# faster than reading each apk file with Python!)
index = "/tmp/APKINDEX_without_replaced_archs"
index_outside = args.work + "/chroot_native" + index
pmb.chroot.user(args, ["apk", "-q", "index", "--output", index, "*.apk"],
working_dir="/mnt/pmbootstrap-packages/" + args.arch_native)
# Iterate over noarch packages
for package, data in pmb.parse.apkindex.parse(args, index_outside).items():
if data["arch"] != "noarch":
continue
# Create missing symlinks
apk_file = data["pkgname"] + "-" + data["version"] + ".apk"
for arch in architectures:
if os.path.exists(args.work + "/packages/" + arch + "/" + apk_file):
continue
arch_folder = "/mnt/pmbootstrap-packages/" + arch
source = "../" + args.arch_native + "/" + apk_file
pmb.chroot.user(args, ["ln", "-sf", source, "."],
working_dir=arch_folder)
# Rewrite indexes
for arch in architectures:
index_repo(args, arch)
if os.path.exists(path):
path_arch = os.path.basename(path)
path_repo_chroot = "/home/pmos/packages/pmos/" + path_arch
logging.debug("(native) index " + path_arch + " repository")
commands = [
["apk", "-q", "index", "--output", "APKINDEX.tar.gz_",
"--rewrite-arch", path_arch, "*.apk"],
["abuild-sign", "APKINDEX.tar.gz_"],
["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
]
for command in commands:
pmb.chroot.user(args, command, working_dir=path_repo_chroot)
else:
logging.debug("NOTE: Can't build index for non-existing path: " +
path)
pmb.parse.apkindex.clear_cache(args, path + "/APKINDEX.tar.gz")
def ccache_stats(args, arch):

144
pmb/build/package.py
View File

@ -1,144 +0,0 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import logging
import pmb.build
import pmb.build.autodetect
import pmb.build.buildinfo
import pmb.chroot
import pmb.chroot.apk
import pmb.chroot.distccd
import pmb.helpers.repo
import pmb.parse
import pmb.parse.arch
def package(args, pkgname, carch, force=False, buildinfo=False, strict=False,
init_buildenv=True):
"""
Build a package with Alpine Linux' abuild.
:param force: even build, if not necessary
:returns: output path relative to the packages folder
"""
# Get existing binary package indexes
pmb.helpers.repo.update(args)
# Get aport, skip upstream only packages
aport = pmb.build.find_aport(args, pkgname, False)
if not aport:
if pmb.parse.apkindex.read_any_index(args, pkgname, carch):
return
raise RuntimeError("Package " + pkgname + ": Could not find aport,"
" and could not find this package in any APKINDEX!")
# Autodetect the build environment
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
pkgname = apkbuild["pkgname"]
carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch, strict)
suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv)
cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv,
suffix)
# Skip already built versions
if not force and not pmb.build.is_necessary(args, carch_buildenv, apkbuild):
return
# Initialize build environment, install/build makedepends
if init_buildenv:
pmb.build.init(args, suffix)
if len(apkbuild["makedepends"]):
if strict:
for makedepend in apkbuild["makedepends"]:
package(args, makedepend, carch_buildenv, strict=True)
else:
pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix)
if cross:
pmb.chroot.apk.install(args, ["gcc-" + carch_buildenv,
"g++-" + carch_buildenv,
"ccache-cross-symlinks"])
if cross == "distcc":
pmb.chroot.apk.install(args, ["distcc"], suffix=suffix,
build=False)
pmb.chroot.distccd.start(args, carch_buildenv)
# Avoid re-building for circular dependencies
if not force and not pmb.build.is_necessary(args, carch, apkbuild):
return
# Configure abuild.conf
pmb.build.other.configure_abuild(args, suffix)
# Generate output name, log build message
output = (carch_buildenv + "/" + apkbuild["pkgname"] + "-" +
apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + ".apk")
logging.info("(" + suffix + ") build " + output)
# Sanity check
if cross == "native" and "!tracedeps" not in apkbuild["options"]:
logging.info("WARNING: Option !tracedeps is not set, but we're"
" cross-compiling in the native chroot. This will probably"
" fail!")
# Run abuild
pmb.build.copy_to_buildpath(args, pkgname, suffix)
cmd = []
env = {"CARCH": carch_buildenv}
if cross == "native":
hostspec = pmb.parse.arch.alpine_to_hostspec(carch_buildenv)
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
if cross == "distcc":
env["PATH"] = "/usr/lib/distcc/bin:" + pmb.config.chroot_path
env["DISTCC_HOSTS"] = "127.0.0.1:" + args.port_distccd
for key, value in env.items():
cmd += [key + "=" + value]
cmd += ["abuild"]
if strict:
cmd += ["-r"] # install depends with abuild
else:
cmd += ["-d"] # do not install depends with abuild
if force:
cmd += ["-f"]
pmb.chroot.user(args, cmd, suffix, "/home/pmos/build")
# Verify output file
path = args.work + "/packages/" + output
if not os.path.exists(path):
raise RuntimeError("Package not found after build: " + path)
# Create .buildinfo.json file
if buildinfo:
logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
pmb.build.buildinfo.write(args, output, carch_buildenv, suffix,
apkbuild)
# Symlink noarch package (and subpackages)
if "noarch" in apkbuild["arch"]:
pmb.build.symlink_noarch_packages(args)
# Clean up (APKINDEX cache, depends when strict)
pmb.parse.apkindex.clear_cache(args, args.work + "/packages/" +
carch_buildenv + "/APKINDEX.tar.gz")
if strict:
logging.info("(" + suffix + ") uninstall makedepends")
pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build")
return output

4
pmb/build/qemu_workaround_aarch64.py
View File

@ -16,7 +16,7 @@ GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import pmb.build.package
import pmb.build
import pmb.chroot.apk
@ -30,5 +30,5 @@ def qemu_workaround_aarch64(args, suffix="buildroot_aarch64"):
"""
pkgname = "abuild-aarch64-qemu-workaround"
pmb.build.package(args, pkgname, "aarch64", True,
init_buildenv=False)
skip_init_buildenv=True)
pmb.chroot.apk.install(args, [pkgname], suffix, False)

2
pmb/chroot/apk.py
View File

@ -196,7 +196,7 @@ def install(args, packages, suffix="native", build=True):
packages_with_depends = pmb.parse.depends.recurse(args, packages, arch,
strict=True)
# Filter out up-to-date packages
# Filter outdated packages (build them if required)
packages_installed = installed(args, suffix)
packages_todo = []
for package in packages_with_depends:

55
pmb/helpers/frontend.py
View File

@ -42,6 +42,50 @@ import pmb.parse
import pmb.qemu
def _build_verify_usage_device_package(args, pkgname):
"""
Detect if the user is about to build a device- package for the wrong
architecture. The package is noarch, but the dependencies (kernel!) will get
pulled in with the same arch as dependency.
"""
# Skip non-device-packages
if not pkgname.startswith("device-"):
return
# Only continue when the --arch parameter is *not* the device architecture
deviceinfo = args.aports + "/device/" + pkgname + "/deviceinfo"
if not os.path.exists(deviceinfo):
return
device = pkgname.split("-", 1)[1]
arch = pmb.parse.deviceinfo(args, device)["arch"]
if args.arch == arch:
return
# Abort with a big note
logging.info("Dependency handling in 'pmbootstrap build' has been"
" changed.")
logging.info("Previously we only built and installed the 'makedepends'"
" from the APKBUILDs, now we use the 'depends', too.")
logging.info("")
logging.info("Your options:")
logging.info("* Ignore depends (fast, old behavior, may cause problems"
" with some packages):")
logging.info(" pmbootstrap build " + pkgname + " -i")
logging.info("* Build with depends (kernel!) and specify the right"
" architecture:")
logging.info(" pmbootstrap build " + pkgname + " --arch=" + arch)
logging.info("")
logging.info("This change was necessary to be more compatible with Alpine's"
" abuild.")
logging.info("The default architecture is the native one (" +
args.arch_native + " in your case), so you need to overwrite")
logging.info("it now to get the kernel dependency of your device package"
" for the right architecture.")
logging.info("Sorry for the inconvenience.")
logging.info("")
raise RuntimeError("Missing -i or --arch parameter")
def _parse_flavor(args):
"""
Verify the flavor argument if specified, or return a default value.
@ -86,8 +130,16 @@ def aportgen(args):
def build(args):
# Strict mode: zap everything
if args.strict:
pmb.chroot.zap(args, False)
# Detect wrong usage for device- packages
if not args.ignore_depends:
for package in args.packages:
_build_verify_usage_device_package(args, package)
# Build all packages
for package in args.packages:
pmb.build.package(args, package, args.arch, args.force,
args.buildinfo, args.strict)
@ -136,8 +188,7 @@ def config(args):
def index(args):
pmb.build.index_repo(args, args.arch_native)
pmb.build.symlink_noarch_packages(args)
pmb.build.index_repo(args)
def initfs(args):

5
pmb/parse/apkindex.py
View File

@ -251,10 +251,9 @@ def read_any_index(args, package, arch=None):
# Return first match
for index in pmb.helpers.repo.apkindex_files(args, arch):
index_data = read(args, package, index, False)
logging.verbose("Search for " + package + " in " + index +
" - result: " + str(index_data))
if index_data:
logging.verbose(package + ": found in " + index)
return index_data
logging.verbose("No match found in any APKINDEX.tar.gz!")
logging.verbose(package + ": no match found in any APKINDEX.tar.gz!")
return None

14
pmb/parse/arguments.py
View File

@ -270,11 +270,21 @@ def arguments():
" (aport/APKBUILD) based on an upstream aport from Alpine")
build = sub.add_parser("build", help="create a package for a"
" specific architecture")
build.add_argument("--arch", choices=arch_choices)
build.add_argument("--force", action="store_true")
build.add_argument("--arch", choices=arch_choices, default=arch_native,
help="CPU architecture to build for (default: " +
arch_native + ")")
build.add_argument("--force", action="store_true", help="even build if not"
" necessary")
build.add_argument("--buildinfo", action="store_true")
build.add_argument("--strict", action="store_true", help="(slower) zap and install only"
" required depends when building, to detect dependency errors")
build.add_argument("-i", "--ignore-depends", action="store_true",
help="only build and install makedepends from an"
" APKBUILD, ignore the depends (old behavior). This is"
" faster for device packages for example, because then"
" you don't need to build and install the kernel. But it"
" is incompatible with how Alpine's abuild handles it.",
dest="ignore_depends")
build.add_argument("--noarch-arch", dest="noarch_arch", default=None,
help="which architecture to use to build 'noarch'"
" packages. Defaults to the native arch normally,"

11
pmb/parse/depends.py
View File

@ -62,13 +62,12 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
continue
# Get depends and pkgname from aports
logging.verbose("Get dependencies of: " + pkgname_depend)
depends = None
pkgname = None
if in_aports:
aport = pmb.build.find_aport(args, pkgname_depend, False)
if aport:
logging.verbose("-> Found aport: " + aport)
logging.verbose(pkgname_depend + ": found aport: " + aport)
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
depends = apkbuild["depends"]
if pkgname_depend in apkbuild["subpackages"]:
@ -78,7 +77,6 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
# Get depends and pkgname from APKINDEX
if depends is None and in_apkindexes:
logging.verbose("-> Search through APKINDEX files")
index_data = pmb.parse.apkindex.read_any_index(args, pkgname_depend,
arch)
if index_data:
@ -95,12 +93,11 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
# Append to todo/ret (unless it is a duplicate)
if pkgname != pkgname_depend:
logging.verbose("-> '" + pkgname_depend + "' is provided by '" +
pkgname + "'")
logging.verbose(pkgname_depend + ": provided by '" + pkgname + "'")
if pkgname in ret:
logging.verbose("-> '" + pkgname + "' already found")
logging.verbose(pkgname + ": already found")
else:
logging.verbose("-> '" + pkgname + "' depends on: " + str(depends))
logging.verbose(pkgname + ": depends on: " + ",".join(depends))
if depends:
todo += depends
ret.append(pkgname)

2
test/test_aportgen_device_wizard.py
View File

@ -32,7 +32,7 @@ import pmb.parse
@pytest.fixture
def args(tmpdir, request):
sys.argv = ["pmbootstrap.py", "chroot"]
sys.argv = ["pmbootstrap.py", "build", "-i", "device-testsuite-testdevice"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)

51
test/test_build.py
View File

@ -1,51 +0,0 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import sys
import pytest
# Import from parent directory
sys.path.append(os.path.realpath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.aportgen
import pmb.config
import pmb.helpers.logging
@pytest.fixture
def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def test_build(args):
pmb.build.package(args, "hello-world", args.arch_native, True)
def test_build_cross(args):
"""
Build in non-native chroot, with cross-compiler through distcc.
"""
for arch in pmb.config.build_device_architectures:
pmb.build.package(args, "hello-world", arch, True)

289
test/test_build_package.py Normal file
View File

@ -0,0 +1,289 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
"""
This file tests all functions from pmb.build._package.
"""
import os
import pytest
import sys
# Import from parent directory
sys.path.append(os.path.realpath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build
import pmb.build._package
import pmb.config
import pmb.config.init
import pmb.helpers.logging
@pytest.fixture
def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap", "init"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def return_none(*args, **kwargs):
return None
def return_string(*args, **kwargs):
return "some/random/path.apk"
def return_true(*args, **kwargs):
return True
def return_false(*args, **kwargs):
return False
def return_fake_build_depends(*args, **kwargs):
"""
Fake return value for pmb.build._package.build_depends:
depends: ["alpine-base"], depends_built: []
"""
return (["alpine-base"], [])
def args_patched(monkeypatch, argv):
monkeypatch.setattr(sys, "argv", argv)
return pmb.parse.arguments()
def test_get_apkbuild(args):
func = pmb.build._package.get_apkbuild
# Valid aport
pkgname = "postmarketos-base"
assert func(args, pkgname, "x86_64")["pkgname"] == pkgname
# Valid binary package
assert func(args, "alpine-base", "x86_64") is None
# Invalid package
with pytest.raises(RuntimeError) as e:
func(args, "invalid-package-name", "x86_64")
assert "Could not find" in str(e.value)
def test_check_arch(args):
func = pmb.build._package.check_arch
apkbuild = {"pkgname": "test"}
# Arch is right
apkbuild["arch"] = ["armhf"]
func(args, apkbuild, "armhf")
apkbuild["arch"] = ["noarch"]
func(args, apkbuild, "armhf")
apkbuild["arch"] = ["all"]
func(args, apkbuild, "armhf")
# Arch is wrong
apkbuild["arch"] = ["x86_64"]
with pytest.raises(RuntimeError) as e:
func(args, apkbuild, "armhf")
assert "Can't build" in str(e.value)
def test_get_depends(monkeypatch):
func = pmb.build._package.get_depends
apkbuild = {"depends": ["a"], "makedepends": ["c", "b"]}
# Depends + makedepends
args = args_patched(monkeypatch, ["pmbootstrap", "build", "test"])
assert func(args, apkbuild) == ["a", "b", "c"]
args = args_patched(monkeypatch, ["pmbootstrap", "install"])
assert func(args, apkbuild) == ["a", "b", "c"]
# Ignore depends (-i)
args = args_patched(monkeypatch, ["pmbootstrap", "build", "-i", "test"])
assert func(args, apkbuild) == ["b", "c"]
def test_build_depends(args, monkeypatch):
# Shortcut and fake apkbuild
func = pmb.build._package.build_depends
apkbuild = {"pkgname": "test", "depends": ["a"], "makedepends": ["b"]}
# No depends built (first makedepends + depends, then only makedepends)
monkeypatch.setattr(pmb.build._package, "package", return_none)
assert func(args, apkbuild, "armhf", True) == (["a", "b"], [])
# All depends built (makedepends only)
monkeypatch.setattr(pmb.build._package, "package", return_string)
assert func(args, apkbuild, "armhf", False) == (["a", "b"], ["a", "b"])
def test_is_necessary_warn_depends(args, monkeypatch):
# Shortcut and fake apkbuild
func = pmb.build._package.is_necessary_warn_depends
apkbuild = {"pkgname": "test"}
# Necessary
monkeypatch.setattr(pmb.build, "is_necessary", return_true)
assert func(args, apkbuild, "armhf", False, []) is True
# Necessary (strict=True overrides is_necessary())
monkeypatch.setattr(pmb.build, "is_necessary", return_false)
assert func(args, apkbuild, "armhf", True, []) is True
# Not necessary (with depends: different code path that prints a warning)
assert func(args, apkbuild, "armhf", False, []) is False
assert func(args, apkbuild, "armhf", False, ["first", "second"]) is False
def test_init_buildenv(args, monkeypatch):
# Disable effects of functions we don't want to test here
monkeypatch.setattr(pmb.build._package, "build_depends",
return_fake_build_depends)
monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends",
return_true)
monkeypatch.setattr(pmb.chroot.apk, "install", return_none)
monkeypatch.setattr(pmb.chroot.distccd, "start", return_none)
# Shortcut and fake apkbuild
func = pmb.build._package.init_buildenv
apkbuild = {"pkgname": "test", "depends": ["a"], "makedepends": ["b"]}
# Build is necessary (various code paths)
assert func(args, apkbuild, "armhf", strict=True) is True
assert func(args, apkbuild, "armhf", cross="native") is True
assert func(args, apkbuild, "armhf", cross="distcc") is True
# Build is not necessary (only builds dependencies)
monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends",
return_false)
assert func(args, apkbuild, "armhf") is False
def test_run_abuild(args, monkeypatch):
# Disable effects of functions we don't want to test here
monkeypatch.setattr(pmb.build, "copy_to_buildpath", return_none)
monkeypatch.setattr(pmb.chroot, "user", return_none)
# Shortcut and fake apkbuild
func = pmb.build._package.run_abuild
apkbuild = {"pkgname": "test", "pkgver": "1", "pkgrel": "2", "options": []}
# Normal run
output = "armhf/test-1-r2.apk"
env = {"CARCH": "armhf"}
cmd = ["CARCH=armhf", "abuild", "-d"]
assert func(args, apkbuild, "armhf") == (output, cmd, env)
# Force and strict
cmd = ["CARCH=armhf", "abuild", "-r", "-f"]
assert func(args, apkbuild, "armhf", True, True) == (output, cmd, env)
# cross=native
env = {"CARCH": "armhf",
"CROSS_COMPILE": "armv6-alpine-linux-muslgnueabihf-",
"CC": "armv6-alpine-linux-muslgnueabihf-gcc"}
cmd = ["CARCH=armhf", "CROSS_COMPILE=armv6-alpine-linux-muslgnueabihf-",
"CC=armv6-alpine-linux-muslgnueabihf-gcc", "abuild", "-d"]
assert func(args, apkbuild, "armhf", cross="native") == (output, cmd, env)
# cross=distcc
env = {"CARCH": "armhf",
"PATH": "/usr/lib/distcc/bin:" + pmb.config.chroot_path,
"DISTCC_HOSTS": "127.0.0.1:33632"}
cmd = ["CARCH=armhf", "PATH=" + "/usr/lib/distcc/bin:" +
pmb.config.chroot_path, "DISTCC_HOSTS=127.0.0.1:33632", "abuild",
"-d"]
assert func(args, apkbuild, "armhf", cross="distcc") == (output, cmd, env)
def test_finish(args, monkeypatch):
# Real output path
output = pmb.build.package(args, "hello-world", force=True)
# Disable effects of functions we don't want to test below
monkeypatch.setattr(pmb.build.buildinfo, "write", return_none)
monkeypatch.setattr(pmb.chroot, "user", return_none)
# Shortcut and fake apkbuild
func = pmb.build._package.finish
apkbuild = {}
# Non-existing output path
with pytest.raises(RuntimeError) as e:
func(args, apkbuild, "armhf", "/invalid/path")
assert "Package not found" in str(e.value)
# Existing output path
func(args, apkbuild, args.arch_native, output)
def test_package(args):
# First build
assert pmb.build.package(args, "hello-world", force=True)
# Package exists
assert pmb.build.package(args, "hello-world") is None
# Force building again
assert pmb.build.package(args, "hello-world", force=True)
# Build for another architecture
assert pmb.build.package(args, "hello-world", "armhf", force=True)
# Upstream package, for which we don't have an aport
assert pmb.build.package(args, "alpine-base") is None
def test_build_depends_high_level(args, monkeypatch):
"""
"hello-world-wrapper" depends on "hello-world". We build both, then delete
"hello-world" and check that it gets rebuilt correctly again.
"""
# Patch pmb.build.is_necessary() to always build the hello-world package
def fake_build_is_necessary(args, arch, apkbuild, apkindex_path=None):
if apkbuild["pkgname"] == "hello-world":
return True
return pmb.build.other.is_necessary(args, arch, apkbuild,
apkindex_path)
monkeypatch.setattr(pmb.build, "is_necessary",
fake_build_is_necessary)
# Build hello-world to get its full output path
output_hello = pmb.build.package(args, "hello-world")
output_hello_outside = args.work + "/packages/" + output_hello
assert os.path.exists(output_hello_outside)
# Make sure the wrapper exists
pmb.build.package(args, "hello-world-wrapper")
# Remove hello-world
pmb.helpers.run.root(args, ["rm", output_hello_outside])
pmb.build.index_repo(args, args.arch_native)
# Ask to build the wrapper. It should not build the wrapper (it exists, not
# using force), but build/update its missing dependency "hello-world"
# instead.
assert pmb.build.package(args, "hello-world-wrapper") is None
assert os.path.exists(output_hello_outside)

76
test/test_buildroot_aarch64_init.py Normal file
View File

@ -0,0 +1,76 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
"""
This file tests all functions from pmb.build._package.
"""
import glob
import os
import pytest
import sys
# Import from parent directory
sys.path.append(os.path.realpath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build
import pmb.helpers.logging
@pytest.fixture
def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap", "init"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def test_buildroot_aarch64_init(args, monkeypatch):
# Patch pmb.build.is_necessary() to always build the workaround package
def fake_build_is_necessary(args, arch, apkbuild, apkindex_path=None):
if apkbuild["pkgname"] == "abuild-aarch64-qemu-workaround":
return True
return pmb.build.other.is_necessary(args, arch, apkbuild,
apkindex_path)
monkeypatch.setattr(pmb.build, "is_necessary",
fake_build_is_necessary)
# Remove aarch64 chroot
pmb.chroot.shutdown(args)
path = args.work + "/chroot_buildroot_aarch64"
if os.path.exists(path):
pmb.helpers.run.root(args, ["rm", "-rf", path])
# Remove existing workaround packages
pattern_workaround_apk = (args.work + "/packages/aarch64/"
"abuild-aarch64-qemu-workaround-*")
for match in glob.glob(pattern_workaround_apk):
pmb.helpers.run.root(args, ["rm", match])
pmb.build.index_repo(args, "aarch64")
# Build hello-world for aarch64, causing the chroot to initialize properly
pmb.build.package(args, "hello-world", "aarch64", force=True)
# Verify that the workaround was built and installed
assert len(glob.glob(pattern_workaround_apk))
assert os.path.exists(args.work + "/chroot_buildroot_aarch64/usr/bin"
"/abuild-tar-patched")

2
test/test_challenge_build.py
View File

@ -23,7 +23,7 @@ import pytest
# Import from parent directory
sys.path.append(os.path.realpath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build.package
import pmb.build
import pmb.challenge.build
import pmb.config
import pmb.helpers.logging

2
test/test_repo.py
View File

@ -25,7 +25,7 @@ import time
# Import from parent directory
pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.build.package
import pmb.build
import pmb.helpers.logging
import pmb.helpers.repo

2
test/testcases_fast.sh
View File

@ -3,12 +3,10 @@
# Disable slow testcases
# aport_in_sync_with_git: clones Alpine's aports repo
# aportgen: clones Alpine's aports repo
# build: builds cross-compilers for aarch64 and armhf
# version: clones Alpine's apk repo
disabled="
aport_in_sync_with_git
aportgen
build
version
"