Gracefully handle packages breaking because of soname bumps (#1116)

Fixes #893. Changes:
* New action: "pmbootstrap pkgrel_bump"
* pmbootstrap detects missing soname depends when trying to install
  anyting, and suggests "pkgrel_bump --auto" to fix it
* Testcase test_soname_bump.py checks the pmOS binary package repo
  for soname breakage, so we see it when CI runs for new PRs
* libsamsung-ipc: bump pkgrel because of soname bump
This commit is contained in:
Oliver Smith 2018-01-14 01:26:42 +00:00 committed by GitHub
parent 219aee8ab7
commit 1992f37036
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 537 additions and 6 deletions

View File

@ -1,7 +1,7 @@
pkgname=libsamsung-ipc
pkgver=6.0_0002
_pkgver=${pkgver/_/-}
pkgrel=1
pkgrel=2
pkgdesc="Implementation of Samsung modem protocol"
url="https://redmine.replicant.us/projects/replicant/wiki/Libsamsung-ipc"
arch="all"

View File

@ -36,6 +36,7 @@ import pmb.chroot.other
import pmb.flasher
import pmb.helpers.logging
import pmb.helpers.other
import pmb.helpers.pkgrel_bump
import pmb.helpers.repo
import pmb.helpers.run
import pmb.install
@ -252,6 +253,24 @@ def parse_apkindex(args):
print(json.dumps(result, indent=4))
def pkgrel_bump(args):
would_bump = True
if args.auto:
would_bump = pmb.helpers.pkgrel_bump.auto(args, args.dry)
else:
# Each package must exist
for package in args.packages:
pmb.build.other.find_aport(args, package)
# Increase pkgrel
for package in args.packages:
pmb.helpers.pkgrel_bump.package(args, package, dry=args.dry)
if args.dry and would_bump:
logging.info("Pkgrels of package(s) would have been bumped!")
sys.exit(1)
def qemu(args):
pmb.qemu.run(args)

162
pmb/helpers/pkgrel_bump.py Normal file
View File

@ -0,0 +1,162 @@
"""
Copyright 2018 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import glob
import logging
import os
import pmb.build.other
import pmb.helpers.file
import pmb.helpers.repo
import pmb.parse
def package(args, pkgname, reason="", dry=False):
"""
Increase the pkgrel in the APKBUILD of a specific package.
:param pkgname: name of the package
:param reason: string to display as reason why it was increased
:param dry: don't modify the APKBUILD, just print the message
"""
# Current and new pkgrel
path = pmb.build.other.find_aport(args, pkgname) + "/APKBUILD"
apkbuild = pmb.parse.apkbuild(args, path)
pkgrel = int(apkbuild["pkgrel"])
pkgrel_new = pkgrel + 1
# Display the message, bail out in dry mode
logging.info("Increase '" + pkgname + "' pkgrel (" + str(pkgrel) + " -> " +
str(pkgrel_new) + ")" + reason)
if dry:
return
# Increase
old = "\npkgrel=" + str(pkgrel) + "\n"
new = "\npkgrel=" + str(pkgrel_new) + "\n"
pmb.helpers.file.replace(path, old, new)
# Verify
del(args.cache["apkbuild"][path])
apkbuild = pmb.parse.apkbuild(args, path)
if int(apkbuild["pkgrel"]) != pkgrel_new:
raise RuntimeError("Failed to bump pkgrel for package '" + pkgname +
"'. Make sure that there's a line with exactly the"
" string '" + old + "' and nothing else in: " +
path)
def auto_apkindex_files(args):
"""
Get the paths to the APKINDEX files, that need to be analyzed, sorted by
arch. Relevant are the local pmbootstrap generated APKINDEX as well as the
APKINDEX from the pmOS binary repo.
.
:returns: {"armhf": "...../APKINDEX.tar.gz", ...}
"""
pmb.helpers.repo.update(args)
ret = {}
for arch in pmb.config.build_device_architectures:
ret[arch] = []
local = args.work + "/packages/" + arch + "/APKINDEX.tar.gz"
if os.path.exists(local):
ret[arch].append(local)
if args.mirror_postmarketos:
path = (args.work + "/cache_apk_" + arch + "/APKINDEX." +
pmb.helpers.repo.hash(args.mirror_postmarketos) + ".tar.gz")
ret[arch].append(path)
return ret
def auto_apkindex_package(args, pkgname, aport_version, apkindex, arch,
dry=False):
"""
Bump the pkgrel of a specific package if it is outdated in the given
APKINDEX.
:param pkgname: name of the package
:param aport_version: combination of pkgver and pkgrel (e.g. "1.23-r1")
:param apkindex: path to the APKINDEX.tar.gz file
:param arch: the architecture, e.g. "armhf"
:param dry: don't modify the APKBUILD, just print the message
:returns: True when there was an APKBUILD that needed to be changed.
"""
# Binary package
binary = pmb.parse.apkindex.read(args, pkgname, apkindex,
False)
if not binary:
return
# Skip when aport version != binary package version
compare = pmb.parse.version.compare(aport_version,
binary["version"])
if compare == -1:
logging.warning("WARNING: Skipping '" + pkgname +
"' in index " + apkindex + ", because the"
" binary version " + binary["version"] +
" is higher than the aport version " +
aport_version)
return
if compare == 1:
logging.verbose(pkgname + ": aport version bigger than the"
" one in the APKINDEX, skipping:" +
apkindex)
return
# Find missing depends
logging.verbose(pkgname + ": checking depends: " +
",".join(binary["depends"]))
missing = []
for depend in binary["depends"]:
if not pmb.parse.apkindex.read_any_index(args, depend,
arch):
# We're only interested in missing depends starting with "so:"
# (which means dynamic libraries that the package was linked
# against) and packages for which no aport exists.
if (depend.startswith("so:") or
not pmb.build.other.find_aport(args, depend)):
missing.append(depend)
# Increase pkgrel
if len(missing):
package(args, pkgname, reason=", missing depend(s): " +
", ".join(missing), dry=dry)
return True
def auto(args, dry=False):
"""
:returns: True when there was an APKBUILD that needed to be changed.
"""
# Get APKINDEX files
arch_apkindexes = auto_apkindex_files(args)
# Iterate over aports
ret = False
for aport in glob.glob(args.aports + "/*/*"):
pkgname = os.path.basename(aport)
aport = pmb.parse.apkbuild(args, aport + "/APKBUILD")
aport_version = aport["pkgver"] + "-r" + aport["pkgrel"]
for arch, apkindexes in arch_apkindexes.items():
for apkindex in apkindexes:
if auto_apkindex_package(args, pkgname, aport_version, apkindex,
arch, dry):
ret = True
return ret

View File

@ -123,6 +123,23 @@ def arguments_qemu(subparser):
return ret
def arguments_pkgrel_bump(subparser):
ret = subparser.add_parser("pkgrel_bump", help="increase the pkgrel to"
" indicate that a package must be rebuilt"
" because of a dependency change")
ret.add_argument("--dry", action="store_true", help="instead of modifying"
" APKBUILDs, exit with >0 when a package would have been"
" bumped")
# Mutually exclusive: "--auto" or package names
mode = ret.add_mutually_exclusive_group(required=True)
mode.add_argument("--auto", action="store_true", help="all packages which"
" depend on a library which had an incompatible update"
" (libraries with a soname bump)")
mode.add_argument("packages", nargs="*", default=[])
return ret
def arguments():
parser = argparse.ArgumentParser(prog="pmbootstrap")
arch_native = pmb.parse.arch.alpine_native()
@ -179,6 +196,7 @@ def arguments():
arguments_flasher(sub)
arguments_initfs(sub)
arguments_qemu(sub)
arguments_pkgrel_bump(sub)
# Action: log
log = sub.add_parser("log", help="follow the pmbootstrap logfile")

View File

@ -30,7 +30,7 @@ def recurse_error_message(pkgname, in_aports, in_apkindexes):
ret += " and could not find it"
if in_apkindexes:
ret += " in any APKINDEX"
return ret
return ret + "."
def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
@ -64,27 +64,47 @@ def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
# Get depends and pkgname from aports
depends = None
pkgname = None
version = None
if in_aports:
aport = pmb.build.find_aport(args, pkgname_depend, False)
if aport:
logging.verbose(pkgname_depend + ": found aport: " + aport)
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
depends = apkbuild["depends"]
version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
logging.verbose(pkgname_depend + ": " + version +
" found in " + aport)
if pkgname_depend in apkbuild["subpackages"]:
pkgname = pkgname_depend
else:
pkgname = apkbuild["pkgname"]
# Get depends and pkgname from APKINDEX
if depends is None and in_apkindexes:
if in_apkindexes:
index_data = pmb.parse.apkindex.read_any_index(args, pkgname_depend,
arch)
if index_data:
depends = index_data["depends"]
pkgname = index_data["pkgname"]
# The binary package's depends override the aport's depends in
# case it has the same or a higher version. Binary packages have
# sonames in their dependencies, which we need to detect
# breakage (#893).
outdated = (version and pmb.parse.version.compare(version,
index_data["version"]) == 1)
if not outdated:
if version:
logging.verbose(pkgname_depend + ": binary package is"
" up to date, using binary dependencies"
" instead of the ones from the aport")
depends = index_data["depends"]
pkgname = index_data["pkgname"]
# Nothing found
if pkgname is None and strict:
logging.info("NOTE: Run 'pmbootstrap pkgrel_bump --auto' to mark"
" packages with outdated dependencies for rebuild."
" This will most likely fix this issue (soname"
" bump?).")
logging.info("NOTE: More dependency calculation logging with"
" 'pmbootstrap -v'.")
raise RuntimeError(
recurse_error_message(
pkgname_depend,

172
test/test_pkgrel_bump.py Normal file
View File

@ -0,0 +1,172 @@
"""
Copyright 2018 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
"""
This file tests pmb.helper.pkgrel_bump
"""
import glob
import os
import pytest
import sys
# Import from parent directory
pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.helpers.pkgrel_bump
import pmb.helpers.logging
@pytest.fixture
def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def pmbootstrap(args, tmpdir, parameters, zero_exit=True):
"""
Helper function for running pmbootstrap inside the fake work folder (created
by setup() below) with the binary repo disabled and with the testdata
configured as aports.
:param parameters: what to pass to pmbootstrap, e.g. ["build", "testlib"]
:param zero_exit: expect pmbootstrap to exit with 0 (no error)
"""
# Run pmbootstrap
aports = tmpdir + "/_aports"
config = tmpdir + "/_pmbootstrap.cfg"
try:
pmb.helpers.run.user(args, ["./pmbootstrap.py", "--work=" + tmpdir,
"--mirror-pmOS=", "--aports=" + aports,
"--config=" + config] + parameters,
working_dir=pmb_src)
# Verify that it exits as desired
except Exception as exc:
if zero_exit:
raise RuntimeError("pmbootstrap failed") from exc
else:
return
if not zero_exit:
raise RuntimeError("Expected pmbootstrap to fail, but it did not!")
def setup_work(args, tmpdir):
"""
Create fake work folder in tmpdir with everything symlinked except for the
built packages. The aports testdata gets copied to the tempfolder as
well, so it can be modified during testing.
"""
# Clean the chroots, and initialize the build chroot in the native chroot.
# We do this before creating the fake work folder, because then all packages
# are still present.
os.chdir(pmb_src)
pmb.helpers.run.user(args, ["./pmbootstrap.py", "-y", "zap"])
pmb.helpers.run.user(args, ["./pmbootstrap.py", "build_init"])
pmb.helpers.run.user(args, ["./pmbootstrap.py", "shutdown"])
# Link everything from work (except for "packages") to the tmpdir
for path in glob.glob(args.work + "/*"):
if os.path.basename(path) != "packages":
pmb.helpers.run.user(args, ["ln", "-s", path, tmpdir + "/"])
# Copy testdata and selected device aport
for folder in ["device", "main"]:
pmb.helpers.run.user(args, ["mkdir", "-p", args.aports, tmpdir +
"/_aports/" + folder])
pmb.helpers.run.user(args, ["cp", "-r", args.aports + "/device/device-" +
args.device, tmpdir + "/_aports/device"])
for pkgname in ["testlib", "testapp"]:
pmb.helpers.run.user(args, ["cp", "-r",
"test/testdata/pkgrel_bump/aports/" + pkgname,
tmpdir + "/_aports/main/" + pkgname])
# Empty packages folder
pmb.helpers.run.user(args, ["mkdir", "-p", tmpdir + "/packages"])
pmb.helpers.run.user(args, ["chmod", "777", tmpdir + "/packages"])
# Copy over the pmbootstrap config, disable timestamp based rebuilds
pmb.helpers.run.user(args, ["cp", args.config, tmpdir +
"/_pmbootstrap.cfg"])
pmbootstrap(args, tmpdir, ["config", "timestamp_based_rebuild", "false"])
def verify_pkgrels(args, tmpdir, pkgrel_testlib, pkgrel_testapp):
"""
Verify the pkgrels of the two test APKBUILDs "testlib" and "testapp".
"""
args.cache["apkbuild"] = {}
mapping = {"testlib": pkgrel_testlib, "testapp": pkgrel_testapp}
for pkgname, pkgrel in mapping.items():
# APKBUILD path
path = tmpdir + "/_aports/main/" + pkgname + "/APKBUILD"
# Parse and verify
apkbuild = pmb.parse.apkbuild(args, path)
assert pkgrel == int(apkbuild["pkgrel"])
def test_pkgrel_bump_high_level(args, tmpdir):
# Tempdir setup
tmpdir = str(tmpdir)
setup_work(args, tmpdir)
# Let pkgrel_bump exit normally
pmbootstrap(args, tmpdir, ["build", "testlib"])
pmbootstrap(args, tmpdir, ["build", "testapp"])
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"])
verify_pkgrels(args, tmpdir, 0, 0)
# Increase soname (testlib soname changes with the pkgrel)
pmbootstrap(args, tmpdir, ["pkgrel_bump", "testlib"])
verify_pkgrels(args, tmpdir, 1, 0)
pmbootstrap(args, tmpdir, ["build", "testlib"])
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"])
verify_pkgrels(args, tmpdir, 1, 0)
# Delete package with previous soname (--auto-dry exits with >0 now)
pmb.helpers.run.root(args, ["rm", tmpdir + "/packages/" +
args.arch_native + "/testlib-1.0-r0.apk"])
pmbootstrap(args, tmpdir, ["index"])
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"], False)
verify_pkgrels(args, tmpdir, 1, 0)
# Bump the pkgrel of testapp and build it
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--auto"])
verify_pkgrels(args, tmpdir, 1, 1)
pmbootstrap(args, tmpdir, ["build", "testapp"])
# After rebuilding, pkgrel_bump --auto-dry exits with 0
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "--auto"])
verify_pkgrels(args, tmpdir, 1, 1)
# Test running with specific package names
pmbootstrap(args, tmpdir, ["pkgrel_bump", "invalid_package_name"], False)
pmbootstrap(args, tmpdir, ["pkgrel_bump", "--dry", "testlib"], False)
verify_pkgrels(args, tmpdir, 1, 1)
# Clean up
pmbootstrap(args, tmpdir, ["shutdown"])
pmb.helpers.run.root(args, ["rm", "-rf", tmpdir])

58
test/test_soname_bump.py Normal file
View File

@ -0,0 +1,58 @@
"""
Copyright 2018 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
"""
This file uses pmb.helper.pkgrel_bump to check if the aports need a pkgrel bump
for any package, caused by a soname bump. Example: A new libressl/openssl
version was released, which increased the soname version, and now all packages
that link against it, need to be rebuilt.
"""
import os
import pytest
import sys
# Import from parent directory
pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.helpers.pkgrel_bump
import pmb.helpers.logging
@pytest.fixture
def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def test_soname_bump(args):
if pmb.helpers.pkgrel_bump.auto(args, True):
raise RuntimeError("One or more packages need to be rebuilt, because"
" a library they link against had an incompatible"
" upgrade (soname bump). Run 'pmbootstrap"
" pkgrel_bump --auto' to automatically increase the"
" pkgrel in order to trigger a rebuild. If this"
" test case failed during a pull request, the issue"
" needs to be fixed on the 'master' branch first,"
" then rebase your PR on 'master' afterwards.")

View File

@ -0,0 +1,29 @@
pkgname=testapp
pkgver=1.0
pkgrel=0
pkgdesc="program using the testlib (for testing soname bumps)"
url="https://postmarketos.org"
arch="all"
license="MIT"
depends="testlib"
makedepends=""
subpackages=""
source="testapp.c"
options=""
build() {
cd "$srcdir"
$CC testapp.c -o testapp -L/usr/lib/ -ltestlib
}
check() {
cd "$srcdir"
printf 'hello, world from testlib!\n' > expected
./testapp > real
diff -q expected real
}
package() {
install -Dm755 "$srcdir/testapp" "$pkgdir/usr/bin/testapp"
}
sha512sums="73b167575dc0082a1277b0430f095509885c7aaf55e59bad148825a9879f91fe41c6479bb7f34c0cdd15284b0aadd904a5ba2c1ea85fb8bfb061e1cbf4322d76 testapp.c"

View File

@ -0,0 +1,7 @@
#include <stdio.h>
#include <testlib.h>
int main(int argc, char **argv) {
testlib_hello();
return 0;
}

View File

@ -0,0 +1,38 @@
pkgname=testlib
pkgver=1.0
pkgrel=0
pkgdesc="testing soname bumps (soname changes with pkgrel!)"
url="https://postmarketos.org"
arch="all"
license="MIT"
depends=""
makedepends=""
subpackages=""
source="testlib.c testlib.h"
options="!check"
build() {
cd "$srcdir"
local major="$pkgrel"
local minor="0"
local soname="libtestlib.so.$major"
local realname="libtestlib.so.$minor.$major"
$CC -fPIC -c -g -Wall testlib.c -o libtestlib.o
$CC -shared -Wl,-soname,$soname -o $realname libtestlib.o
ln -sf $realname $soname
ln -sf $soname "libtestlib.so"
}
package() {
cd "$srcdir"
install -Dm755 testlib.h "$pkgdir/usr/include/testlib.h"
mkdir -p "$pkgdir/usr/lib/"
local i
for i in *.so*; do
cp -a "$i" "$pkgdir/usr/lib/$i"
done
}
sha512sums="15c671462a2f043e798b2998e8706f3ac119648c3d3ae946a0115c1f1aec567537f44e7e778bc77d3af4cd05a2d684677dabd56bb35799fca5939c6c087b4e27 testlib.c
16be61567995052e20f9436c6834c2ca2afcfb04fea15c5d02eb576ecfdc9ef4fed8d977468b2564bbe934d098d111837d96cc323dae3f4dd033aa1d061063ee testlib.h"

View File

@ -0,0 +1,5 @@
#include <stdio.h>
void testlib_hello() {
printf("hello, world from testlib!\n");
}

View File

@ -0,0 +1,3 @@
#pragma once
void testlib_hello();