diff --git a/README.md b/README.md index d2063d39..c2231e77 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,11 @@ $ pmbootstrap flasher --method=adb sideload ``` ### Repository Maintenance +List pmaports that don't have a binary package: +``` +$ pmbootstrap repo_missing --arch=armhf --overview +``` + Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps): ``` $ pmbootstrap pkgrel_bump --auto diff --git a/pmb/build/_package.py b/pmb/build/_package.py index 6ea36759..fd92e92f 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -50,35 +50,33 @@ def skip_already_built(args, pkgname, arch): def get_apkbuild(args, pkgname, arch): """ - Find the APKBUILD path for pkgname. When there is none, try to find it in + Parse the APKBUILD path for pkgname. When there is none, try to find it in the binary package APKINDEX files or raise an exception. :param pkgname: package name to be built, as specified in the APKBUILD - :returns: None or full path to APKBUILD + :returns: None or parsed APKBUILD """ # Get existing binary package indexes pmb.helpers.repo.update(args, arch) - # Get aport, skip upstream only packages - aport = pmb.helpers.pmaports.find(args, pkgname, False) - if aport: - return pmb.parse.apkbuild(args, aport + "/APKBUILD") + # Get pmaport, skip upstream only packages + pmaport = pmb.helpers.pmaports.get(args, pkgname, False) + if pmaport: + return pmaport if pmb.parse.apkindex.providers(args, pkgname, arch, False): return None raise RuntimeError("Package '" + pkgname + "': Could not find aport, and" " could not find this package in any APKINDEX!") -def check_arch(args, apkbuild, arch): +def check_arch_abort(args, pkgname, arch): """ Check if the APKBUILD can be built for a specific architecture and abort with a helpful message if it is not the case. """ - for value in [arch, "all", "noarch"]: - if value in apkbuild["arch"]: - return + if pmb.helpers.package.check_arch(args, pkgname, arch, False): + return - pkgname = apkbuild["pkgname"] logging.info("NOTE: You can edit the 'arch=' line inside the APKBUILD") if args.action == "build": logging.info("NOTE: Alternatively, use --arch to build for another" @@ -435,7 +433,7 @@ def package(args, pkgname, arch=None, force=False, strict=False, return # Detect the build environment (skip unnecessary builds) - check_arch(args, apkbuild, arch) + check_arch_abort(args, pkgname, arch) suffix = pmb.build.autodetect.suffix(args, apkbuild, arch) cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix) if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix, diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index e435d254..79d7a429 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -116,7 +116,9 @@ def add_cache(args): "apk_repository_list_updated": [], "built": {}, "find_aport": {}, - "offline_msg_shown": False}) + "offline_msg_shown": False, + "pmb.helpers.package.depends_recurse": {}, + "pmb.helpers.package.get": {}}) def add_deviceinfo(args): diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 7ecd7d65..72b5405e 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -38,6 +38,7 @@ import pmb.helpers.logging import pmb.helpers.pkgrel_bump import pmb.helpers.pmaports import pmb.helpers.repo +import pmb.helpers.repo_missing import pmb.helpers.run import pmb.install import pmb.parse @@ -161,6 +162,12 @@ def config(args): pmb.helpers.logging.disable() +def repo_missing(args): + missing = pmb.helpers.repo_missing.generate(args, args.arch, args.overview, + args.package, args.built) + print(json.dumps(missing, indent=4)) + + def index(args): pmb.build.index_repo(args) @@ -259,11 +266,9 @@ def apkbuild_parse(args): # Default to all packages packages = args.packages if not packages: - for apkbuild in glob.glob(args.aports + "/*/*/APKBUILD"): - packages.append(os.path.basename(os.path.dirname(apkbuild))) + packages = pmb.helpers.pmaports.get_list(args) # Iterate over all packages - packages.sort() for package in packages: print(package + ":") aport = pmb.helpers.pmaports.find(args, package) diff --git a/pmb/helpers/package.py b/pmb/helpers/package.py new file mode 100644 index 00000000..08adc392 --- /dev/null +++ b/pmb/helpers/package.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +""" +Functions that work on both pmaports and (binary package) repos. See also: +- pmb/helpers/pmaports.py (work on pmaports) +- pmb/helpers/repo.py (work on binary package repos) +""" + +import logging +import copy + +import pmb.helpers.pmaports + + +def get(args, pkgname, arch): + """ Find a package in pmaports, and as fallback in the APKINDEXes of the + binary packages. + :param pkgname: package name (e.g. "hello-world") + :param arch: preferred architecture of the binary package. When it + can't be found for this arch, we'll still look for another + arch to see whether the package exists at all. So make + sure to check the returned arch against what you wanted + with check_arch(). Example: "armhf" + :returns: data from the parsed APKBUILD or APKINDEX in the following + format: {"arch": ["noarch"], + "depends": ["busybox-extras", "lddtree", ...], + "pkgname": "postmarketos-mkinitfs", + "provides": ["mkinitfs=0..1"], + "version": "0.0.4-r10"} """ + # Cached result + cache_key = "pmb.helpers.package.get" + if (arch in args.cache[cache_key] and + pkgname in args.cache[cache_key][arch]): + return args.cache[cache_key][arch][pkgname] + + # Find in pmaports + ret = None + pmaport = pmb.helpers.pmaports.get(args, pkgname, False) + if pmaport: + ret = {"arch": pmaport["arch"], + "depends": pmb.build._package.get_depends(args, pmaport), + "pkgname": pkgname, + "provides": pmaport["provides"], + "version": pmaport["pkgver"] + "-r" + pmaport["pkgrel"]} + + # Find in APKINDEX (given arch) + if not ret: + pmb.helpers.repo.update(args, arch) + ret = pmb.parse.apkindex.package(args, pkgname, arch, False) + + # Find in APKINDEX (other arches) + if not ret: + for arch_i in pmb.config.build_device_architectures: + if arch_i != arch: + ret = pmb.parse.apkindex.package(args, pkgname, arch_i, False) + if ret: + break + + # Copy ret (it might have references to caches of the APKINDEX or APKBUILDs + # and we don't want to modify those!) + if ret: + ret = copy.deepcopy(ret) + + # Make sure ret["arch"] is a list (APKINDEX code puts a string there) + if ret and isinstance(ret["arch"], str): + ret["arch"] = [ret["arch"]] + + # Save to cache and return + if ret: + if arch not in args.cache[cache_key]: + args.cache[cache_key][arch] = {} + args.cache[cache_key][arch][pkgname] = ret + return ret + + # Could not find the package + raise RuntimeError("Package '" + pkgname + "': Could not find aport, and" + " could not find this package in any APKINDEX!") + + +def depends_recurse(args, pkgname, arch): + """ Recursively resolve all of the package's dependencies. + :param pkgname: name of the package (e.g. "device-samsung-i9100") + :param arch: preferred architecture for binary packages + :returns: a list of pkgname_start and all its dependencies, e.g: + ["busybox-static-armhf", "device-samsung-i9100", + "linux-samsung-i9100", ...] """ + # Cached result + cache_key = "pmb.helpers.package.depends_recurse" + if (arch in args.cache[cache_key] and + pkgname in args.cache[cache_key][arch]): + return args.cache[cache_key][arch][pkgname] + + # Build ret (by iterating over the queue) + queue = [pkgname] + ret = [] + while len(queue): + pkgname_queue = queue.pop() + package = get(args, pkgname_queue, arch) + + # Add its depends to the queue + for depend in package["depends"]: + if depend not in ret: + queue += [depend] + if pkgname_queue not in ret: + ret += [pkgname_queue] + ret.sort() + + # Save to cache and return + if arch not in args.cache[cache_key]: + args.cache[cache_key][arch] = {} + args.cache[cache_key][arch][pkgname] = ret + return ret + + +def check_arch(args, pkgname, arch, binary=True): + """ Can a package be built for a certain architecture, or is there a binary + package for it? + + :param pkgname: name of the package + :param arch: architecture to check against + :param binary: set to False to only look at the pmaports, not at binary + packages + :returns: True when the package can be built, or there is a binary + package, False otherwise + """ + if binary: + arches = get(args, pkgname, arch)["arch"] + else: + arches = pmb.helpers.pmaports.get(args, pkgname)["arch"] + + if "!" + arch in arches: + return False + for value in [arch, "all", "noarch"]: + if value in arches: + return True + return False + + +def check_arch_recurse(args, pkgname, arch): + """ Recursively check if a package and its dependencies exist (binary repo) + or can be built (pmaports) for a certain architecture. + :param pkgname: name of the package + :param arch: architecture to check against + :returns: True when all the package's dependencies can be built or + exist for the arch in question + """ + for pkgname_i in depends_recurse(args, pkgname, arch): + if not check_arch(args, pkgname_i, arch): + if pkgname_i != pkgname: + logging.verbose(pkgname_i + ": (indirectly) depends on " + + pkgname) + logging.verbose(pkgname_i + ": can't be built for " + arch) + return False + return True diff --git a/pmb/helpers/pmaports.py b/pmb/helpers/pmaports.py index 8c37b722..cad90554 100644 --- a/pmb/helpers/pmaports.py +++ b/pmb/helpers/pmaports.py @@ -19,6 +19,12 @@ You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ +""" +Functions that work only on pmaports. See also: +- pmb/helpers/repo.py (only work on binary package repos) +- pmb/helpers/package.py (work on both) +""" + import glob import logging import os @@ -26,6 +32,15 @@ import os import pmb.parse +def get_list(args): + """ :returns: list of all pmaport pkgnames (["hello-world", ...]) """ + ret = [] + for apkbuild in glob.glob(args.aports + "/*/*/APKBUILD"): + ret.append(os.path.basename(os.path.dirname(apkbuild))) + ret.sort() + return ret + + def guess_main(args, subpkgname): """ Find the main package by assuming it is a prefix of the subpkgname. @@ -56,7 +71,8 @@ def guess_main(args, subpkgname): def find(args, package, must_exist=True): """ - Find the aport, that provides a certain subpackage. + Find the aport path, that provides a certain subpackage. + If you want the parsed APKBUILD instead, use pmb.helpers.pmaports.get(). :param must_exist: Raise an exception, when not found :returns: the full path to the aport folder @@ -101,3 +117,37 @@ def find(args, package, must_exist=True): # Save result in cache args.cache["find_aport"][package] = ret return ret + + +def get(args, pkgname, must_exist=True): + """ Find and parse an APKBUILD file. + Run 'pmbootstrap apkbuild_parse hello-world' for a full output example. + Relevant variables are defined in pmb.config.apkbuild_attributes. + + :param pkgname: the package name to find + :param must_exist: raise an exception when it can't be found + :returns: relevant variables from the APKBUILD as dictionary, e.g.: + { "pkgname": "hello-world", + "arch": ["all"], + "pkgrel": "4", + "pkgrel": "1", + "options": [], + ... } + """ + aport = find(args, pkgname, must_exist) + if aport: + return pmb.parse.apkbuild(args, aport + "/APKBUILD") + return None + + +def get_repo(args, pkgname, must_exist=True): + """ Get the repository folder of an aport. + + :pkgname: package name + :must_exist: raise an exception when it can't be found + :returns: a string like "main", "device", "cross", ... + or None when the aport could not be found """ + aport = find(args, pkgname, must_exist) + if not aport: + return None + return os.path.basename(os.path.dirname(aport)) diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 9965907e..573e51c1 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -16,6 +16,13 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ + +""" +Functions that work on both (binary package) repos. See also: +- pmb/helpers/pmaports.py (work on pmaports) +- pmb/helpers/package.py (work on both) +""" + import os import hashlib import logging diff --git a/pmb/helpers/repo_missing.py b/pmb/helpers/repo_missing.py new file mode 100644 index 00000000..a500527b --- /dev/null +++ b/pmb/helpers/repo_missing.py @@ -0,0 +1,156 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging + +import pmb.build +import pmb.helpers.package +import pmb.helpers.pmaports + + +def filter_missing_packages(args, arch, pkgnames): + """ Create a subset of pkgnames with missing or outdated binary packages. + + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) """ + ret = [] + for pkgname in pkgnames: + binary = pmb.parse.apkindex.package(args, pkgname, arch, False) + must_exist = False if binary else True + pmaport = pmb.helpers.pmaports.get(args, pkgname, must_exist) + if pmaport and pmb.build.is_necessary(args, arch, pmaport): + ret.append(pkgname) + return ret + + +def filter_aport_packages(args, arch, pkgnames): + """ Create a subset of pkgnames where each one has an aport. + + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) """ + ret = [] + for pkgname in pkgnames: + if pmb.helpers.pmaports.find(args, pkgname, False): + ret += [pkgname] + return ret + + +def filter_arch_packages(args, arch, pkgnames): + """ Create a subset of pkgnames with packages removed that can not be + built for a certain arch. The check is recursive, if one of the + dependencies of a package can not be built for the arch in question, + then it will not be in the final list either. + + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) """ + ret = [] + for pkgname in pkgnames: + if pmb.helpers.package.check_arch_recurse(args, pkgname, arch): + ret += [pkgname] + return ret + + +def get_relevant_packages(args, arch, pkgname=None, built=False): + """ Get all packages that can be built for the architecture in question. + + :param arch: architecture (e.g. "armhf") + :param pkgname: only look at a specific package (and its dependencies) + :param built: include packages that have already been built + :returns: an alphabetically sorted list of pkgnames, e.g.: + ["devicepkg-dev", "hello-world", "osk-sdl"] """ + if pkgname: + if not pmb.helpers.package.check_arch_recurse(args, pkgname, arch): + raise RuntimeError(pkgname + " can't be built for " + arch + "." + " Either itself or one if its dependencies is" + " not available for that architecture. Run with" + " -v for details.") + ret = pmb.helpers.package.depends_recurse(args, pkgname, arch) + else: + ret = pmb.helpers.pmaports.get_list(args) + ret = filter_arch_packages(args, arch, ret) + if built: + ret = filter_aport_packages(args, arch, ret) + if not len(ret): + logging.info("NOTE: no aport found for any package in the" + " dependency tree, it seems they are all provided by" + " upstream (Alpine).") + else: + ret = filter_missing_packages(args, arch, ret) + if not len(ret): + logging.info("NOTE: all relevant packages are up to date, use" + " --built to include the ones that have already been" + " built.") + + # Sort alphabetically (to get a deterministic build order) + ret.sort() + return ret + + +def generate_output_format(args, arch, pkgnames): + """ Generate the detailed output format. + :param arch: architecture + :param pkgnames: list of package names that should be in the output, + e.g.: ["hello-world", "pkg-depending-on-hello-world"] + :returns: a list like the following: + [{"pkgname": "hello-world", + "repo": "main", + "version": "1-r4", + "depends": []}, + {"pkgname": "pkg-depending-on-hello-world", + "version": "0.5-r0", + "repo": "main", + "depends": ["hello-world"]}] """ + ret = [] + for pkgname in pkgnames: + entry = pmb.helpers.package.get(args, pkgname, arch) + ret += [{"pkgname": entry["pkgname"], + "repo": pmb.helpers.pmaports.get_repo(args, pkgname), + "version": entry["version"], + "depends": entry["depends"]}] + return ret + + +def generate(args, arch, overview, pkgname=None, built=False): + """ Get packages that need to be built, with all their dependencies. + + :param arch: architecture (e.g. "armhf") + :param pkgname: only look at a specific package + :param built: include packages that have already been built + :returns: a list like the following: + [{"pkgname": "hello-world", + "repo": "main", + "version": "1-r4"}, + {"pkgname": "package-depending-on-hello-world", + "version": "0.5-r0", + "repo": "main"}] + """ + # Log message + packages_str = pkgname if pkgname else "all packages" + logging.info("Calculate packages that need to be built ({}, {}) - this may" + " take some time".format(packages_str, arch)) + + # Order relevant packages + ret = get_relevant_packages(args, arch, pkgname, built) + + # Output format + if overview: + return ret + return generate_output_format(args, arch, ret) diff --git a/pmb/parse/_apkbuild.py b/pmb/parse/_apkbuild.py index 9af535fc..0610fc8c 100644 --- a/pmb/parse/_apkbuild.py +++ b/pmb/parse/_apkbuild.py @@ -89,6 +89,7 @@ def apkbuild(args, path, check_pkgver=True, check_pkgname=True): to be perfect and catch every edge case (for that, a full shell parser would be necessary!). Instead, it should just work with the use-cases covered by pmbootstrap and not take too long. + Run 'pmbootstrap apkbuild_parse hello-world' for a full output example. :param path: full path to the APKBUILD :param check_pkgver: verify that the pkgver is valid. diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 0ed8a575..4ef87505 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -18,8 +18,6 @@ along with pmbootstrap. If not, see . """ import argparse import copy -import glob -import os try: import argcomplete @@ -29,6 +27,7 @@ except ImportError: import pmb.config import pmb.parse.arch import pmb.helpers.args +import pmb.helpers.pmaports """ This file is about parsing command line arguments passed to pmbootstrap, as well as generating the help pages (pmbootstrap -h). All this is done with @@ -235,12 +234,25 @@ def arguments_kconfig(subparser): edit.add_argument("package") +def arguments_repo_missing(subparser): + ret = subparser.add_parser("repo_missing") + package = ret.add_argument("package", nargs="?", help="only look at a" + " specific package and its dependencies") + if argcomplete: + package.completer = packagecompleter + ret.add_argument("--arch", choices=pmb.config.build_device_architectures, + default=pmb.parse.arch.alpine_native()) + ret.add_argument("--built", action="store_true", + help="include packages which exist in the binary repos") + ret.add_argument("--overview", action="store_true", + help="only print the pkgnames without any details") + return ret + + def packagecompleter(prefix, action, parser, parsed_args): args = parsed_args pmb.config.merge_with_args(args) - packages = set() - for apkbuild in glob.glob(args.aports + "/*/" + prefix + "*/APKBUILD"): - packages.add(os.path.basename(os.path.dirname(apkbuild))) + packages = set(pmb.helpers.pmaports.get_list(args)) return packages @@ -315,6 +327,7 @@ def arguments(): sub.add_parser("work_migrate", help="run this before using pmbootstrap" " non-interactively to migrate the" " work folder version on demand") + arguments_repo_missing(sub) arguments_kconfig(sub) arguments_export(sub) arguments_flasher(sub) diff --git a/test/test_build_package.py b/test/test_build_package.py index 1bd2c48d..91ee8fb8 100644 --- a/test/test_build_package.py +++ b/test/test_build_package.py @@ -102,22 +102,27 @@ def test_get_apkbuild(args): assert "Could not find" in str(e.value) -def test_check_arch(args): - func = pmb.build._package.check_arch - apkbuild = {"pkgname": "test"} +def test_check_arch_abort(monkeypatch, args): + # Fake APKBUILD data + apkbuild = {"pkgname": "testpkgname"} + + def fake_helpers_pmaports_get(args, pkgname): + return apkbuild + monkeypatch.setattr(pmb.helpers.pmaports, "get", fake_helpers_pmaports_get) # Arch is right + func = pmb.build._package.check_arch_abort apkbuild["arch"] = ["armhf"] - func(args, apkbuild, "armhf") + func(args, "testpkgname", "armhf") apkbuild["arch"] = ["noarch"] - func(args, apkbuild, "armhf") + func(args, "testpkgname", "armhf") apkbuild["arch"] = ["all"] - func(args, apkbuild, "armhf") + func(args, "testpkgname", "armhf") # Arch is wrong apkbuild["arch"] = ["x86_64"] with pytest.raises(RuntimeError) as e: - func(args, apkbuild, "armhf") + func(args, "testpkgname", "armhf") assert "Can't build" in str(e.value) diff --git a/test/test_helpers_package.py b/test/test_helpers_package.py new file mode 100644 index 00000000..d8195570 --- /dev/null +++ b/test/test_helpers_package.py @@ -0,0 +1,185 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import sys +import pytest + +# Import from parent directory +sys.path.insert(0, os.path.realpath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.helpers.logging +import pmb.helpers.package + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap", "init"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_helpers_package_get_pmaports_and_cache(args, monkeypatch): + """ Test pmb.helpers.package.get(): find in pmaports, use cached result """ + + # Fake APKBUILD data + def stub(args, pkgname, must_exist): + return {"arch": ["armhf"], + "depends": ["testdepend"], + "pkgname": "testpkgname", + "provides": ["testprovide"], + "options": [], + "checkdepends": [], + "subpackages": [], + "makedepends": [], + "pkgver": "1.0", + "pkgrel": "1"} + monkeypatch.setattr(pmb.helpers.pmaports, "get", stub) + + package = {"arch": ["armhf"], + "depends": ["testdepend"], + "pkgname": "testpkgname", + "provides": ["testprovide"], + "version": "1.0-r1"} + func = pmb.helpers.package.get + assert func(args, "testpkgname", "armhf") == package + + # Cached result + monkeypatch.delattr(pmb.helpers.pmaports, "get") + assert func(args, "testpkgname", "armhf") == package + + +def test_helpers_package_get_apkindex(args, monkeypatch): + """ Test pmb.helpers.package.get(): find in apkindex """ + + # Fake APKINDEX data + fake_apkindex_data = {"arch": "armhf", + "depends": ["testdepend"], + "pkgname": "testpkgname", + "provides": ["testprovide"], + "version": "1.0-r1"} + + def stub(args, pkgname, arch, must_exist): + if arch != fake_apkindex_data["arch"]: + return None + return fake_apkindex_data + monkeypatch.setattr(pmb.parse.apkindex, "package", stub) + + # Given arch + package = {"arch": ["armhf"], + "depends": ["testdepend"], + "pkgname": "testpkgname", + "provides": ["testprovide"], + "version": "1.0-r1"} + func = pmb.helpers.package.get + assert func(args, "testpkgname", "armhf") == package + + # Other arch + assert func(args, "testpkgname", "x86_64") == package + + +def test_helpers_package_depends_recurse(args): + """ Test pmb.helpers.package.depends_recurse() """ + + # Put fake data into the pmb.helpers.package.get() cache + cache = {"a": {"depends": ["b", "c"]}, + "b": {"depends": []}, + "c": {"depends": ["d"]}, + "d": {"depends": ["b"]}} + args.cache["pmb.helpers.package.get"]["armhf"] = cache + + # Normal runs + func = pmb.helpers.package.depends_recurse + assert func(args, "a", "armhf") == ["a", "b", "c", "d"] + assert func(args, "d", "armhf") == ["b", "d"] + + # Cached result + args.cache["pmb.helpers.package.get"]["armhf"] = {} + assert func(args, "d", "armhf") == ["b", "d"] + + +def test_helpers_package_check_arch_package(args): + """ Test pmb.helpers.package.check_arch(): binary = True """ + # Put fake data into the pmb.helpers.package.get() cache + func = pmb.helpers.package.check_arch + cache = {"a": {"arch": []}} + args.cache["pmb.helpers.package.get"]["armhf"] = cache + + cache["a"]["arch"] = ["all !armhf"] + assert func(args, "a", "armhf") is False + + cache["a"]["arch"] = ["all"] + assert func(args, "a", "armhf") is True + + cache["a"]["arch"] = ["noarch"] + assert func(args, "a", "armhf") is True + + cache["a"]["arch"] = ["armhf"] + assert func(args, "a", "armhf") is True + + cache["a"]["arch"] = ["aarch64"] + assert func(args, "a", "armhf") is False + + +def test_helpers_package_check_arch_pmaports(args, monkeypatch): + """ Test pmb.helpers.package.check_arch(): binary = False """ + func = pmb.helpers.package.check_arch + fake_pmaport = {"arch": []} + + def fake_pmaports_get(args, pkgname, must_exist=False): + return fake_pmaport + monkeypatch.setattr(pmb.helpers.pmaports, "get", fake_pmaports_get) + + fake_pmaport["arch"] = ["armhf"] + assert func(args, "a", "armhf", False) is True + + fake_pmaport["arch"] = ["all", "!armhf"] + assert func(args, "a", "armhf", False) is False + + +def test_helpers_package_check_arch_recurse(args, monkeypatch): + """ Test pmb.helpers.package.check_arch_recurse() """ + # Test data + func = pmb.helpers.package.check_arch_recurse + depends = ["a", "b", "c"] + arch_check_results = {} + + def fake_depends_recurse(args, pkgname, arch): + return depends + monkeypatch.setattr(pmb.helpers.package, "depends_recurse", + fake_depends_recurse) + + def fake_check_arch(args, pkgname, arch): + return arch_check_results[pkgname] + monkeypatch.setattr(pmb.helpers.package, "check_arch", fake_check_arch) + + # Result: True + arch_check_results = {"a": True, + "b": True, + "c": True} + assert func(args, "a", "armhf") is True + + # Result: False + arch_check_results = {"a": True, + "b": False, + "c": True} + assert func(args, "a", "armhf") is False diff --git a/test/test_helpers_repo_missing.py b/test/test_helpers_repo_missing.py new file mode 100644 index 00000000..c9a87b8e --- /dev/null +++ b/test/test_helpers_repo_missing.py @@ -0,0 +1,158 @@ +""" +Copyright 2018 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + +import os +import pytest +import sys + +# Import from parent directory +sys.path.insert(0, os.path.realpath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.build.other + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap", "init"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def test_filter_missing_packages_invalid(args): + """ Test ...repo_missing.filter_missing_packages(): invalid package """ + func = pmb.helpers.repo_missing.filter_missing_packages + with pytest.raises(RuntimeError) as e: + func(args, "armhf", ["invalid-package-name"]) + assert str(e.value).startswith("Could not find aport") + + +def test_filter_missing_packages_binary_exists(args): + """ Test ...repo_missing.filter_missing_packages(): binary exists """ + func = pmb.helpers.repo_missing.filter_missing_packages + assert func(args, "armhf", ["busybox"]) == [] + + +def test_filter_missing_packages_pmaports(args, monkeypatch): + """ Test ...repo_missing.filter_missing_packages(): pmaports """ + build_is_necessary = None + func = pmb.helpers.repo_missing.filter_missing_packages + + def stub(args, arch, pmaport): + return build_is_necessary + monkeypatch.setattr(pmb.build, "is_necessary", stub) + + build_is_necessary = True + assert func(args, "x86_64", ["busybox", "hello-world"]) == ["hello-world"] + + build_is_necessary = False + assert func(args, "x86_64", ["busybox", "hello-world"]) == [] + + +def test_filter_aport_packages(args): + """ Test ...repo_missing.filter_aport_packages() """ + func = pmb.helpers.repo_missing.filter_aport_packages + assert func(args, "armhf", ["busybox", "hello-world"]) == ["hello-world"] + + +def test_filter_arch_packages(args, monkeypatch): + """ Test ...repo_missing.filter_arch_packages() """ + func = pmb.helpers.repo_missing.filter_arch_packages + check_arch = None + + def stub(args, arch, pmaport): + return check_arch + monkeypatch.setattr(pmb.helpers.package, "check_arch_recurse", stub) + + check_arch = False + assert func(args, "armhf", ["hello-world"]) == [] + + check_arch = True + assert func(args, "armhf", []) == [] + + +def test_get_relevant_packages(args, monkeypatch): + """ Test ...repo_missing.get_relevant_packages() """ + + # Set up fake return values + stub_data = {"check_arch_recurse": False, + "depends_recurse": ["a", "b", "c", "d"], + "filter_arch_packages": ["a", "b", "c"], + "filter_aport_packages": ["b", "a"], + "filter_missing_packages": ["a"]} + + def stub(args, arch, pmaport): + return stub_data["check_arch_recurse"] + monkeypatch.setattr(pmb.helpers.package, "check_arch_recurse", stub) + + def stub(args, arch, pmaport): + return stub_data["depends_recurse"] + monkeypatch.setattr(pmb.helpers.package, "depends_recurse", stub) + + def stub(args, arch, pmaport): + return stub_data["filter_arch_packages"] + monkeypatch.setattr(pmb.helpers.repo_missing, "filter_arch_packages", stub) + + def stub(args, arch, pmaport): + return stub_data["filter_aport_packages"] + monkeypatch.setattr(pmb.helpers.repo_missing, "filter_aport_packages", + stub) + + def stub(args, arch, pmaport): + return stub_data["filter_missing_packages"] + monkeypatch.setattr(pmb.helpers.repo_missing, "filter_missing_packages", + stub) + + # No given package + func = pmb.helpers.repo_missing.get_relevant_packages + assert func(args, "armhf") == ["a"] + assert func(args, "armhf", built=True) == ["a", "b"] + + # Package can't be built for given arch + with pytest.raises(RuntimeError) as e: + func(args, "armhf", "a") + assert "can't be built" in str(e.value) + + # Package can be built for given arch + stub_data["check_arch_recurse"] = True + assert func(args, "armhf", "a") == ["a"] + assert func(args, "armhf", "a", True) == ["a", "b"] + + +def test_generate_output_format(args, monkeypatch): + """ Test ...repo_missing.generate_output_format() """ + + def stub(args, pkgname, arch): + return {"pkgname": "hello-world", "version": "1.0-r0", + "depends": ["depend1", "depend2"]} + monkeypatch.setattr(pmb.helpers.package, "get", stub) + + def stub(args, pkgname): + return "main" + monkeypatch.setattr(pmb.helpers.pmaports, "get_repo", stub) + + func = pmb.helpers.repo_missing.generate_output_format + ret = [{"pkgname": "hello-world", + "repo": "main", + "version": "1.0-r0", + "depends": ["depend1", "depend2"]}] + assert func(args, "armhf", ["hello-world"]) == ret