new action: 'pmbootstrap repo_missing'
Add a new action that lists all aports, for which no binary packages exist. Only list packages that can be built for the relevant arch (specified with --arch). This works recursively: when a package can be built for a certain arch, but one of its dependencies (or their depends) can not be built for that arch, then don't list it. This action will be used for the new sr.ht based build infrastructure, to figure out which packages need to be built ahead of time (so we can trigger each of them as single build job). Determining the order of the packages to be built is not determined with pmbootstrap, the serverside code of build.postmarketos.org takes care of that. For testing purposes, a single package can also be specified and the action will list if it can be built for that arch with its dependencies, and what needs to be built exactly. Add pmb/helpers/package.py to hold functions that work on both pmaports and (binary package) repos - in contrary to the existing pmb/helpers/pmaports.py (see previous commit) and pmb/helpers/repo.py, which only work with one of those. Refactoring: * pmb/helpers/pmaports.py: add a get_list() function, which lists all aports and use it instead of writing the same glob loop over and over * add pmb.helpers.pmaports.get(), which finds an APKBUILD and parses it in one step. * rename pmb.build._package.check_arch to ...check_arch_abort to distinguish it from the other check_arch function
This commit is contained in:
Oliver Smith
Martijn Braam
parent
a44b80b31d
commit
933c4d0f0d
|
|
@ -129,6 +129,11 @@ $ pmbootstrap flasher --method=adb sideload
|
|||
```
|
||||
|
||||
### Repository Maintenance
|
||||
List pmaports that don't have a binary package:
|
||||
```
|
||||
$ pmbootstrap repo_missing --arch=armhf --overview
|
||||
```
|
||||
|
||||
Increase the `pkgrel` for each aport where the binary package has outdated dependencies (e.g. after soname bumps):
|
||||
```
|
||||
$ pmbootstrap pkgrel_bump --auto
|
||||
|
|
|
|||
|
|
@ -50,35 +50,33 @@ def skip_already_built(args, pkgname, arch):
|
|||
|
||||
def get_apkbuild(args, pkgname, arch):
|
||||
"""
|
||||
Find the APKBUILD path for pkgname. When there is none, try to find it in
|
||||
Parse the APKBUILD path for pkgname. When there is none, try to find it in
|
||||
the binary package APKINDEX files or raise an exception.
|
||||
|
||||
:param pkgname: package name to be built, as specified in the APKBUILD
|
||||
:returns: None or full path to APKBUILD
|
||||
:returns: None or parsed APKBUILD
|
||||
"""
|
||||
# Get existing binary package indexes
|
||||
pmb.helpers.repo.update(args, arch)
|
||||
|
||||
# Get aport, skip upstream only packages
|
||||
aport = pmb.helpers.pmaports.find(args, pkgname, False)
|
||||
if aport:
|
||||
return pmb.parse.apkbuild(args, aport + "/APKBUILD")
|
||||
# Get pmaport, skip upstream only packages
|
||||
pmaport = pmb.helpers.pmaports.get(args, pkgname, False)
|
||||
if pmaport:
|
||||
return pmaport
|
||||
if pmb.parse.apkindex.providers(args, pkgname, arch, False):
|
||||
return None
|
||||
raise RuntimeError("Package '" + pkgname + "': Could not find aport, and"
|
||||
" could not find this package in any APKINDEX!")
|
||||
|
||||
|
||||
def check_arch(args, apkbuild, arch):
|
||||
def check_arch_abort(args, pkgname, arch):
|
||||
"""
|
||||
Check if the APKBUILD can be built for a specific architecture and abort
|
||||
with a helpful message if it is not the case.
|
||||
"""
|
||||
for value in [arch, "all", "noarch"]:
|
||||
if value in apkbuild["arch"]:
|
||||
return
|
||||
if pmb.helpers.package.check_arch(args, pkgname, arch, False):
|
||||
return
|
||||
|
||||
pkgname = apkbuild["pkgname"]
|
||||
logging.info("NOTE: You can edit the 'arch=' line inside the APKBUILD")
|
||||
if args.action == "build":
|
||||
logging.info("NOTE: Alternatively, use --arch to build for another"
|
||||
|
|
@ -435,7 +433,7 @@ def package(args, pkgname, arch=None, force=False, strict=False,
|
|||
return
|
||||
|
||||
# Detect the build environment (skip unnecessary builds)
|
||||
check_arch(args, apkbuild, arch)
|
||||
check_arch_abort(args, pkgname, arch)
|
||||
suffix = pmb.build.autodetect.suffix(args, apkbuild, arch)
|
||||
cross = pmb.build.autodetect.crosscompile(args, apkbuild, arch, suffix)
|
||||
if not init_buildenv(args, apkbuild, arch, strict, force, cross, suffix,
|
||||
|
|
|
|||
|
|
@ -116,7 +116,9 @@ def add_cache(args):
|
|||
"apk_repository_list_updated": [],
|
||||
"built": {},
|
||||
"find_aport": {},
|
||||
"offline_msg_shown": False})
|
||||
"offline_msg_shown": False,
|
||||
"pmb.helpers.package.depends_recurse": {},
|
||||
"pmb.helpers.package.get": {}})
|
||||
|
||||
|
||||
def add_deviceinfo(args):
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ import pmb.helpers.logging
|
|||
import pmb.helpers.pkgrel_bump
|
||||
import pmb.helpers.pmaports
|
||||
import pmb.helpers.repo
|
||||
import pmb.helpers.repo_missing
|
||||
import pmb.helpers.run
|
||||
import pmb.install
|
||||
import pmb.parse
|
||||
|
|
@ -161,6 +162,12 @@ def config(args):
|
|||
pmb.helpers.logging.disable()
|
||||
|
||||
|
||||
def repo_missing(args):
|
||||
missing = pmb.helpers.repo_missing.generate(args, args.arch, args.overview,
|
||||
args.package, args.built)
|
||||
print(json.dumps(missing, indent=4))
|
||||
|
||||
|
||||
def index(args):
|
||||
pmb.build.index_repo(args)
|
||||
|
||||
|
|
@ -259,11 +266,9 @@ def apkbuild_parse(args):
|
|||
# Default to all packages
|
||||
packages = args.packages
|
||||
if not packages:
|
||||
for apkbuild in glob.glob(args.aports + "/*/*/APKBUILD"):
|
||||
packages.append(os.path.basename(os.path.dirname(apkbuild)))
|
||||
packages = pmb.helpers.pmaports.get_list(args)
|
||||
|
||||
# Iterate over all packages
|
||||
packages.sort()
|
||||
for package in packages:
|
||||
print(package + ":")
|
||||
aport = pmb.helpers.pmaports.find(args, package)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,173 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
"""
|
||||
Functions that work on both pmaports and (binary package) repos. See also:
|
||||
- pmb/helpers/pmaports.py (work on pmaports)
|
||||
- pmb/helpers/repo.py (work on binary package repos)
|
||||
"""
|
||||
|
||||
import logging
|
||||
import copy
|
||||
|
||||
import pmb.helpers.pmaports
|
||||
|
||||
|
||||
def get(args, pkgname, arch):
|
||||
""" Find a package in pmaports, and as fallback in the APKINDEXes of the
|
||||
binary packages.
|
||||
:param pkgname: package name (e.g. "hello-world")
|
||||
:param arch: preferred architecture of the binary package. When it
|
||||
can't be found for this arch, we'll still look for another
|
||||
arch to see whether the package exists at all. So make
|
||||
sure to check the returned arch against what you wanted
|
||||
with check_arch(). Example: "armhf"
|
||||
:returns: data from the parsed APKBUILD or APKINDEX in the following
|
||||
format: {"arch": ["noarch"],
|
||||
"depends": ["busybox-extras", "lddtree", ...],
|
||||
"pkgname": "postmarketos-mkinitfs",
|
||||
"provides": ["mkinitfs=0..1"],
|
||||
"version": "0.0.4-r10"} """
|
||||
# Cached result
|
||||
cache_key = "pmb.helpers.package.get"
|
||||
if (arch in args.cache[cache_key] and
|
||||
pkgname in args.cache[cache_key][arch]):
|
||||
return args.cache[cache_key][arch][pkgname]
|
||||
|
||||
# Find in pmaports
|
||||
ret = None
|
||||
pmaport = pmb.helpers.pmaports.get(args, pkgname, False)
|
||||
if pmaport:
|
||||
ret = {"arch": pmaport["arch"],
|
||||
"depends": pmb.build._package.get_depends(args, pmaport),
|
||||
"pkgname": pkgname,
|
||||
"provides": pmaport["provides"],
|
||||
"version": pmaport["pkgver"] + "-r" + pmaport["pkgrel"]}
|
||||
|
||||
# Find in APKINDEX (given arch)
|
||||
if not ret:
|
||||
pmb.helpers.repo.update(args, arch)
|
||||
ret = pmb.parse.apkindex.package(args, pkgname, arch, False)
|
||||
|
||||
# Find in APKINDEX (other arches)
|
||||
if not ret:
|
||||
for arch_i in pmb.config.build_device_architectures:
|
||||
if arch_i != arch:
|
||||
ret = pmb.parse.apkindex.package(args, pkgname, arch_i, False)
|
||||
if ret:
|
||||
break
|
||||
|
||||
# Copy ret (it might have references to caches of the APKINDEX or APKBUILDs
|
||||
# and we don't want to modify those!)
|
||||
if ret:
|
||||
ret = copy.deepcopy(ret)
|
||||
|
||||
# Make sure ret["arch"] is a list (APKINDEX code puts a string there)
|
||||
if ret and isinstance(ret["arch"], str):
|
||||
ret["arch"] = [ret["arch"]]
|
||||
|
||||
# Save to cache and return
|
||||
if ret:
|
||||
if arch not in args.cache[cache_key]:
|
||||
args.cache[cache_key][arch] = {}
|
||||
args.cache[cache_key][arch][pkgname] = ret
|
||||
return ret
|
||||
|
||||
# Could not find the package
|
||||
raise RuntimeError("Package '" + pkgname + "': Could not find aport, and"
|
||||
" could not find this package in any APKINDEX!")
|
||||
|
||||
|
||||
def depends_recurse(args, pkgname, arch):
|
||||
""" Recursively resolve all of the package's dependencies.
|
||||
:param pkgname: name of the package (e.g. "device-samsung-i9100")
|
||||
:param arch: preferred architecture for binary packages
|
||||
:returns: a list of pkgname_start and all its dependencies, e.g:
|
||||
["busybox-static-armhf", "device-samsung-i9100",
|
||||
"linux-samsung-i9100", ...] """
|
||||
# Cached result
|
||||
cache_key = "pmb.helpers.package.depends_recurse"
|
||||
if (arch in args.cache[cache_key] and
|
||||
pkgname in args.cache[cache_key][arch]):
|
||||
return args.cache[cache_key][arch][pkgname]
|
||||
|
||||
# Build ret (by iterating over the queue)
|
||||
queue = [pkgname]
|
||||
ret = []
|
||||
while len(queue):
|
||||
pkgname_queue = queue.pop()
|
||||
package = get(args, pkgname_queue, arch)
|
||||
|
||||
# Add its depends to the queue
|
||||
for depend in package["depends"]:
|
||||
if depend not in ret:
|
||||
queue += [depend]
|
||||
if pkgname_queue not in ret:
|
||||
ret += [pkgname_queue]
|
||||
ret.sort()
|
||||
|
||||
# Save to cache and return
|
||||
if arch not in args.cache[cache_key]:
|
||||
args.cache[cache_key][arch] = {}
|
||||
args.cache[cache_key][arch][pkgname] = ret
|
||||
return ret
|
||||
|
||||
|
||||
def check_arch(args, pkgname, arch, binary=True):
|
||||
""" Can a package be built for a certain architecture, or is there a binary
|
||||
package for it?
|
||||
|
||||
:param pkgname: name of the package
|
||||
:param arch: architecture to check against
|
||||
:param binary: set to False to only look at the pmaports, not at binary
|
||||
packages
|
||||
:returns: True when the package can be built, or there is a binary
|
||||
package, False otherwise
|
||||
"""
|
||||
if binary:
|
||||
arches = get(args, pkgname, arch)["arch"]
|
||||
else:
|
||||
arches = pmb.helpers.pmaports.get(args, pkgname)["arch"]
|
||||
|
||||
if "!" + arch in arches:
|
||||
return False
|
||||
for value in [arch, "all", "noarch"]:
|
||||
if value in arches:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def check_arch_recurse(args, pkgname, arch):
|
||||
""" Recursively check if a package and its dependencies exist (binary repo)
|
||||
or can be built (pmaports) for a certain architecture.
|
||||
:param pkgname: name of the package
|
||||
:param arch: architecture to check against
|
||||
:returns: True when all the package's dependencies can be built or
|
||||
exist for the arch in question
|
||||
"""
|
||||
for pkgname_i in depends_recurse(args, pkgname, arch):
|
||||
if not check_arch(args, pkgname_i, arch):
|
||||
if pkgname_i != pkgname:
|
||||
logging.verbose(pkgname_i + ": (indirectly) depends on " +
|
||||
pkgname)
|
||||
logging.verbose(pkgname_i + ": can't be built for " + arch)
|
||||
return False
|
||||
return True
|
||||
|
|
@ -19,6 +19,12 @@ You should have received a copy of the GNU General Public License
|
|||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
"""
|
||||
Functions that work only on pmaports. See also:
|
||||
- pmb/helpers/repo.py (only work on binary package repos)
|
||||
- pmb/helpers/package.py (work on both)
|
||||
"""
|
||||
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -26,6 +32,15 @@ import os
|
|||
import pmb.parse
|
||||
|
||||
|
||||
def get_list(args):
|
||||
""" :returns: list of all pmaport pkgnames (["hello-world", ...]) """
|
||||
ret = []
|
||||
for apkbuild in glob.glob(args.aports + "/*/*/APKBUILD"):
|
||||
ret.append(os.path.basename(os.path.dirname(apkbuild)))
|
||||
ret.sort()
|
||||
return ret
|
||||
|
||||
|
||||
def guess_main(args, subpkgname):
|
||||
"""
|
||||
Find the main package by assuming it is a prefix of the subpkgname.
|
||||
|
|
@ -56,7 +71,8 @@ def guess_main(args, subpkgname):
|
|||
|
||||
def find(args, package, must_exist=True):
|
||||
"""
|
||||
Find the aport, that provides a certain subpackage.
|
||||
Find the aport path, that provides a certain subpackage.
|
||||
If you want the parsed APKBUILD instead, use pmb.helpers.pmaports.get().
|
||||
|
||||
:param must_exist: Raise an exception, when not found
|
||||
:returns: the full path to the aport folder
|
||||
|
|
@ -101,3 +117,37 @@ def find(args, package, must_exist=True):
|
|||
# Save result in cache
|
||||
args.cache["find_aport"][package] = ret
|
||||
return ret
|
||||
|
||||
|
||||
def get(args, pkgname, must_exist=True):
|
||||
""" Find and parse an APKBUILD file.
|
||||
Run 'pmbootstrap apkbuild_parse hello-world' for a full output example.
|
||||
Relevant variables are defined in pmb.config.apkbuild_attributes.
|
||||
|
||||
:param pkgname: the package name to find
|
||||
:param must_exist: raise an exception when it can't be found
|
||||
:returns: relevant variables from the APKBUILD as dictionary, e.g.:
|
||||
{ "pkgname": "hello-world",
|
||||
"arch": ["all"],
|
||||
"pkgrel": "4",
|
||||
"pkgrel": "1",
|
||||
"options": [],
|
||||
... }
|
||||
"""
|
||||
aport = find(args, pkgname, must_exist)
|
||||
if aport:
|
||||
return pmb.parse.apkbuild(args, aport + "/APKBUILD")
|
||||
return None
|
||||
|
||||
|
||||
def get_repo(args, pkgname, must_exist=True):
|
||||
""" Get the repository folder of an aport.
|
||||
|
||||
:pkgname: package name
|
||||
:must_exist: raise an exception when it can't be found
|
||||
:returns: a string like "main", "device", "cross", ...
|
||||
or None when the aport could not be found """
|
||||
aport = find(args, pkgname, must_exist)
|
||||
if not aport:
|
||||
return None
|
||||
return os.path.basename(os.path.dirname(aport))
|
||||
|
|
|
|||
|
|
@ -16,6 +16,13 @@ GNU General Public License for more details.
|
|||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
"""
|
||||
Functions that work on both (binary package) repos. See also:
|
||||
- pmb/helpers/pmaports.py (work on pmaports)
|
||||
- pmb/helpers/package.py (work on both)
|
||||
"""
|
||||
|
||||
import os
|
||||
import hashlib
|
||||
import logging
|
||||
|
|
|
|||
|
|
@ -0,0 +1,156 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import logging
|
||||
|
||||
import pmb.build
|
||||
import pmb.helpers.package
|
||||
import pmb.helpers.pmaports
|
||||
|
||||
|
||||
def filter_missing_packages(args, arch, pkgnames):
|
||||
""" Create a subset of pkgnames with missing or outdated binary packages.
|
||||
|
||||
:param arch: architecture (e.g. "armhf")
|
||||
:param pkgnames: list of package names (e.g. ["hello-world", "test12"])
|
||||
:returns: subset of pkgnames (e.g. ["hello-world"]) """
|
||||
ret = []
|
||||
for pkgname in pkgnames:
|
||||
binary = pmb.parse.apkindex.package(args, pkgname, arch, False)
|
||||
must_exist = False if binary else True
|
||||
pmaport = pmb.helpers.pmaports.get(args, pkgname, must_exist)
|
||||
if pmaport and pmb.build.is_necessary(args, arch, pmaport):
|
||||
ret.append(pkgname)
|
||||
return ret
|
||||
|
||||
|
||||
def filter_aport_packages(args, arch, pkgnames):
|
||||
""" Create a subset of pkgnames where each one has an aport.
|
||||
|
||||
:param arch: architecture (e.g. "armhf")
|
||||
:param pkgnames: list of package names (e.g. ["hello-world", "test12"])
|
||||
:returns: subset of pkgnames (e.g. ["hello-world"]) """
|
||||
ret = []
|
||||
for pkgname in pkgnames:
|
||||
if pmb.helpers.pmaports.find(args, pkgname, False):
|
||||
ret += [pkgname]
|
||||
return ret
|
||||
|
||||
|
||||
def filter_arch_packages(args, arch, pkgnames):
|
||||
""" Create a subset of pkgnames with packages removed that can not be
|
||||
built for a certain arch. The check is recursive, if one of the
|
||||
dependencies of a package can not be built for the arch in question,
|
||||
then it will not be in the final list either.
|
||||
|
||||
:param arch: architecture (e.g. "armhf")
|
||||
:param pkgnames: list of package names (e.g. ["hello-world", "test12"])
|
||||
:returns: subset of pkgnames (e.g. ["hello-world"]) """
|
||||
ret = []
|
||||
for pkgname in pkgnames:
|
||||
if pmb.helpers.package.check_arch_recurse(args, pkgname, arch):
|
||||
ret += [pkgname]
|
||||
return ret
|
||||
|
||||
|
||||
def get_relevant_packages(args, arch, pkgname=None, built=False):
|
||||
""" Get all packages that can be built for the architecture in question.
|
||||
|
||||
:param arch: architecture (e.g. "armhf")
|
||||
:param pkgname: only look at a specific package (and its dependencies)
|
||||
:param built: include packages that have already been built
|
||||
:returns: an alphabetically sorted list of pkgnames, e.g.:
|
||||
["devicepkg-dev", "hello-world", "osk-sdl"] """
|
||||
if pkgname:
|
||||
if not pmb.helpers.package.check_arch_recurse(args, pkgname, arch):
|
||||
raise RuntimeError(pkgname + " can't be built for " + arch + "."
|
||||
" Either itself or one if its dependencies is"
|
||||
" not available for that architecture. Run with"
|
||||
" -v for details.")
|
||||
ret = pmb.helpers.package.depends_recurse(args, pkgname, arch)
|
||||
else:
|
||||
ret = pmb.helpers.pmaports.get_list(args)
|
||||
ret = filter_arch_packages(args, arch, ret)
|
||||
if built:
|
||||
ret = filter_aport_packages(args, arch, ret)
|
||||
if not len(ret):
|
||||
logging.info("NOTE: no aport found for any package in the"
|
||||
" dependency tree, it seems they are all provided by"
|
||||
" upstream (Alpine).")
|
||||
else:
|
||||
ret = filter_missing_packages(args, arch, ret)
|
||||
if not len(ret):
|
||||
logging.info("NOTE: all relevant packages are up to date, use"
|
||||
" --built to include the ones that have already been"
|
||||
" built.")
|
||||
|
||||
# Sort alphabetically (to get a deterministic build order)
|
||||
ret.sort()
|
||||
return ret
|
||||
|
||||
|
||||
def generate_output_format(args, arch, pkgnames):
|
||||
""" Generate the detailed output format.
|
||||
:param arch: architecture
|
||||
:param pkgnames: list of package names that should be in the output,
|
||||
e.g.: ["hello-world", "pkg-depending-on-hello-world"]
|
||||
:returns: a list like the following:
|
||||
[{"pkgname": "hello-world",
|
||||
"repo": "main",
|
||||
"version": "1-r4",
|
||||
"depends": []},
|
||||
{"pkgname": "pkg-depending-on-hello-world",
|
||||
"version": "0.5-r0",
|
||||
"repo": "main",
|
||||
"depends": ["hello-world"]}] """
|
||||
ret = []
|
||||
for pkgname in pkgnames:
|
||||
entry = pmb.helpers.package.get(args, pkgname, arch)
|
||||
ret += [{"pkgname": entry["pkgname"],
|
||||
"repo": pmb.helpers.pmaports.get_repo(args, pkgname),
|
||||
"version": entry["version"],
|
||||
"depends": entry["depends"]}]
|
||||
return ret
|
||||
|
||||
|
||||
def generate(args, arch, overview, pkgname=None, built=False):
|
||||
""" Get packages that need to be built, with all their dependencies.
|
||||
|
||||
:param arch: architecture (e.g. "armhf")
|
||||
:param pkgname: only look at a specific package
|
||||
:param built: include packages that have already been built
|
||||
:returns: a list like the following:
|
||||
[{"pkgname": "hello-world",
|
||||
"repo": "main",
|
||||
"version": "1-r4"},
|
||||
{"pkgname": "package-depending-on-hello-world",
|
||||
"version": "0.5-r0",
|
||||
"repo": "main"}]
|
||||
"""
|
||||
# Log message
|
||||
packages_str = pkgname if pkgname else "all packages"
|
||||
logging.info("Calculate packages that need to be built ({}, {}) - this may"
|
||||
" take some time".format(packages_str, arch))
|
||||
|
||||
# Order relevant packages
|
||||
ret = get_relevant_packages(args, arch, pkgname, built)
|
||||
|
||||
# Output format
|
||||
if overview:
|
||||
return ret
|
||||
return generate_output_format(args, arch, ret)
|
||||
|
|
@ -89,6 +89,7 @@ def apkbuild(args, path, check_pkgver=True, check_pkgname=True):
|
|||
to be perfect and catch every edge case (for that, a full shell parser
|
||||
would be necessary!). Instead, it should just work with the use-cases
|
||||
covered by pmbootstrap and not take too long.
|
||||
Run 'pmbootstrap apkbuild_parse hello-world' for a full output example.
|
||||
|
||||
:param path: full path to the APKBUILD
|
||||
:param check_pkgver: verify that the pkgver is valid.
|
||||
|
|
|
|||
|
|
@ -18,8 +18,6 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|||
"""
|
||||
import argparse
|
||||
import copy
|
||||
import glob
|
||||
import os
|
||||
|
||||
try:
|
||||
import argcomplete
|
||||
|
|
@ -29,6 +27,7 @@ except ImportError:
|
|||
import pmb.config
|
||||
import pmb.parse.arch
|
||||
import pmb.helpers.args
|
||||
import pmb.helpers.pmaports
|
||||
|
||||
""" This file is about parsing command line arguments passed to pmbootstrap, as
|
||||
well as generating the help pages (pmbootstrap -h). All this is done with
|
||||
|
|
@ -235,12 +234,25 @@ def arguments_kconfig(subparser):
|
|||
edit.add_argument("package")
|
||||
|
||||
|
||||
def arguments_repo_missing(subparser):
|
||||
ret = subparser.add_parser("repo_missing")
|
||||
package = ret.add_argument("package", nargs="?", help="only look at a"
|
||||
" specific package and its dependencies")
|
||||
if argcomplete:
|
||||
package.completer = packagecompleter
|
||||
ret.add_argument("--arch", choices=pmb.config.build_device_architectures,
|
||||
default=pmb.parse.arch.alpine_native())
|
||||
ret.add_argument("--built", action="store_true",
|
||||
help="include packages which exist in the binary repos")
|
||||
ret.add_argument("--overview", action="store_true",
|
||||
help="only print the pkgnames without any details")
|
||||
return ret
|
||||
|
||||
|
||||
def packagecompleter(prefix, action, parser, parsed_args):
|
||||
args = parsed_args
|
||||
pmb.config.merge_with_args(args)
|
||||
packages = set()
|
||||
for apkbuild in glob.glob(args.aports + "/*/" + prefix + "*/APKBUILD"):
|
||||
packages.add(os.path.basename(os.path.dirname(apkbuild)))
|
||||
packages = set(pmb.helpers.pmaports.get_list(args))
|
||||
return packages
|
||||
|
||||
|
||||
|
|
@ -315,6 +327,7 @@ def arguments():
|
|||
sub.add_parser("work_migrate", help="run this before using pmbootstrap"
|
||||
" non-interactively to migrate the"
|
||||
" work folder version on demand")
|
||||
arguments_repo_missing(sub)
|
||||
arguments_kconfig(sub)
|
||||
arguments_export(sub)
|
||||
arguments_flasher(sub)
|
||||
|
|
|
|||
|
|
@ -102,22 +102,27 @@ def test_get_apkbuild(args):
|
|||
assert "Could not find" in str(e.value)
|
||||
|
||||
|
||||
def test_check_arch(args):
|
||||
func = pmb.build._package.check_arch
|
||||
apkbuild = {"pkgname": "test"}
|
||||
def test_check_arch_abort(monkeypatch, args):
|
||||
# Fake APKBUILD data
|
||||
apkbuild = {"pkgname": "testpkgname"}
|
||||
|
||||
def fake_helpers_pmaports_get(args, pkgname):
|
||||
return apkbuild
|
||||
monkeypatch.setattr(pmb.helpers.pmaports, "get", fake_helpers_pmaports_get)
|
||||
|
||||
# Arch is right
|
||||
func = pmb.build._package.check_arch_abort
|
||||
apkbuild["arch"] = ["armhf"]
|
||||
func(args, apkbuild, "armhf")
|
||||
func(args, "testpkgname", "armhf")
|
||||
apkbuild["arch"] = ["noarch"]
|
||||
func(args, apkbuild, "armhf")
|
||||
func(args, "testpkgname", "armhf")
|
||||
apkbuild["arch"] = ["all"]
|
||||
func(args, apkbuild, "armhf")
|
||||
func(args, "testpkgname", "armhf")
|
||||
|
||||
# Arch is wrong
|
||||
apkbuild["arch"] = ["x86_64"]
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
func(args, apkbuild, "armhf")
|
||||
func(args, "testpkgname", "armhf")
|
||||
assert "Can't build" in str(e.value)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,185 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
|
||||
# Import from parent directory
|
||||
sys.path.insert(0, os.path.realpath(
|
||||
os.path.join(os.path.dirname(__file__) + "/..")))
|
||||
import pmb.helpers.logging
|
||||
import pmb.helpers.package
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args(request):
|
||||
import pmb.parse
|
||||
sys.argv = ["pmbootstrap", "init"]
|
||||
args = pmb.parse.arguments()
|
||||
args.log = args.work + "/log_testsuite.txt"
|
||||
pmb.helpers.logging.init(args)
|
||||
request.addfinalizer(args.logfd.close)
|
||||
return args
|
||||
|
||||
|
||||
def test_helpers_package_get_pmaports_and_cache(args, monkeypatch):
|
||||
""" Test pmb.helpers.package.get(): find in pmaports, use cached result """
|
||||
|
||||
# Fake APKBUILD data
|
||||
def stub(args, pkgname, must_exist):
|
||||
return {"arch": ["armhf"],
|
||||
"depends": ["testdepend"],
|
||||
"pkgname": "testpkgname",
|
||||
"provides": ["testprovide"],
|
||||
"options": [],
|
||||
"checkdepends": [],
|
||||
"subpackages": [],
|
||||
"makedepends": [],
|
||||
"pkgver": "1.0",
|
||||
"pkgrel": "1"}
|
||||
monkeypatch.setattr(pmb.helpers.pmaports, "get", stub)
|
||||
|
||||
package = {"arch": ["armhf"],
|
||||
"depends": ["testdepend"],
|
||||
"pkgname": "testpkgname",
|
||||
"provides": ["testprovide"],
|
||||
"version": "1.0-r1"}
|
||||
func = pmb.helpers.package.get
|
||||
assert func(args, "testpkgname", "armhf") == package
|
||||
|
||||
# Cached result
|
||||
monkeypatch.delattr(pmb.helpers.pmaports, "get")
|
||||
assert func(args, "testpkgname", "armhf") == package
|
||||
|
||||
|
||||
def test_helpers_package_get_apkindex(args, monkeypatch):
|
||||
""" Test pmb.helpers.package.get(): find in apkindex """
|
||||
|
||||
# Fake APKINDEX data
|
||||
fake_apkindex_data = {"arch": "armhf",
|
||||
"depends": ["testdepend"],
|
||||
"pkgname": "testpkgname",
|
||||
"provides": ["testprovide"],
|
||||
"version": "1.0-r1"}
|
||||
|
||||
def stub(args, pkgname, arch, must_exist):
|
||||
if arch != fake_apkindex_data["arch"]:
|
||||
return None
|
||||
return fake_apkindex_data
|
||||
monkeypatch.setattr(pmb.parse.apkindex, "package", stub)
|
||||
|
||||
# Given arch
|
||||
package = {"arch": ["armhf"],
|
||||
"depends": ["testdepend"],
|
||||
"pkgname": "testpkgname",
|
||||
"provides": ["testprovide"],
|
||||
"version": "1.0-r1"}
|
||||
func = pmb.helpers.package.get
|
||||
assert func(args, "testpkgname", "armhf") == package
|
||||
|
||||
# Other arch
|
||||
assert func(args, "testpkgname", "x86_64") == package
|
||||
|
||||
|
||||
def test_helpers_package_depends_recurse(args):
|
||||
""" Test pmb.helpers.package.depends_recurse() """
|
||||
|
||||
# Put fake data into the pmb.helpers.package.get() cache
|
||||
cache = {"a": {"depends": ["b", "c"]},
|
||||
"b": {"depends": []},
|
||||
"c": {"depends": ["d"]},
|
||||
"d": {"depends": ["b"]}}
|
||||
args.cache["pmb.helpers.package.get"]["armhf"] = cache
|
||||
|
||||
# Normal runs
|
||||
func = pmb.helpers.package.depends_recurse
|
||||
assert func(args, "a", "armhf") == ["a", "b", "c", "d"]
|
||||
assert func(args, "d", "armhf") == ["b", "d"]
|
||||
|
||||
# Cached result
|
||||
args.cache["pmb.helpers.package.get"]["armhf"] = {}
|
||||
assert func(args, "d", "armhf") == ["b", "d"]
|
||||
|
||||
|
||||
def test_helpers_package_check_arch_package(args):
|
||||
""" Test pmb.helpers.package.check_arch(): binary = True """
|
||||
# Put fake data into the pmb.helpers.package.get() cache
|
||||
func = pmb.helpers.package.check_arch
|
||||
cache = {"a": {"arch": []}}
|
||||
args.cache["pmb.helpers.package.get"]["armhf"] = cache
|
||||
|
||||
cache["a"]["arch"] = ["all !armhf"]
|
||||
assert func(args, "a", "armhf") is False
|
||||
|
||||
cache["a"]["arch"] = ["all"]
|
||||
assert func(args, "a", "armhf") is True
|
||||
|
||||
cache["a"]["arch"] = ["noarch"]
|
||||
assert func(args, "a", "armhf") is True
|
||||
|
||||
cache["a"]["arch"] = ["armhf"]
|
||||
assert func(args, "a", "armhf") is True
|
||||
|
||||
cache["a"]["arch"] = ["aarch64"]
|
||||
assert func(args, "a", "armhf") is False
|
||||
|
||||
|
||||
def test_helpers_package_check_arch_pmaports(args, monkeypatch):
|
||||
""" Test pmb.helpers.package.check_arch(): binary = False """
|
||||
func = pmb.helpers.package.check_arch
|
||||
fake_pmaport = {"arch": []}
|
||||
|
||||
def fake_pmaports_get(args, pkgname, must_exist=False):
|
||||
return fake_pmaport
|
||||
monkeypatch.setattr(pmb.helpers.pmaports, "get", fake_pmaports_get)
|
||||
|
||||
fake_pmaport["arch"] = ["armhf"]
|
||||
assert func(args, "a", "armhf", False) is True
|
||||
|
||||
fake_pmaport["arch"] = ["all", "!armhf"]
|
||||
assert func(args, "a", "armhf", False) is False
|
||||
|
||||
|
||||
def test_helpers_package_check_arch_recurse(args, monkeypatch):
|
||||
""" Test pmb.helpers.package.check_arch_recurse() """
|
||||
# Test data
|
||||
func = pmb.helpers.package.check_arch_recurse
|
||||
depends = ["a", "b", "c"]
|
||||
arch_check_results = {}
|
||||
|
||||
def fake_depends_recurse(args, pkgname, arch):
|
||||
return depends
|
||||
monkeypatch.setattr(pmb.helpers.package, "depends_recurse",
|
||||
fake_depends_recurse)
|
||||
|
||||
def fake_check_arch(args, pkgname, arch):
|
||||
return arch_check_results[pkgname]
|
||||
monkeypatch.setattr(pmb.helpers.package, "check_arch", fake_check_arch)
|
||||
|
||||
# Result: True
|
||||
arch_check_results = {"a": True,
|
||||
"b": True,
|
||||
"c": True}
|
||||
assert func(args, "a", "armhf") is True
|
||||
|
||||
# Result: False
|
||||
arch_check_results = {"a": True,
|
||||
"b": False,
|
||||
"c": True}
|
||||
assert func(args, "a", "armhf") is False
|
||||
|
|
@ -0,0 +1,158 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
# Import from parent directory
|
||||
sys.path.insert(0, os.path.realpath(
|
||||
os.path.join(os.path.dirname(__file__) + "/..")))
|
||||
import pmb.build.other
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args(request):
|
||||
import pmb.parse
|
||||
sys.argv = ["pmbootstrap", "init"]
|
||||
args = pmb.parse.arguments()
|
||||
args.log = args.work + "/log_testsuite.txt"
|
||||
pmb.helpers.logging.init(args)
|
||||
request.addfinalizer(args.logfd.close)
|
||||
return args
|
||||
|
||||
|
||||
def test_filter_missing_packages_invalid(args):
|
||||
""" Test ...repo_missing.filter_missing_packages(): invalid package """
|
||||
func = pmb.helpers.repo_missing.filter_missing_packages
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
func(args, "armhf", ["invalid-package-name"])
|
||||
assert str(e.value).startswith("Could not find aport")
|
||||
|
||||
|
||||
def test_filter_missing_packages_binary_exists(args):
|
||||
""" Test ...repo_missing.filter_missing_packages(): binary exists """
|
||||
func = pmb.helpers.repo_missing.filter_missing_packages
|
||||
assert func(args, "armhf", ["busybox"]) == []
|
||||
|
||||
|
||||
def test_filter_missing_packages_pmaports(args, monkeypatch):
|
||||
""" Test ...repo_missing.filter_missing_packages(): pmaports """
|
||||
build_is_necessary = None
|
||||
func = pmb.helpers.repo_missing.filter_missing_packages
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return build_is_necessary
|
||||
monkeypatch.setattr(pmb.build, "is_necessary", stub)
|
||||
|
||||
build_is_necessary = True
|
||||
assert func(args, "x86_64", ["busybox", "hello-world"]) == ["hello-world"]
|
||||
|
||||
build_is_necessary = False
|
||||
assert func(args, "x86_64", ["busybox", "hello-world"]) == []
|
||||
|
||||
|
||||
def test_filter_aport_packages(args):
|
||||
""" Test ...repo_missing.filter_aport_packages() """
|
||||
func = pmb.helpers.repo_missing.filter_aport_packages
|
||||
assert func(args, "armhf", ["busybox", "hello-world"]) == ["hello-world"]
|
||||
|
||||
|
||||
def test_filter_arch_packages(args, monkeypatch):
|
||||
""" Test ...repo_missing.filter_arch_packages() """
|
||||
func = pmb.helpers.repo_missing.filter_arch_packages
|
||||
check_arch = None
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return check_arch
|
||||
monkeypatch.setattr(pmb.helpers.package, "check_arch_recurse", stub)
|
||||
|
||||
check_arch = False
|
||||
assert func(args, "armhf", ["hello-world"]) == []
|
||||
|
||||
check_arch = True
|
||||
assert func(args, "armhf", []) == []
|
||||
|
||||
|
||||
def test_get_relevant_packages(args, monkeypatch):
|
||||
""" Test ...repo_missing.get_relevant_packages() """
|
||||
|
||||
# Set up fake return values
|
||||
stub_data = {"check_arch_recurse": False,
|
||||
"depends_recurse": ["a", "b", "c", "d"],
|
||||
"filter_arch_packages": ["a", "b", "c"],
|
||||
"filter_aport_packages": ["b", "a"],
|
||||
"filter_missing_packages": ["a"]}
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return stub_data["check_arch_recurse"]
|
||||
monkeypatch.setattr(pmb.helpers.package, "check_arch_recurse", stub)
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return stub_data["depends_recurse"]
|
||||
monkeypatch.setattr(pmb.helpers.package, "depends_recurse", stub)
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return stub_data["filter_arch_packages"]
|
||||
monkeypatch.setattr(pmb.helpers.repo_missing, "filter_arch_packages", stub)
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return stub_data["filter_aport_packages"]
|
||||
monkeypatch.setattr(pmb.helpers.repo_missing, "filter_aport_packages",
|
||||
stub)
|
||||
|
||||
def stub(args, arch, pmaport):
|
||||
return stub_data["filter_missing_packages"]
|
||||
monkeypatch.setattr(pmb.helpers.repo_missing, "filter_missing_packages",
|
||||
stub)
|
||||
|
||||
# No given package
|
||||
func = pmb.helpers.repo_missing.get_relevant_packages
|
||||
assert func(args, "armhf") == ["a"]
|
||||
assert func(args, "armhf", built=True) == ["a", "b"]
|
||||
|
||||
# Package can't be built for given arch
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
func(args, "armhf", "a")
|
||||
assert "can't be built" in str(e.value)
|
||||
|
||||
# Package can be built for given arch
|
||||
stub_data["check_arch_recurse"] = True
|
||||
assert func(args, "armhf", "a") == ["a"]
|
||||
assert func(args, "armhf", "a", True) == ["a", "b"]
|
||||
|
||||
|
||||
def test_generate_output_format(args, monkeypatch):
|
||||
""" Test ...repo_missing.generate_output_format() """
|
||||
|
||||
def stub(args, pkgname, arch):
|
||||
return {"pkgname": "hello-world", "version": "1.0-r0",
|
||||
"depends": ["depend1", "depend2"]}
|
||||
monkeypatch.setattr(pmb.helpers.package, "get", stub)
|
||||
|
||||
def stub(args, pkgname):
|
||||
return "main"
|
||||
monkeypatch.setattr(pmb.helpers.pmaports, "get_repo", stub)
|
||||
|
||||
func = pmb.helpers.repo_missing.generate_output_format
|
||||
ret = [{"pkgname": "hello-world",
|
||||
"repo": "main",
|
||||
"version": "1.0-r0",
|
||||
"depends": ["depend1", "depend2"]}]
|
||||
assert func(args, "armhf", ["hello-world"]) == ret
|
||||
Loading…
Reference in New Issue