synit
/
pmbootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129) 

TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.

I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.

To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.

In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.

In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".

This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.

Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).

The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
This commit is contained in:
Oliver Smith 2017-07-10 15:23:43 +00:00 committed by GitHub
parent b4c9f93f0d
commit 51bdc24315
35 changed files with 784 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
pmb/build/buildinfo.py
View File

@ -18,43 +18,10 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import json
import logging
import pmb.chroot
import pmb.chroot.apk
import pmb.parse.apkindex
def get_depends_recursively(args, pkgnames, arch=None):
"""
:param pkgnames: List of pkgnames, for which the dependencies shall be
retrieved.
"""
todo = list(pkgnames)
ret = []
seen = []
while len(todo):
pkgname = todo.pop(0)
index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch)
if not index_data:
logging.debug(
"NOTE: Could not find dependency " +
pkgname +
" in any APKINDEX.")
continue
pkgname = index_data["pkgname"]
if pkgname not in pkgnames and pkgname not in ret:
ret.append(pkgname)
for depend in index_data["depends"]:
if depend not in ret:
if depend.startswith("!"):
continue
for operator in [">", "="]:
if operator in depend:
depend = depend.split(operator)[0]
if depend not in seen:
todo.append(depend)
seen.append(depend)
return ret
import pmb.parse.depends
def generate(args, apk_path, arch, suffix, apkbuild):
@ -71,9 +38,12 @@ def generate(args, apk_path, arch, suffix, apkbuild):
# Add makedepends versions
installed = pmb.chroot.apk.installed(args, suffix)
relevant = (apkbuild["makedepends"] +
get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"]))
relevant = (apkbuild["makedepends"] + [apkbuild["pkgname"], "abuild",
"build-base"])
relevant = pmb.parse.depends.recurse(args, relevant, arch, in_aports=False)
for pkgname in relevant:
if pkgname == apkbuild["pkgname"]:
continue
if pkgname in installed:
ret["versions"][pkgname] = installed[pkgname]["version"]
return ret

2
pmb/build/menuconfig.py
View File

@ -33,7 +33,7 @@ def menuconfig(args, pkgname, arch):
aport = pmb.build.find_aport(args, pkgname, False)
if not aport:
raise RuntimeError("Package " + pkgname + ": Could not find aport!")
apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD")
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
# Set up build tools and makedepends
pmb.build.init(args)

129
pmb/build/other.py
View File

@ -19,6 +19,7 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
import os
import logging
import glob
import shutil
import pmb.chroot
import pmb.helpers.run
@ -37,14 +38,23 @@ def find_aport(args, package, must_exist=True):
if os.path.exists(path):
return path
# Try to get a cached result first (we assume, that the aports don't change
# in one pmbootstrap call)
if path in args.cache["find_aport"]:
return args.cache["find_aport"][path]
ret = None
for path_current in glob.glob(args.aports + "/*/APKBUILD"):
apkbuild = pmb.parse.apkbuild(path_current)
apkbuild = pmb.parse.apkbuild(args, path_current)
if package in apkbuild["subpackages"]:
return os.path.dirname(path_current)
if must_exist:
ret = os.path.dirname(path_current)
break
if ret is None and must_exist:
raise RuntimeError("Could not find aport for package: " +
package)
return None
args.cache["find_aport"][path] = ret
return ret
def copy_to_buildpath(args, package, suffix="native"):
@ -66,6 +76,83 @@ def copy_to_buildpath(args, package, suffix="native"):
"/home/user/build"], suffix=suffix)
def aports_files_out_of_sync_with_git(args, package=None):
"""
Get a list of files, about which git says, that they have changed in
comparison to upstream. We need this for the timestamp based rebuild check,
where it does not only rely on the APKBUILD pkgver and pkgrel, but also on
the file's last modified date to decide if it needs to be rebuilt. Git sets
the last modified timestamp to the last checkout date, so we must ignore
all files, that have not been modified, or else we would trigger rebuilds
for all packages, from the pmOS binary repository.
:returns: list of absolute paths to all files not in sync with upstream
"""
# Filter out a specific package
if package:
ret = []
prefix = os.path.abspath(args.aports + "/" + package + "/")
for file in aports_files_out_of_sync_with_git(args):
if file.startswith(prefix):
ret.append(file)
return ret
# Use cached result if possible
if args.cache["aports_files_out_of_sync_with_git"] is not None:
return args.cache["aports_files_out_of_sync_with_git"]
# Get the aport's git repository folder
git_root = None
if shutil.which("git"):
git_root = pmb.helpers.run.user(args, ["git", "rev-parse",
"--show-toplevel"],
working_dir=args.aports,
return_stdout=True,
check=False)
if git_root:
git_root = git_root.rstrip()
ret = []
if git_root and os.path.exists(git_root):
# Find tracked files out of sync with upstream
tracked = pmb.helpers.run.user(args, ["git", "diff", "--name-only", "origin"],
working_dir=git_root, return_stdout=True)
# Find all untracked files
untracked = pmb.helpers.run.user(
args, ["git", "ls-files", "--others", "--exclude-standard"],
working_dir=git_root, return_stdout=True)
# Set absolute path, filter out aports files
aports_absolute = os.path.abspath(args.aports)
files = tracked.rstrip().split("\n") + untracked.rstrip().split("\n")
for file in files:
file = os.path.abspath(git_root + "/" + file)
if file.startswith(aports_absolute):
ret.append(file)
else:
logging.warning("WARNING: Can not determine, which aport-files have been"
" changed from upstream!")
logging.info("* Aports-folder is not a git repository or git is not"
" installed")
logging.info("* You can turn timestamp-based rebuilds off in"
" 'pmbootstrap init'")
# Save cache
args.cache["aports_files_out_of_sync_with_git"] = ret
return ret
def sources_newer_than_binary_package(args, package, index_data):
path_sources = []
for file in glob.glob(args.aports + "/" + package + "/*"):
path_sources.append(file)
lastmod_target = float(index_data["timestamp"])
return not pmb.helpers.file.is_up_to_date(path_sources,
lastmod_target=lastmod_target)
def is_necessary(args, arch, apkbuild, apkindex_path=None):
"""
Check if the package has already been built. Compared to abuild's check,
@ -78,9 +165,10 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
:param apkindex_path: override the APKINDEX.tar.gz path
:returns: boolean
"""
# Get new version from APKBUILD
# Get package name, version, define start of debug message
package = apkbuild["pkgname"]
version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
msg = "Build is neccessary for package '" + package + "': "
# Get old version from APKINDEX
if apkindex_path:
@ -89,6 +177,7 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
else:
index_data = pmb.parse.apkindex.read_any_index(args, package, arch)
if not index_data:
logging.debug(msg + "No binary package available")
return True
# a) Binary repo has a newer version
@ -102,17 +191,31 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
# b) Aports folder has a newer version
if version_new != version_old:
logging.debug(msg + "Binary package out of date (binary: " + version_old +
", aport: " + version_new + ")")
return True
# c) The version is the same. Check if all files in the aport folder have an
# older timestamp, than the package. This way the pkgrel doesn't need to be
# increased while developing locally.
lastmod_target = float(index_data["timestamp"])
path_sources = glob.glob(args.aports + "/" + package + "/*")
if pmb.helpers.file.is_up_to_date(
path_sources, lastmod_target=lastmod_target):
# Aports and binary repo have the same version.
if not args.timestamp_based_rebuild:
return False
# c) Same version, source files out of sync with upstream, source
# files newer than binary package
files_out_of_sync = aports_files_out_of_sync_with_git(args, package)
sources_newer = sources_newer_than_binary_package(
args, package, index_data)
if len(files_out_of_sync) and sources_newer:
logging.debug(msg + "Binary package and aport have the same pkgver and"
" pkgrel, but there are aport source files out of sync"
" with the upstream git repository *and* these source"
" files have a more recent 'last modified' timestamp than"
" the binary package's build timestamp.")
return True
# d) Same version, source files *in sync* with upstream *or* source
# files *older* than binary package
else:
return False
return True
def index_repo(args, arch=None):

11
pmb/build/package.py
View File

@ -30,7 +30,7 @@ import pmb.parse
import pmb.parse.arch
def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False):
def package(args, pkgname, carch, force=False, buildinfo=False):
"""
Build a package with Alpine Linux' abuild.
@ -46,24 +46,19 @@ def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False):
" and could not find this package in any APKINDEX!")
# Autodetect the build environment
apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD")
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
pkgname = apkbuild["pkgname"]
carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch)
suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv)
cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv,
suffix)
# Build dependencies first (they may be outdated, even if they exist)
if recurse:
for depend in apkbuild["depends"]:
package(args, depend, carch)
# Skip already built versions
if not force and not pmb.build.is_necessary(
args, carch_buildenv, apkbuild):
return
# Install build tools and makedepends
# Initialize build environment, install/build makedepends
pmb.build.init(args, suffix)
if len(apkbuild["makedepends"]):
pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix)

108
pmb/chroot/apk.py
View File

@ -21,6 +21,8 @@ import logging
import pmb.chroot
import pmb.config
import pmb.parse.apkindex
import pmb.parse.arch
import pmb.parse.depends
def check_min_version(args, suffix="native"):
@ -54,41 +56,110 @@ def check_min_version(args, suffix="native"):
args.cache["apk_min_version_checked"].append(suffix)
def install_is_necessary(args, build, arch, package, packages_installed):
"""
This function optionally builds an out of date package, and checks if the
version installed inside a chroot is up to date.
:param build: Set to true to build the package, if the binary packages are
out of date, and it is in the aports folder.
:param packages_installed: Return value from installed().
:returns: True if the package needs to be installed/updated, False otherwise.
"""
# Build package
if build:
pmb.build.package(args, package, arch)
# No further checks when not installed
if package not in packages_installed:
return True
# Compare the installed version vs. the version in the repos
data_installed = packages_installed[package]
data_repo = pmb.parse.apkindex.read_any_index(args, package, arch)
compare = pmb.parse.apkindex.compare_version(data_installed["version"],
data_repo["version"])
# a) Installed newer (should not happen normally)
if compare == 1:
logging.info("WARNING: " + arch + " package '" + package +
"' installed version " + data_installed["version"] +
" is newer, than the version in the repositories: " +
data_repo["version"])
return False
# b) Repo newer
elif compare == -1:
return True
# c) Same version, look at last modified
elif compare == 0:
time_installed = float(data_installed["timestamp"])
time_repo = float(data_repo["timestamp"])
return time_repo > time_installed
def replace_aports_packages_with_path(args, packages, suffix, arch):
"""
apk will only re-install packages with the same pkgname, pkgver and pkgrel,
when you give it the absolute path to the package. This function replaces
all packages, that were built locally, with the absolute path to the package.
"""
ret = []
for package in packages:
aport = pmb.build.find_aport(args, package, False)
if aport:
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
apk_path = ("/home/user/packages/user/" + arch + "/" +
package + "-" + apkbuild["pkgver"] + "-r" +
apkbuild["pkgrel"] + ".apk")
if os.path.exists(args.work + "/chroot_" + suffix + apk_path):
package = apk_path
ret.append(package)
return ret
def install(args, packages, suffix="native", build=True):
"""
:param build: automatically build the package, when it does not exist yet
and it is inside the pm-aports folder. Checking this is expensive - if
you know, that all packages are provides by upstream repos, set this to
False!
or needs to be updated, and it is inside the pm-aports
folder. Checking this is expensive - if you know, that all
packages are provides by upstream repos, set this to False!
"""
# Initialize chroot
check_min_version(args, suffix)
pmb.chroot.init(args, suffix)
# Filter already installed packages
# Add depends to packages
arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
packages_with_depends = pmb.parse.depends.recurse(args, packages, arch,
strict=True)
# Filter out up-to-date packages
packages_installed = installed(args, suffix)
packages_todo = []
for package in packages:
if package not in packages_installed:
for package in packages_with_depends:
if install_is_necessary(
args, build, arch, package, packages_installed):
packages_todo.append(package)
if not len(packages_todo):
return
# Build packages if necessary
arch = pmb.parse.arch.from_chroot_suffix(args, suffix)
if build:
for package in packages_todo:
pmb.build.package(args, package, arch)
# Sanitize packages: don't allow '--allow-untrusted' and other options
# to be passed to apk!
for package in packages_todo:
if package.startswith("-"):
raise ValueError("Invalid package name: " + package)
# Install everything
logging.info("(" + suffix + ") install " + " ".join(packages_todo))
pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo,
# Readable install message without dependencies
message = "(" + suffix + ") install"
for pkgname in packages:
if pkgname not in packages_installed:
message += " " + pkgname
logging.info(message)
# Install/update everything
packages_todo = replace_aports_packages_with_path(args, packages_todo,
suffix, arch)
pmb.chroot.root(args, ["apk", "--no-progress", "add", "-u"] + packages_todo,
suffix)
@ -96,19 +167,22 @@ def upgrade(args, suffix="native", update_index=True):
"""
Upgrade all packages installed in a chroot
"""
# Prepare apk and update index
check_min_version(args, suffix)
pmb.chroot.init(args, suffix)
if update_index:
pmb.chroot.root(args, ["apk", "update"], suffix)
# -a: also update previously downgraded (and therefore pinned) packages
pmb.chroot.root(args, ["apk", "upgrade", "-a"], suffix)
# Rebuild and upgrade out-of-date packages
packages = installed(args, suffix).keys()
install(args, packages, suffix)
def installed(args, suffix="native"):
"""
Read the list of installed packages (which has almost the same format, as
an APKINDEX, but with more keys).
:returns: a dictionary with the following structure:
{ "postmarketos-mkinitfs":
{

12
pmb/chroot/init.py
View File

@ -25,6 +25,7 @@ import filecmp
import pmb.chroot
import pmb.chroot.apk_static
import pmb.config
import pmb.helpers.repo
import pmb.helpers.run
import pmb.parse.arch
@ -78,16 +79,7 @@ def init(args, suffix="native"):
# Write /etc/apk/repositories
repos_path = chroot + "/etc/apk/repositories"
if not os.path.exists(repos_path):
lines = ["/home/user/packages/user"]
if args.mirror_postmarketos:
lines.append(args.mirror_postmarketos)
directories = ["main", "community"]
if args.alpine_version == "edge":
directories.append("testing")
for dir in directories:
lines.append(args.mirror_alpine + args.alpine_version +
"/" + dir)
for line in lines:
for line in pmb.helpers.repo.urls(args):
pmb.helpers.run.root(args, ["sh", "-c",
"echo " + shlex.quote(line) + " >> " + repos_path])

5
pmb/chroot/initfs.py
View File

@ -25,6 +25,11 @@ import pmb.helpers.cli
def build(args, flavor, suffix):
# Update mkinitfs and hooks
pmb.chroot.apk.install(args, ["postmarketos-mkinitfs"], suffix)
pmb.chroot.initfs_hooks.update(args, suffix)
# Call mkinitfs
logging.info("(" + suffix + ") mkinitfs " + flavor)
release_file = (args.work + "/chroot_" + suffix + "/usr/share/kernel/" +
flavor + "/kernel.release")

14
pmb/chroot/initfs_hooks.py
View File

@ -24,12 +24,15 @@ import pmb.config
import pmb.chroot.apk
def list_chroot(args, suffix):
def list_chroot(args, suffix, remove_prefix=True):
ret = []
prefix = pmb.config.initfs_hook_prefix
for pkgname in pmb.chroot.apk.installed(args, suffix):
if pkgname.startswith(prefix):
ret.append(pkgname[len(prefix):])
if remove_prefix:
ret.append(pkgname[len(prefix):])
else:
ret.append(pkgname)
return ret
@ -65,3 +68,10 @@ def delete(args, hook, suffix):
raise RuntimeError("There is no such hook installed!")
prefix = pmb.config.initfs_hook_prefix
pmb.chroot.root(args, ["apk", "del", prefix + hook], suffix)
def update(args, suffix):
"""
Rebuild and update all hooks, that are out of date
"""
pmb.chroot.apk.install(args, list_chroot(args, suffix, False), suffix)

3
pmb/chroot/root.py
View File

@ -53,7 +53,8 @@ def root(args, cmd, suffix="native", working_dir="/", log=True,
if not auto_init and not os.path.islink(chroot + "/bin/sh"):
raise RuntimeError("Chroot does not exist: " + chroot)
pmb.chroot.init(args, suffix)
if auto_init:
pmb.chroot.init(args, suffix)
# Run the args with sudo chroot, and with cleaned environment
# variables

2
pmb/chroot/zap.py
View File

@ -28,7 +28,7 @@ def zap(args):
patterns = [
"chroot_native",
"chroot_buildroot_*",
"chroot_rootfs_" + args.device,
"chroot_rootfs_*",
]
# Only ask for removal, if the user specificed the extra '-p' switch.

1
pmb/config/__init__.py
View File

@ -46,6 +46,7 @@ defaults = {
"aports": os.path.normpath(pmb_src + "/aports"),
"config": os.path.expanduser("~") + "/.config/pmbootstrap.cfg",
"device": "samsung-i9100",
"timestamp_based_rebuild": True,
"log": "$WORK/log.txt",
"mirror_alpine": "https://nl.alpinelinux.org/alpine/",
"mirror_postmarketos": "",

11
pmb/config/init.py
View File

@ -52,6 +52,16 @@ def init(args):
" compiling?")
cfg["pmbootstrap"]["jobs"] = pmb.helpers.cli.ask(args, "Jobs",
None, default)
# Timestamp based rebuilds
default = "y"
if not args.timestamp_based_rebuild:
default = "n"
logging.info("Rebuild packages, when the last modified timestamp changed,"
" even if the version did not change? This makes pmbootstrap"
" behave more like 'make'.")
answer = pmb.helpers.cli.ask(args, "Timestamp based rebuilds",
default=default)
cfg["pmbootstrap"]["timestamp_based_rebuild"] = str(answer == "y")
# Save config
pmb.config.save(args, cfg)
@ -61,5 +71,4 @@ def init(args):
logging.info("Run 'pmbootstrap zap' to delete all chroots once a day before"
" working with pmbootstrap!")
logging.info("It only takes a few seconds, and all packages are cached.")
logging.info("Done!")

2
pmb/config/load.py
View File

@ -31,6 +31,6 @@ def load(args):
for key in pmb.config.defaults:
if key not in cfg["pmbootstrap"]:
cfg["pmbootstrap"][key] = pmb.config.defaults[key]
cfg["pmbootstrap"][key] = str(pmb.config.defaults[key])
return cfg

2
pmb/helpers/devices.py
View File

@ -40,5 +40,5 @@ def list_apkbuilds(args):
ret = {}
for device in list(args):
apkbuild_path = args.aports + "/device-" + device + "/APKBUILD"
ret[device] = pmb.parse.apkbuild(apkbuild_path)
ret[device] = pmb.parse.apkbuild(args, apkbuild_path)
return ret

37
pmb/helpers/logging.py
View File

@ -48,28 +48,47 @@ class log_handler(logging.StreamHandler):
self.handleError(record)
def add_verbose_log_level():
"""
Add a new log level "verbose", which is below "debug". Also monkeypatch
logging, so it can be used with logging.verbose().
This function is based on work by Voitek Zylinski and sleepycal:
https://stackoverflow.com/a/20602183
All stackoverflow user contributions are licensed as CC-BY-SA:
https://creativecommons.org/licenses/by-sa/3.0/
"""
logging.VERBOSE = 5
logging.addLevelName(logging.VERBOSE, "VERBOSE")
logging.Logger.verbose = lambda inst, msg, * \
args, **kwargs: inst.log(logging.VERBOSE, msg, *args, **kwargs)
logging.verbose = lambda msg, *args, **kwargs: logging.log(logging.VERBOSE, msg,
*args, **kwargs)
def init(args):
"""
Set log format and add the log file descriptor to args.logfd.
Set log format and add the log file descriptor to args.logfd, add the
verbose log level.
"""
# Open logfile
if not os.path.exists(args.work):
os.makedirs(args.work)
date_format = "%H:%M:%S"
setattr(args, "logfd", open(args.log, "a+"))
# Set log format
root_logger = logging.getLogger()
root_logger.handlers = []
formatter = logging.Formatter("[%(asctime)s] %(message)s",
datefmt="%H:%M:%S")
formatter = None
# Set log level
add_verbose_log_level()
root_logger.setLevel(logging.DEBUG)
if args.verbose:
formatter = logging.Formatter("[%(asctime)s %(module)s]"
" %(message)s", datefmt=date_format)
else:
formatter = logging.Formatter("[%(asctime)s] %(message)s",
datefmt=date_format)
root_logger.setLevel(logging.VERBOSE)
# Add a custom log handler
handler = log_handler()
log_handler._args = args
handler.setFormatter(formatter)

72
pmb/helpers/repo.py
View File

@ -18,6 +18,7 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import glob
import os
import hashlib
def files(args):
@ -64,3 +65,74 @@ def diff(args, files_a, files_b=None):
ret.append(arch + "/" + file)
return sorted(ret)
def hash(url, length=8):
"""
Generate the hash, that APK adds to the APKINDEX and apk packages
in its apk cache folder. It is the "12345678" part in this example:
"APKINDEX.12345678.tar.gz".
:param length: The length of the hash in the output file.
See also: official implementation in apk-tools:
<https://git.alpinelinux.org/cgit/apk-tools/>
blob.c: apk_blob_push_hexdump(), "const char *xd"
apk_defines.h: APK_CACHE_CSUM_BYTES
database.c: apk_repo_format_cache_index()
"""
binary = hashlib.sha1(url.encode("utf-8")).digest()
xd = "0123456789abcdefghijklmnopqrstuvwxyz"
csum_bytes = int(length / 2)
ret = ""
for i in range(csum_bytes):
ret += xd[(binary[i] >> 4) & 0xf]
ret += xd[binary[i] & 0xf]
return ret
def urls(args, user_repository=True):
"""
Get a list of repository URLs, as they are in /etc/apk/repositories.
"""
ret = []
# Local user repository (for packages compiled with pmbootstrap)
if user_repository:
ret.append("/home/user/packages/user")
# Upstream postmarketOS binary repository
if args.mirror_postmarketos:
ret.append(args.mirror_postmarketos)
# Upstream Alpine Linux repositories
directories = ["main", "community"]
if args.alpine_version == "edge":
directories.append("testing")
for dir in directories:
ret.append(args.mirror_alpine + args.alpine_version + "/" + dir)
return ret
def apkindex_files(args, arch="native"):
"""
Get a list of outside paths to all resolved APKINDEX.tar.gz files
from the urls() list for a specific arch.
"""
if arch == "native":
arch = args.arch_native
# Try to get a cached result first.
if arch in args.cache["apkindex_files"]:
return args.cache["apkindex_files"][arch]
# Add the non-hashed user path and the upstream paths with hashes
ret = [args.work + "/packages/" + arch + "/APKINDEX.tar.gz"]
for url in urls(args, False):
ret.append(args.work + "/cache_apk_" + arch + "/APKINDEX." +
hash(url) + ".tar.gz")
args.cache["apkindex_files"][arch] = ret
return ret

23
pmb/helpers/run.py
View File

@ -18,9 +18,11 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import subprocess
import logging
import os
def core(args, cmd, log_message, log, return_stdout, check=True):
def core(args, cmd, log_message, log, return_stdout, check=True,
working_dir=None):
logging.debug(log_message)
"""
Run the command and write the output to the log.
@ -28,6 +30,10 @@ def core(args, cmd, log_message, log, return_stdout, check=True):
:param check: raise an exception, when the command fails
"""
if working_dir:
working_dir_old = os.getcwd()
os.chdir(working_dir)
try:
ret = None
if log:
@ -52,19 +58,22 @@ def core(args, cmd, log_message, log, return_stdout, check=True):
raise RuntimeError("Command failed: " + log_message) from exc
else:
pass
if working_dir:
os.chdir(working_dir_old)
return ret
def user(args, cmd, log=True, working_dir=None, return_stdout=False,
check=True):
"""
:param working_dir: defaults to args.work
"""
if not working_dir:
working_dir = args.work
if working_dir:
msg = "% cd " + working_dir + " && " + " ".join(cmd)
else:
msg = "% " + " ".join(cmd)
# TODO: maintain and check against a whitelist
return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check)
return core(args, cmd, msg, log, return_stdout, check, working_dir)
def root(args, cmd, log=True, working_dir=None, return_stdout=False,

2
pmb/parse/__init__.py
View File

@ -18,6 +18,6 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
from pmb.parse.arguments import arguments
from pmb.parse.apkbuild import apkbuild
from pmb.parse.deviceinfo import deviceinfo
from pmb.parse.binfmt_info import binfmt_info
from pmb.parse.deviceinfo import deviceinfo
import pmb.parse.arch

8
pmb/parse/apkbuild.py
View File

@ -69,7 +69,7 @@ def cut_off_function_names(apkbuild):
return apkbuild
def apkbuild(path):
def apkbuild(args, path):
"""
Parse relevant information out of the APKBUILD file. This is not meant
to be perfect and catch every edge case (for that, a full shell parser
@ -80,6 +80,11 @@ def apkbuild(path):
:returns: Relevant variables from the APKBUILD. Arrays get returned as
arrays.
"""
# Try to get a cached result first (we assume, that the aports don't change
# in one pmbootstrap call)
if path in args.cache["apkbuild"]:
return args.cache["apkbuild"][path]
with open(path, encoding="utf-8") as handle:
lines = handle.readlines()
@ -122,4 +127,5 @@ def apkbuild(path):
ret = replace_variables(ret)
ret = cut_off_function_names(ret)
args.cache["apkbuild"][path] = ret
return ret

23
pmb/parse/apkindex.py
View File

@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import distutils.version
import glob
import logging
import os
import tarfile
import pmb.chroot.apk
import pmb.helpers.repo
def compare_version(a_str, b_str):
@ -118,8 +120,7 @@ def parse_next_block(args, path, lines, start):
return None
def parse_add_block(path, strict, ret, block, pkgname=None,
version_new=None):
def parse_add_block(path, strict, ret, block, pkgname=None):
"""
Add one block to the return dictionary of parse().
@ -140,8 +141,6 @@ def parse_add_block(path, strict, ret, block, pkgname=None,
# Defaults
if not pkgname:
pkgname = block["pkgname"]
if not version_new:
version_new = block["version"]
# Handle duplicate entries
if pkgname in ret:
@ -151,6 +150,7 @@ def parse_add_block(path, strict, ret, block, pkgname=None,
# Ignore the block, if the block we already have has a higher
# version
version_old = ret[pkgname]["version"]
version_new = block["version"]
if compare_version(version_old, version_new) == 1:
return
@ -207,8 +207,7 @@ def parse(args, path, strict=False):
for alias in block["provides"]:
split = alias.split("=")
if len(split) == 2:
parse_add_block(path, strict, ret, block, split[0],
split[1])
parse_add_block(path, strict, ret, block, split[0])
# Update the cache
args.cache["apkindex"][path] = {"lastmod": lastmod, "ret": ret}
@ -254,12 +253,14 @@ def read_any_index(args, package, arch=None):
"""
if not arch:
arch = args.arch_native
indexes = [args.work + "/packages/" + arch + "/APKINDEX.tar.gz"]
pattern = args.work + "/cache_apk_" + arch + "/APKINDEX.*.tar.gz"
indexes += glob.glob(pattern)
for index in indexes:
# Return first match
for index in pmb.helpers.repo.apkindex_files(args, arch):
index_data = read(args, package, index, False)
logging.verbose("Search for " + package + " in " + index +
" - result: " + str(index_data))
if index_data:
return index_data
logging.verbose("No match found in any APKINDEX.tar.gz!")
return None

27
pmb/parse/arguments.py
View File

@ -107,8 +107,8 @@ def arguments():
# Logging
parser.add_argument("-l", "--log", dest="log", default=None)
parser.add_argument("-v", "--verbose", dest="verbose",
action="store_true", help="output the source file, where the log"
" message originated from with each log message")
action="store_true", help="write even more to the"
"logfile (this may reduce performance)")
parser.add_argument("-q", "--quiet", dest="quiet",
action="store_true", help="do not output any log messages")
@ -195,23 +195,38 @@ def arguments():
" .apk, or must be named"
" APKINDEX.tar.gz.")
# Action: parse_apkindex
parse_apkindex = sub.add_parser("parse_apkindex")
parse_apkindex.add_argument("apkindex_path")
# Use defaults from the user's config file
args = parser.parse_args()
cfg = pmb.config.load(args)
for varname in cfg["pmbootstrap"]:
if varname not in args or not getattr(args, varname):
setattr(args, varname, cfg["pmbootstrap"][varname])
value = cfg["pmbootstrap"][varname]
if varname in pmb.config.defaults:
default = pmb.config.defaults[varname]
if isinstance(default, bool):
value = (value.lower() == "true")
setattr(args, varname, value)
# Replace $WORK in variables from user's config
for varname in cfg["pmbootstrap"]:
old = getattr(args, varname)
setattr(args, varname, old.replace("$WORK", args.work))
if isinstance(old, str):
setattr(args, varname, old.replace("$WORK", args.work))
# Add convenience shortcuts
setattr(args, "arch_native", pmb.parse.arch.alpine_native())
# Add a caching dict
setattr(args, "cache", {"apkindex": {}, "apk_min_version_checked": []})
# Add a caching dict (caches parsing of files etc. for the current session)
setattr(args, "cache", {"apkindex": {},
"apkindex_files": {},
"apkbuild": {},
"apk_min_version_checked": [],
"aports_files_out_of_sync_with_git": None,
"find_aport": {}})
# Add and verify the deviceinfo (only after initialization)
if args.action != "init":

107
pmb/parse/depends.py Normal file
View File

@ -0,0 +1,107 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import logging
import pmb.chroot
import pmb.chroot.apk
import pmb.parse.apkindex
def apkindex(args, pkgname, arch):
"""
Non-recursively get the dependencies of one package in any APKINDEX.
"""
index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch)
if not index_data:
return None
# Remove operators from the depends list
ret = []
for depend in index_data["depends"]:
if depend.startswith("!"):
continue
for operator in [">", "="]:
if operator in depend:
depend = depend.split(operator)[0]
if depend not in ret:
ret.append(depend)
return ret
def recurse_error_message(pkgname, in_aports, in_apkindexes):
ret = "Could not find package '" + pkgname + "'"
if in_aports:
ret += " aport"
if in_apkindexes:
ret += " and could not find it"
if in_apkindexes:
ret += " in any APKINDEX"
return ret
def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True,
strict=False):
"""
Find all dependencies of the given pkgnames.
:param in_apkindexes: look through all APKINDEX files (with the specified arch)
:param in_aports: look through the aports folder
:param strict: raise RuntimeError, when a dependency can not be found.
"""
logging.debug("Calculate depends of packages " + str(pkgnames) +
", arch: " + arch)
logging.verbose("Search in_aports: " + str(in_aports) + ", in_apkindexes: " +
str(in_apkindexes))
# Sanity check
if not apkindex and not in_aports:
raise RuntimeError("Set at least one of apkindex or aports to True.")
todo = list(pkgnames)
ret = []
while len(todo):
# Skip already passed entries
pkgname = todo.pop(0)
if pkgname in ret:
continue
# Get depends
logging.verbose("Getting depends of single package: " + pkgname)
depends = None
if in_aports:
aport = pmb.build.find_aport(args, pkgname, False)
if aport:
logging.verbose("-> Found aport: " + aport)
apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD")
depends = apkbuild["depends"]
if depends is None and in_apkindexes:
logging.verbose("-> Search through APKINDEX files")
depends = apkindex(args, pkgname, arch)
if depends is None and strict:
raise RuntimeError(
recurse_error_message(
pkgname,
in_aports,
in_apkindexes))
# Append to todo/ret
logging.verbose("-> Depends: " + str(depends))
todo += depends
ret.append(pkgname)
return ret

11
pmbootstrap.py
View File

@ -62,7 +62,7 @@ def main():
if args.action == "aportgen":
pmb.aportgen.generate(args, args.package)
elif args.action == "build":
pmb.build.package(args, args.package, args.arch, args.force, False,
pmb.build.package(args, args.package, args.arch, args.force,
args.buildinfo)
elif args.action == "build_init":
pmb.build.init(args, args.suffix)
@ -84,8 +84,15 @@ def main():
elif args.action == "menuconfig":
pmb.build.menuconfig(args, args.package, args.deviceinfo["arch"])
elif args.action == "parse_apkbuild":
print(json.dumps(pmb.parse.apkbuild(args.aports + "/" +
print(json.dumps(pmb.parse.apkbuild(args, args.aports + "/" +
args.package + "/APKBUILD"), indent=4))
elif args.action == "parse_apkindex":
print(
json.dumps(
pmb.parse.apkindex.parse(
args,
args.apkindex_path),
indent=4))
elif args.action == "shutdown":
pmb.chroot.shutdown(args)
elif args.action == "stats":

4
test/test_apk_static.py
View File

@ -27,6 +27,7 @@ pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.chroot.apk_static
import pmb.parse.apkindex
import pmb.helpers.logging
@pytest.fixture
@ -34,7 +35,8 @@ def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args

119
test/test_aport_in_sync_with_git.py Normal file
View File

@ -0,0 +1,119 @@
"""
Copyright 2017 Oliver Smith
This file is part of pmbootstrap.
pmbootstrap is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
pmbootstrap is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
"""
import os
import sys
import pytest
# Import from parent directory
sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build.other
import pmb.chroot.apk
import pmb.chroot.root
import pmb.helpers.run
import pmb.helpers.logging
import pmb.helpers.git
@pytest.fixture
def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
def temp_aports_repo(args):
# Temp folder
temp = "/tmp/test_aport_in_sync_with_git"
temp_outside = args.work + "/chroot_native" + temp
if os.path.exists(temp_outside):
pmb.chroot.root(args, ["rm", "-rf", temp])
pmb.chroot.user(args, ["mkdir", temp])
# Create fake "aports" repo
# For this test to work, we need a git repository cloned from a real upstream
# location. It does not work, when cloned from the same file system. The
# apk-tools repo also gets used in test_versions.py, so we use that.
pmb.chroot.apk.install(args, ["git"])
pmb.helpers.git.clone(args, "apk-tools")
pmb.chroot.user(args, ["cp", "-r", "/home/user/git/apk-tools",
temp + "/aports"])
# Configure git
pmb.chroot.user(args, ["git", "config", "user.email", "user@localhost"],
working_dir=temp + "/aports")
pmb.chroot.user(args, ["git", "config", "user.name", "User"],
working_dir=temp + "/aports")
# Update args.aports
setattr(args, "aports", temp_outside + "/aports")
return temp + "/aports"
def out_of_sync_files(args):
"""
Clear the cache again (because when running pmbootstrap normally, we assume,
that the contents of the aports folder does not change during one run) and
return the files out of sync for the hello-world package.
"""
args.cache["aports_files_out_of_sync_with_git"] = None
return pmb.build.other.aports_files_out_of_sync_with_git(args,
"hello-world")
def test_aport_in_sync_with_git(args):
aports = temp_aports_repo(args)
ret_in_sync = []
ret_out_of_sync = [args.aports + "/hello-world/APKBUILD"]
# In sync (no files changed)
assert out_of_sync_files(args) == ret_in_sync
# Out of sync: untracked files
pmb.chroot.user(args, ["mkdir", aports + "/hello-world"])
pmb.chroot.user(args, ["touch", aports + "/hello-world/APKBUILD"])
assert out_of_sync_files(args) == ret_out_of_sync
# Out of sync: tracked files
pmb.chroot.user(args, ["git", "add", aports + "/hello-world/APKBUILD"],
working_dir=aports)
assert out_of_sync_files(args) == ret_out_of_sync
# Out of sync: comitted files
pmb.chroot.user(args, ["git", "commit", "-m", "test"], working_dir=aports)
assert out_of_sync_files(args) == ret_out_of_sync
# In sync: undo the commit and check out a new branch
pmb.chroot.user(args, ["git", "reset", "--hard", "origin/master"],
working_dir=aports)
pmb.chroot.user(args, ["git", "checkout", "-b", "pmbootstrap-testbranch"],
working_dir=aports)
assert out_of_sync_files(args) == ret_in_sync
# In sync: not a git repository
pmb.chroot.user(args, ["rm", "-rf", aports + "/.git"])
assert out_of_sync_files(args) == ret_in_sync
# TODO:
# - reinstall git, but rm .git, check again
# - remove temporary folder

6
test/test_aportgen.py
View File

@ -26,6 +26,7 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.aportgen
import pmb.config
import pmb.helpers.logging
@pytest.fixture
@ -33,10 +34,11 @@ def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
setattr(args, "_aports_real", args.aports)
args.aports = str(tmpdir)
request.addfinalizer(args.logfd.close)
return args

4
test/test_build.py
View File

@ -25,6 +25,7 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.aportgen
import pmb.config
import pmb.helpers.logging
@pytest.fixture
@ -32,7 +33,8 @@ def args(tmpdir, request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args

86
test/test_build_is_necessary.py
View File

@ -24,6 +24,7 @@ import pytest
sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build.other
import pmb.helpers.logging
@pytest.fixture
@ -31,12 +32,13 @@ def args(request, tmpdir):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
request.addfinalizer(args.logfd.close)
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
# Create an empty APKINDEX.tar.gz file, so we can use its path and
# timestamp to put test information in the cache.
setattr(args, "timestamp_based_rebuild", True)
apkindex_path = str(tmpdir) + "/APKINDEX.tar.gz"
open(apkindex_path, "a").close()
lastmod = os.path.getmtime(apkindex_path)
@ -44,9 +46,39 @@ def args(request, tmpdir):
return args
def cache_apkindex(args, version=None, timestamp=None):
"""
Modify the cache of the parsed binary package repository's APKINDEX
for the "hello-world" package.
The parameters version and timestamp are optional. If specified, they
change the string in the cache to the new value.
"""
apkindex_path = list(args.cache["apkindex"].keys())[0]
if version is not None:
args.cache["apkindex"][apkindex_path][
"ret"]["hello-world"]["version"] = version
if timestamp is not None:
args.cache["apkindex"][apkindex_path][
"ret"]["hello-world"]["timestamp"] = timestamp
def cache_files_out_of_sync(args, is_out_of_sync):
"""
Modify the cache, so the function aports_files_out_of_sync_with_git()
returns, that there are files out of sync for the "hello-world" package,
or not.
"""
new = []
if is_out_of_sync:
new = [os.path.abspath(args.aports + "/hello-world/APKBUILD")]
args.cache["aports_files_out_of_sync_with_git"] = new
def test_build_is_necessary(args):
# Prepare APKBUILD and APKINDEX data
apkbuild = pmb.parse.apkbuild(args.aports + "/hello-world/APKBUILD")
apkbuild = pmb.parse.apkbuild(args, args.aports + "/hello-world/APKBUILD")
apkbuild["pkgver"] = "1"
apkbuild["pkgrel"] = "2"
apkindex_path = list(args.cache["apkindex"].keys())[0]
@ -55,25 +87,47 @@ def test_build_is_necessary(args):
}
# a) Binary repo has a newer version
args.cache["apkindex"][apkindex_path]["ret"][
"hello-world"]["version"] = "999-r1"
cache_apkindex(args, version="999-r1")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
# b) Aports folder has a newer version
args.cache["apkindex"][apkindex_path][
"ret"]["hello-world"]["version"] = "0-r0"
cache_apkindex(args, version="0-r0")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
# c) Same version
args.cache["apkindex"][apkindex_path][
"ret"]["hello-world"]["version"] = "1-r2"
# c), d) Preparation: same version
cache_apkindex(args, version="1-r2")
# c.1) Newer timestamp in aport (timestamp in repo: 1970-01-01)
args.cache["apkindex"][apkindex_path][
"ret"]["hello-world"]["timestamp"] = "0"
# c) Out of sync sources, newer sources
cache_files_out_of_sync(args, True)
cache_apkindex(args, timestamp="0")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
# c.2) Newer timestamp in binary repo (timestamp in repo: 3000-01-01)
args.cache["apkindex"][apkindex_path]["ret"][
"hello-world"]["timestamp"] = "32503680000"
# Timestamp based rebuild deactivated
setattr(args, "timestamp_based_rebuild", False)
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
setattr(args, "timestamp_based_rebuild", True)
# d1) Out of sync sources, old sources
cache_files_out_of_sync(args, True)
cache_apkindex(args, timestamp="32503680000")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
# d2) Sources in sync, newer sources
cache_files_out_of_sync(args, False)
cache_apkindex(args, timestamp="0")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
# d3) Out of sync sources, old sources
cache_files_out_of_sync(args, False)
cache_apkindex(args, timestamp="32503680000")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
def test_build_is_necessary_no_binary_available(args):
"""
APKINDEX cache is set up to fake an empty APKINDEX, which means, that the
hello-world package has not been built yet.
"""
apkindex_path = list(args.cache["apkindex"].keys())[0]
apkbuild = pmb.parse.apkbuild(args, args.aports + "/hello-world/APKBUILD")
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True

4
test/test_challenge_apk.py
View File

@ -27,6 +27,7 @@ sys.path.append(os.path.abspath(
import pmb.challenge.apk_file
import pmb.config
import pmb.chroot.other
import pmb.helpers.logging
@pytest.fixture
@ -34,7 +35,8 @@ def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args

4
test/test_challenge_apkindex.py
View File

@ -25,6 +25,7 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.challenge.apkindex
import pmb.config
import pmb.helpers.logging
@pytest.fixture
@ -32,7 +33,8 @@ def args(request, tmpdir):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
# Create an empty APKINDEX.tar.gz file, so we can use its path and

9
test/test_challenge_build.py
View File

@ -25,8 +25,9 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.build.package
import pmb.challenge.build
import pmb.parse
import pmb.config
import pmb.helpers.logging
import pmb.parse
@pytest.fixture
@ -34,7 +35,8 @@ def args(request, tmpdir):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
@ -45,7 +47,8 @@ def test_challenge_build(args):
pmb.build.package(args, pkgname, None, force=True, buildinfo=True)
# Copy it to a temporary path
apkbuild = pmb.parse.apkbuild(args.aports + "/" + pkgname + "/APKBUILD")
apkbuild = pmb.parse.apkbuild(args, args.aports + "/" + pkgname +
"/APKBUILD")
version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
temp_path = pmb.chroot.other.tempfolder(args, "/tmp/test_challenge_build/" +
args.arch_native)

4
test/test_keys.py
View File

@ -27,6 +27,7 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.parse.apkindex
import pmb.helpers.git
import pmb.helpers.logging
@pytest.fixture
@ -34,7 +35,8 @@ def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args

42
test/test_repo.py
View File

@ -25,11 +25,24 @@ import time
# Import from parent directory
pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.build.package
import pmb.helpers.logging
import pmb.helpers.repo
@pytest.fixture
def args(request, tmpdir):
def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args
@pytest.fixture
def args_fake_work_dir(request, tmpdir):
args = types.SimpleNamespace()
args.work = str(tmpdir)
return args
@ -45,12 +58,14 @@ def clear_timestamps_from_files(files):
files[arch][file] = None
def test_files_empty(args):
def test_files_empty(args_fake_work_dir):
args = args_fake_work_dir
os.mkdir(args.work + "/packages")
assert pmb.helpers.repo.files(args) == {}
def test_files_not_empty(args):
def test_files_not_empty(args_fake_work_dir):
args = args_fake_work_dir
pkgs = args.work + "/packages"
for dir in ["", "armhf", "x86_64"]:
os.mkdir(pkgs + "/" + dir)
@ -60,7 +75,8 @@ def test_files_not_empty(args):
assert files == {"armhf": {}, "x86_64": {"test": None}}
def test_files_diff(args):
def test_files_diff(args_fake_work_dir):
args = args_fake_work_dir
# Create x86_64/test, x86_64/test2
pkgs = args.work + "/packages"
for dir in ["", "x86_64"]:
@ -85,3 +101,21 @@ def test_files_diff(args):
diff = pmb.helpers.repo.diff(args, first)
assert diff == ["aarch64/test3", "x86_64/test", "x86_64/test4"]
def test_hash():
url = "https://nl.alpinelinux.org/alpine/edge/testing"
hash = "865a153c"
assert pmb.helpers.repo.hash(url, 8) == hash
def test_apkindex_files(args):
# Make sure, that we have a user's APKINDEX.tar.gz
pmb.build.package(args, "hello-world", args.arch_native)
files = pmb.helpers.repo.apkindex_files(args)
for file in files:
assert os.path.exists(file)
# Test cache
assert files == pmb.helpers.repo.apkindex_files(args)

6
test/test_shell_escape.py
View File

@ -23,9 +23,10 @@ import pytest
# Import from parent directory
pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/.."))
sys.path.append(pmb_src)
import pmb.helpers.run
import pmb.chroot.root
import pmb.chroot.user
import pmb.helpers.run
import pmb.helpers.logging
@pytest.fixture
@ -33,7 +34,8 @@ def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args

4
test/test_version.py
View File

@ -25,6 +25,7 @@ sys.path.append(os.path.abspath(
os.path.join(os.path.dirname(__file__) + "/..")))
import pmb.parse.apkindex
import pmb.helpers.git
import pmb.helpers.logging
@pytest.fixture
@ -32,7 +33,8 @@ def args(request):
import pmb.parse
sys.argv = ["pmbootstrap.py", "chroot"]
args = pmb.parse.arguments()
setattr(args, "logfd", open("/dev/null", "a+"))
args.log = args.work + "/log_testsuite.txt"
pmb.helpers.logging.init(args)
request.addfinalizer(args.logfd.close)
return args