From 51bdc24315edfcccafcdd887290048b511c21590 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Mon, 10 Jul 2017 15:23:43 +0000 Subject: [PATCH] Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129) TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution. --- pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has. I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository. To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on. In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option. In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds". This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before. Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases). The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway. --- pmb/build/buildinfo.py | 42 ++------- pmb/build/menuconfig.py | 2 +- pmb/build/other.py | 129 +++++++++++++++++++++++++--- pmb/build/package.py | 11 +-- pmb/chroot/apk.py | 108 +++++++++++++++++++---- pmb/chroot/init.py | 12 +-- pmb/chroot/initfs.py | 5 ++ pmb/chroot/initfs_hooks.py | 14 ++- pmb/chroot/root.py | 3 +- pmb/chroot/zap.py | 2 +- pmb/config/__init__.py | 1 + pmb/config/init.py | 11 ++- pmb/config/load.py | 2 +- pmb/helpers/devices.py | 2 +- pmb/helpers/logging.py | 37 ++++++-- pmb/helpers/repo.py | 72 ++++++++++++++++ pmb/helpers/run.py | 23 +++-- pmb/parse/__init__.py | 2 +- pmb/parse/apkbuild.py | 8 +- pmb/parse/apkindex.py | 23 ++--- pmb/parse/arguments.py | 27 ++++-- pmb/parse/depends.py | 107 +++++++++++++++++++++++ pmbootstrap.py | 11 ++- test/test_apk_static.py | 4 +- test/test_aport_in_sync_with_git.py | 119 +++++++++++++++++++++++++ test/test_aportgen.py | 6 +- test/test_build.py | 4 +- test/test_build_is_necessary.py | 86 +++++++++++++++---- test/test_challenge_apk.py | 4 +- test/test_challenge_apkindex.py | 4 +- test/test_challenge_build.py | 9 +- test/test_keys.py | 4 +- test/test_repo.py | 42 ++++++++- test/test_shell_escape.py | 6 +- test/test_version.py | 4 +- 35 files changed, 784 insertions(+), 162 deletions(-) create mode 100644 pmb/parse/depends.py create mode 100644 test/test_aport_in_sync_with_git.py diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py index b812a697..dd8b9253 100644 --- a/pmb/build/buildinfo.py +++ b/pmb/build/buildinfo.py @@ -18,43 +18,10 @@ along with pmbootstrap. If not, see . """ import os import json -import logging import pmb.chroot import pmb.chroot.apk import pmb.parse.apkindex - - -def get_depends_recursively(args, pkgnames, arch=None): - """ - :param pkgnames: List of pkgnames, for which the dependencies shall be - retrieved. - """ - todo = list(pkgnames) - ret = [] - seen = [] - while len(todo): - pkgname = todo.pop(0) - index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch) - if not index_data: - logging.debug( - "NOTE: Could not find dependency " + - pkgname + - " in any APKINDEX.") - continue - pkgname = index_data["pkgname"] - if pkgname not in pkgnames and pkgname not in ret: - ret.append(pkgname) - for depend in index_data["depends"]: - if depend not in ret: - if depend.startswith("!"): - continue - for operator in [">", "="]: - if operator in depend: - depend = depend.split(operator)[0] - if depend not in seen: - todo.append(depend) - seen.append(depend) - return ret +import pmb.parse.depends def generate(args, apk_path, arch, suffix, apkbuild): @@ -71,9 +38,12 @@ def generate(args, apk_path, arch, suffix, apkbuild): # Add makedepends versions installed = pmb.chroot.apk.installed(args, suffix) - relevant = (apkbuild["makedepends"] + - get_depends_recursively(args, [apkbuild["pkgname"], "abuild", "build-base"])) + relevant = (apkbuild["makedepends"] + [apkbuild["pkgname"], "abuild", + "build-base"]) + relevant = pmb.parse.depends.recurse(args, relevant, arch, in_aports=False) for pkgname in relevant: + if pkgname == apkbuild["pkgname"]: + continue if pkgname in installed: ret["versions"][pkgname] = installed[pkgname]["version"] return ret diff --git a/pmb/build/menuconfig.py b/pmb/build/menuconfig.py index b5e63d29..43364027 100644 --- a/pmb/build/menuconfig.py +++ b/pmb/build/menuconfig.py @@ -33,7 +33,7 @@ def menuconfig(args, pkgname, arch): aport = pmb.build.find_aport(args, pkgname, False) if not aport: raise RuntimeError("Package " + pkgname + ": Could not find aport!") - apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD") + apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") # Set up build tools and makedepends pmb.build.init(args) diff --git a/pmb/build/other.py b/pmb/build/other.py index bffd5e19..3057df5f 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -19,6 +19,7 @@ along with pmbootstrap. If not, see . import os import logging import glob +import shutil import pmb.chroot import pmb.helpers.run @@ -37,14 +38,23 @@ def find_aport(args, package, must_exist=True): if os.path.exists(path): return path + # Try to get a cached result first (we assume, that the aports don't change + # in one pmbootstrap call) + if path in args.cache["find_aport"]: + return args.cache["find_aport"][path] + + ret = None for path_current in glob.glob(args.aports + "/*/APKBUILD"): - apkbuild = pmb.parse.apkbuild(path_current) + apkbuild = pmb.parse.apkbuild(args, path_current) if package in apkbuild["subpackages"]: - return os.path.dirname(path_current) - if must_exist: + ret = os.path.dirname(path_current) + break + if ret is None and must_exist: raise RuntimeError("Could not find aport for package: " + package) - return None + + args.cache["find_aport"][path] = ret + return ret def copy_to_buildpath(args, package, suffix="native"): @@ -66,6 +76,83 @@ def copy_to_buildpath(args, package, suffix="native"): "/home/user/build"], suffix=suffix) +def aports_files_out_of_sync_with_git(args, package=None): + """ + Get a list of files, about which git says, that they have changed in + comparison to upstream. We need this for the timestamp based rebuild check, + where it does not only rely on the APKBUILD pkgver and pkgrel, but also on + the file's last modified date to decide if it needs to be rebuilt. Git sets + the last modified timestamp to the last checkout date, so we must ignore + all files, that have not been modified, or else we would trigger rebuilds + for all packages, from the pmOS binary repository. + + :returns: list of absolute paths to all files not in sync with upstream + """ + + # Filter out a specific package + if package: + ret = [] + prefix = os.path.abspath(args.aports + "/" + package + "/") + for file in aports_files_out_of_sync_with_git(args): + if file.startswith(prefix): + ret.append(file) + return ret + + # Use cached result if possible + if args.cache["aports_files_out_of_sync_with_git"] is not None: + return args.cache["aports_files_out_of_sync_with_git"] + + # Get the aport's git repository folder + git_root = None + if shutil.which("git"): + git_root = pmb.helpers.run.user(args, ["git", "rev-parse", + "--show-toplevel"], + working_dir=args.aports, + return_stdout=True, + check=False) + if git_root: + git_root = git_root.rstrip() + ret = [] + if git_root and os.path.exists(git_root): + # Find tracked files out of sync with upstream + tracked = pmb.helpers.run.user(args, ["git", "diff", "--name-only", "origin"], + working_dir=git_root, return_stdout=True) + + # Find all untracked files + untracked = pmb.helpers.run.user( + args, ["git", "ls-files", "--others", "--exclude-standard"], + working_dir=git_root, return_stdout=True) + + # Set absolute path, filter out aports files + aports_absolute = os.path.abspath(args.aports) + files = tracked.rstrip().split("\n") + untracked.rstrip().split("\n") + for file in files: + file = os.path.abspath(git_root + "/" + file) + if file.startswith(aports_absolute): + ret.append(file) + else: + logging.warning("WARNING: Can not determine, which aport-files have been" + " changed from upstream!") + logging.info("* Aports-folder is not a git repository or git is not" + " installed") + logging.info("* You can turn timestamp-based rebuilds off in" + " 'pmbootstrap init'") + + # Save cache + args.cache["aports_files_out_of_sync_with_git"] = ret + return ret + + +def sources_newer_than_binary_package(args, package, index_data): + path_sources = [] + for file in glob.glob(args.aports + "/" + package + "/*"): + path_sources.append(file) + + lastmod_target = float(index_data["timestamp"]) + return not pmb.helpers.file.is_up_to_date(path_sources, + lastmod_target=lastmod_target) + + def is_necessary(args, arch, apkbuild, apkindex_path=None): """ Check if the package has already been built. Compared to abuild's check, @@ -78,9 +165,10 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None): :param apkindex_path: override the APKINDEX.tar.gz path :returns: boolean """ - # Get new version from APKBUILD + # Get package name, version, define start of debug message package = apkbuild["pkgname"] version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] + msg = "Build is neccessary for package '" + package + "': " # Get old version from APKINDEX if apkindex_path: @@ -89,6 +177,7 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None): else: index_data = pmb.parse.apkindex.read_any_index(args, package, arch) if not index_data: + logging.debug(msg + "No binary package available") return True # a) Binary repo has a newer version @@ -102,17 +191,31 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None): # b) Aports folder has a newer version if version_new != version_old: + logging.debug(msg + "Binary package out of date (binary: " + version_old + + ", aport: " + version_new + ")") return True - # c) The version is the same. Check if all files in the aport folder have an - # older timestamp, than the package. This way the pkgrel doesn't need to be - # increased while developing locally. - lastmod_target = float(index_data["timestamp"]) - path_sources = glob.glob(args.aports + "/" + package + "/*") - if pmb.helpers.file.is_up_to_date( - path_sources, lastmod_target=lastmod_target): + # Aports and binary repo have the same version. + if not args.timestamp_based_rebuild: + return False + + # c) Same version, source files out of sync with upstream, source + # files newer than binary package + files_out_of_sync = aports_files_out_of_sync_with_git(args, package) + sources_newer = sources_newer_than_binary_package( + args, package, index_data) + if len(files_out_of_sync) and sources_newer: + logging.debug(msg + "Binary package and aport have the same pkgver and" + " pkgrel, but there are aport source files out of sync" + " with the upstream git repository *and* these source" + " files have a more recent 'last modified' timestamp than" + " the binary package's build timestamp.") + return True + + # d) Same version, source files *in sync* with upstream *or* source + # files *older* than binary package + else: return False - return True def index_repo(args, arch=None): diff --git a/pmb/build/package.py b/pmb/build/package.py index d7dc5828..f029992b 100644 --- a/pmb/build/package.py +++ b/pmb/build/package.py @@ -30,7 +30,7 @@ import pmb.parse import pmb.parse.arch -def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False): +def package(args, pkgname, carch, force=False, buildinfo=False): """ Build a package with Alpine Linux' abuild. @@ -46,24 +46,19 @@ def package(args, pkgname, carch, force=False, recurse=True, buildinfo=False): " and could not find this package in any APKINDEX!") # Autodetect the build environment - apkbuild = pmb.parse.apkbuild(aport + "/APKBUILD") + apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") pkgname = apkbuild["pkgname"] carch_buildenv = pmb.build.autodetect.carch(args, apkbuild, carch) suffix = pmb.build.autodetect.suffix(args, apkbuild, carch_buildenv) cross = pmb.build.autodetect.crosscompile(args, apkbuild, carch_buildenv, suffix) - # Build dependencies first (they may be outdated, even if they exist) - if recurse: - for depend in apkbuild["depends"]: - package(args, depend, carch) - # Skip already built versions if not force and not pmb.build.is_necessary( args, carch_buildenv, apkbuild): return - # Install build tools and makedepends + # Initialize build environment, install/build makedepends pmb.build.init(args, suffix) if len(apkbuild["makedepends"]): pmb.chroot.apk.install(args, apkbuild["makedepends"], suffix) diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index a4547681..21367589 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -21,6 +21,8 @@ import logging import pmb.chroot import pmb.config import pmb.parse.apkindex +import pmb.parse.arch +import pmb.parse.depends def check_min_version(args, suffix="native"): @@ -54,41 +56,110 @@ def check_min_version(args, suffix="native"): args.cache["apk_min_version_checked"].append(suffix) +def install_is_necessary(args, build, arch, package, packages_installed): + """ + This function optionally builds an out of date package, and checks if the + version installed inside a chroot is up to date. + :param build: Set to true to build the package, if the binary packages are + out of date, and it is in the aports folder. + :param packages_installed: Return value from installed(). + :returns: True if the package needs to be installed/updated, False otherwise. + """ + # Build package + if build: + pmb.build.package(args, package, arch) + + # No further checks when not installed + if package not in packages_installed: + return True + + # Compare the installed version vs. the version in the repos + data_installed = packages_installed[package] + data_repo = pmb.parse.apkindex.read_any_index(args, package, arch) + compare = pmb.parse.apkindex.compare_version(data_installed["version"], + data_repo["version"]) + # a) Installed newer (should not happen normally) + if compare == 1: + logging.info("WARNING: " + arch + " package '" + package + + "' installed version " + data_installed["version"] + + " is newer, than the version in the repositories: " + + data_repo["version"]) + return False + + # b) Repo newer + elif compare == -1: + return True + + # c) Same version, look at last modified + elif compare == 0: + time_installed = float(data_installed["timestamp"]) + time_repo = float(data_repo["timestamp"]) + return time_repo > time_installed + + +def replace_aports_packages_with_path(args, packages, suffix, arch): + """ + apk will only re-install packages with the same pkgname, pkgver and pkgrel, + when you give it the absolute path to the package. This function replaces + all packages, that were built locally, with the absolute path to the package. + """ + ret = [] + for package in packages: + aport = pmb.build.find_aport(args, package, False) + if aport: + apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") + apk_path = ("/home/user/packages/user/" + arch + "/" + + package + "-" + apkbuild["pkgver"] + "-r" + + apkbuild["pkgrel"] + ".apk") + if os.path.exists(args.work + "/chroot_" + suffix + apk_path): + package = apk_path + ret.append(package) + return ret + + def install(args, packages, suffix="native", build=True): """ :param build: automatically build the package, when it does not exist yet - and it is inside the pm-aports folder. Checking this is expensive - if - you know, that all packages are provides by upstream repos, set this to - False! + or needs to be updated, and it is inside the pm-aports + folder. Checking this is expensive - if you know, that all + packages are provides by upstream repos, set this to False! """ # Initialize chroot check_min_version(args, suffix) pmb.chroot.init(args, suffix) - # Filter already installed packages + # Add depends to packages + arch = pmb.parse.arch.from_chroot_suffix(args, suffix) + packages_with_depends = pmb.parse.depends.recurse(args, packages, arch, + strict=True) + + # Filter out up-to-date packages packages_installed = installed(args, suffix) packages_todo = [] - for package in packages: - if package not in packages_installed: + for package in packages_with_depends: + if install_is_necessary( + args, build, arch, package, packages_installed): packages_todo.append(package) if not len(packages_todo): return - # Build packages if necessary - arch = pmb.parse.arch.from_chroot_suffix(args, suffix) - if build: - for package in packages_todo: - pmb.build.package(args, package, arch) - # Sanitize packages: don't allow '--allow-untrusted' and other options # to be passed to apk! for package in packages_todo: if package.startswith("-"): raise ValueError("Invalid package name: " + package) - # Install everything - logging.info("(" + suffix + ") install " + " ".join(packages_todo)) - pmb.chroot.root(args, ["apk", "--no-progress", "add"] + packages_todo, + # Readable install message without dependencies + message = "(" + suffix + ") install" + for pkgname in packages: + if pkgname not in packages_installed: + message += " " + pkgname + logging.info(message) + + # Install/update everything + packages_todo = replace_aports_packages_with_path(args, packages_todo, + suffix, arch) + pmb.chroot.root(args, ["apk", "--no-progress", "add", "-u"] + packages_todo, suffix) @@ -96,19 +167,22 @@ def upgrade(args, suffix="native", update_index=True): """ Upgrade all packages installed in a chroot """ + # Prepare apk and update index check_min_version(args, suffix) pmb.chroot.init(args, suffix) if update_index: pmb.chroot.root(args, ["apk", "update"], suffix) - # -a: also update previously downgraded (and therefore pinned) packages - pmb.chroot.root(args, ["apk", "upgrade", "-a"], suffix) + # Rebuild and upgrade out-of-date packages + packages = installed(args, suffix).keys() + install(args, packages, suffix) def installed(args, suffix="native"): """ Read the list of installed packages (which has almost the same format, as an APKINDEX, but with more keys). + :returns: a dictionary with the following structure: { "postmarketos-mkinitfs": { diff --git a/pmb/chroot/init.py b/pmb/chroot/init.py index 8f1da90c..48a0be5f 100644 --- a/pmb/chroot/init.py +++ b/pmb/chroot/init.py @@ -25,6 +25,7 @@ import filecmp import pmb.chroot import pmb.chroot.apk_static import pmb.config +import pmb.helpers.repo import pmb.helpers.run import pmb.parse.arch @@ -78,16 +79,7 @@ def init(args, suffix="native"): # Write /etc/apk/repositories repos_path = chroot + "/etc/apk/repositories" if not os.path.exists(repos_path): - lines = ["/home/user/packages/user"] - if args.mirror_postmarketos: - lines.append(args.mirror_postmarketos) - directories = ["main", "community"] - if args.alpine_version == "edge": - directories.append("testing") - for dir in directories: - lines.append(args.mirror_alpine + args.alpine_version + - "/" + dir) - for line in lines: + for line in pmb.helpers.repo.urls(args): pmb.helpers.run.root(args, ["sh", "-c", "echo " + shlex.quote(line) + " >> " + repos_path]) diff --git a/pmb/chroot/initfs.py b/pmb/chroot/initfs.py index 62bfe577..b0956a2a 100644 --- a/pmb/chroot/initfs.py +++ b/pmb/chroot/initfs.py @@ -25,6 +25,11 @@ import pmb.helpers.cli def build(args, flavor, suffix): + # Update mkinitfs and hooks + pmb.chroot.apk.install(args, ["postmarketos-mkinitfs"], suffix) + pmb.chroot.initfs_hooks.update(args, suffix) + + # Call mkinitfs logging.info("(" + suffix + ") mkinitfs " + flavor) release_file = (args.work + "/chroot_" + suffix + "/usr/share/kernel/" + flavor + "/kernel.release") diff --git a/pmb/chroot/initfs_hooks.py b/pmb/chroot/initfs_hooks.py index 22a10e91..49507095 100644 --- a/pmb/chroot/initfs_hooks.py +++ b/pmb/chroot/initfs_hooks.py @@ -24,12 +24,15 @@ import pmb.config import pmb.chroot.apk -def list_chroot(args, suffix): +def list_chroot(args, suffix, remove_prefix=True): ret = [] prefix = pmb.config.initfs_hook_prefix for pkgname in pmb.chroot.apk.installed(args, suffix): if pkgname.startswith(prefix): - ret.append(pkgname[len(prefix):]) + if remove_prefix: + ret.append(pkgname[len(prefix):]) + else: + ret.append(pkgname) return ret @@ -65,3 +68,10 @@ def delete(args, hook, suffix): raise RuntimeError("There is no such hook installed!") prefix = pmb.config.initfs_hook_prefix pmb.chroot.root(args, ["apk", "del", prefix + hook], suffix) + + +def update(args, suffix): + """ + Rebuild and update all hooks, that are out of date + """ + pmb.chroot.apk.install(args, list_chroot(args, suffix, False), suffix) diff --git a/pmb/chroot/root.py b/pmb/chroot/root.py index dbf8b9c6..a4e717e9 100644 --- a/pmb/chroot/root.py +++ b/pmb/chroot/root.py @@ -53,7 +53,8 @@ def root(args, cmd, suffix="native", working_dir="/", log=True, if not auto_init and not os.path.islink(chroot + "/bin/sh"): raise RuntimeError("Chroot does not exist: " + chroot) - pmb.chroot.init(args, suffix) + if auto_init: + pmb.chroot.init(args, suffix) # Run the args with sudo chroot, and with cleaned environment # variables diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index b87dc3a8..3b6d3c80 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -28,7 +28,7 @@ def zap(args): patterns = [ "chroot_native", "chroot_buildroot_*", - "chroot_rootfs_" + args.device, + "chroot_rootfs_*", ] # Only ask for removal, if the user specificed the extra '-p' switch. diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 76f59e02..0b333231 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -46,6 +46,7 @@ defaults = { "aports": os.path.normpath(pmb_src + "/aports"), "config": os.path.expanduser("~") + "/.config/pmbootstrap.cfg", "device": "samsung-i9100", + "timestamp_based_rebuild": True, "log": "$WORK/log.txt", "mirror_alpine": "https://nl.alpinelinux.org/alpine/", "mirror_postmarketos": "", diff --git a/pmb/config/init.py b/pmb/config/init.py index b0f897e0..e8bfae5d 100644 --- a/pmb/config/init.py +++ b/pmb/config/init.py @@ -52,6 +52,16 @@ def init(args): " compiling?") cfg["pmbootstrap"]["jobs"] = pmb.helpers.cli.ask(args, "Jobs", None, default) + # Timestamp based rebuilds + default = "y" + if not args.timestamp_based_rebuild: + default = "n" + logging.info("Rebuild packages, when the last modified timestamp changed," + " even if the version did not change? This makes pmbootstrap" + " behave more like 'make'.") + answer = pmb.helpers.cli.ask(args, "Timestamp based rebuilds", + default=default) + cfg["pmbootstrap"]["timestamp_based_rebuild"] = str(answer == "y") # Save config pmb.config.save(args, cfg) @@ -61,5 +71,4 @@ def init(args): logging.info("Run 'pmbootstrap zap' to delete all chroots once a day before" " working with pmbootstrap!") logging.info("It only takes a few seconds, and all packages are cached.") - logging.info("Done!") diff --git a/pmb/config/load.py b/pmb/config/load.py index 4deb6e20..e312e41d 100644 --- a/pmb/config/load.py +++ b/pmb/config/load.py @@ -31,6 +31,6 @@ def load(args): for key in pmb.config.defaults: if key not in cfg["pmbootstrap"]: - cfg["pmbootstrap"][key] = pmb.config.defaults[key] + cfg["pmbootstrap"][key] = str(pmb.config.defaults[key]) return cfg diff --git a/pmb/helpers/devices.py b/pmb/helpers/devices.py index 0c523570..5a49c81c 100644 --- a/pmb/helpers/devices.py +++ b/pmb/helpers/devices.py @@ -40,5 +40,5 @@ def list_apkbuilds(args): ret = {} for device in list(args): apkbuild_path = args.aports + "/device-" + device + "/APKBUILD" - ret[device] = pmb.parse.apkbuild(apkbuild_path) + ret[device] = pmb.parse.apkbuild(args, apkbuild_path) return ret diff --git a/pmb/helpers/logging.py b/pmb/helpers/logging.py index bf7de538..2b89bdc9 100644 --- a/pmb/helpers/logging.py +++ b/pmb/helpers/logging.py @@ -48,28 +48,47 @@ class log_handler(logging.StreamHandler): self.handleError(record) +def add_verbose_log_level(): + """ + Add a new log level "verbose", which is below "debug". Also monkeypatch + logging, so it can be used with logging.verbose(). + + This function is based on work by Voitek Zylinski and sleepycal: + https://stackoverflow.com/a/20602183 + All stackoverflow user contributions are licensed as CC-BY-SA: + https://creativecommons.org/licenses/by-sa/3.0/ + """ + logging.VERBOSE = 5 + logging.addLevelName(logging.VERBOSE, "VERBOSE") + logging.Logger.verbose = lambda inst, msg, * \ + args, **kwargs: inst.log(logging.VERBOSE, msg, *args, **kwargs) + logging.verbose = lambda msg, *args, **kwargs: logging.log(logging.VERBOSE, msg, + *args, **kwargs) + + def init(args): """ - Set log format and add the log file descriptor to args.logfd. + Set log format and add the log file descriptor to args.logfd, add the + verbose log level. """ + # Open logfile if not os.path.exists(args.work): os.makedirs(args.work) - - date_format = "%H:%M:%S" setattr(args, "logfd", open(args.log, "a+")) + # Set log format root_logger = logging.getLogger() root_logger.handlers = [] + formatter = logging.Formatter("[%(asctime)s] %(message)s", + datefmt="%H:%M:%S") - formatter = None + # Set log level + add_verbose_log_level() root_logger.setLevel(logging.DEBUG) if args.verbose: - formatter = logging.Formatter("[%(asctime)s %(module)s]" - " %(message)s", datefmt=date_format) - else: - formatter = logging.Formatter("[%(asctime)s] %(message)s", - datefmt=date_format) + root_logger.setLevel(logging.VERBOSE) + # Add a custom log handler handler = log_handler() log_handler._args = args handler.setFormatter(formatter) diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 92312ea3..60816bf7 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -18,6 +18,7 @@ along with pmbootstrap. If not, see . """ import glob import os +import hashlib def files(args): @@ -64,3 +65,74 @@ def diff(args, files_a, files_b=None): ret.append(arch + "/" + file) return sorted(ret) + + +def hash(url, length=8): + """ + Generate the hash, that APK adds to the APKINDEX and apk packages + in its apk cache folder. It is the "12345678" part in this example: + "APKINDEX.12345678.tar.gz". + + :param length: The length of the hash in the output file. + + See also: official implementation in apk-tools: + + + blob.c: apk_blob_push_hexdump(), "const char *xd" + apk_defines.h: APK_CACHE_CSUM_BYTES + database.c: apk_repo_format_cache_index() + """ + binary = hashlib.sha1(url.encode("utf-8")).digest() + xd = "0123456789abcdefghijklmnopqrstuvwxyz" + csum_bytes = int(length / 2) + + ret = "" + for i in range(csum_bytes): + ret += xd[(binary[i] >> 4) & 0xf] + ret += xd[binary[i] & 0xf] + + return ret + + +def urls(args, user_repository=True): + """ + Get a list of repository URLs, as they are in /etc/apk/repositories. + """ + ret = [] + # Local user repository (for packages compiled with pmbootstrap) + if user_repository: + ret.append("/home/user/packages/user") + + # Upstream postmarketOS binary repository + if args.mirror_postmarketos: + ret.append(args.mirror_postmarketos) + + # Upstream Alpine Linux repositories + directories = ["main", "community"] + if args.alpine_version == "edge": + directories.append("testing") + for dir in directories: + ret.append(args.mirror_alpine + args.alpine_version + "/" + dir) + return ret + + +def apkindex_files(args, arch="native"): + """ + Get a list of outside paths to all resolved APKINDEX.tar.gz files + from the urls() list for a specific arch. + """ + if arch == "native": + arch = args.arch_native + + # Try to get a cached result first. + if arch in args.cache["apkindex_files"]: + return args.cache["apkindex_files"][arch] + + # Add the non-hashed user path and the upstream paths with hashes + ret = [args.work + "/packages/" + arch + "/APKINDEX.tar.gz"] + for url in urls(args, False): + ret.append(args.work + "/cache_apk_" + arch + "/APKINDEX." + + hash(url) + ".tar.gz") + + args.cache["apkindex_files"][arch] = ret + return ret diff --git a/pmb/helpers/run.py b/pmb/helpers/run.py index e9b419a9..cc6c6b4d 100644 --- a/pmb/helpers/run.py +++ b/pmb/helpers/run.py @@ -18,9 +18,11 @@ along with pmbootstrap. If not, see . """ import subprocess import logging +import os -def core(args, cmd, log_message, log, return_stdout, check=True): +def core(args, cmd, log_message, log, return_stdout, check=True, + working_dir=None): logging.debug(log_message) """ Run the command and write the output to the log. @@ -28,6 +30,10 @@ def core(args, cmd, log_message, log, return_stdout, check=True): :param check: raise an exception, when the command fails """ + if working_dir: + working_dir_old = os.getcwd() + os.chdir(working_dir) + try: ret = None if log: @@ -52,19 +58,22 @@ def core(args, cmd, log_message, log, return_stdout, check=True): raise RuntimeError("Command failed: " + log_message) from exc else: pass + + if working_dir: + os.chdir(working_dir_old) return ret def user(args, cmd, log=True, working_dir=None, return_stdout=False, check=True): - """ - :param working_dir: defaults to args.work - """ - if not working_dir: - working_dir = args.work + + if working_dir: + msg = "% cd " + working_dir + " && " + " ".join(cmd) + else: + msg = "% " + " ".join(cmd) # TODO: maintain and check against a whitelist - return core(args, cmd, "% " + " ".join(cmd), log, return_stdout, check) + return core(args, cmd, msg, log, return_stdout, check, working_dir) def root(args, cmd, log=True, working_dir=None, return_stdout=False, diff --git a/pmb/parse/__init__.py b/pmb/parse/__init__.py index e71c5ece..87522a42 100644 --- a/pmb/parse/__init__.py +++ b/pmb/parse/__init__.py @@ -18,6 +18,6 @@ along with pmbootstrap. If not, see . """ from pmb.parse.arguments import arguments from pmb.parse.apkbuild import apkbuild -from pmb.parse.deviceinfo import deviceinfo from pmb.parse.binfmt_info import binfmt_info +from pmb.parse.deviceinfo import deviceinfo import pmb.parse.arch diff --git a/pmb/parse/apkbuild.py b/pmb/parse/apkbuild.py index 9af6fe89..6827280d 100644 --- a/pmb/parse/apkbuild.py +++ b/pmb/parse/apkbuild.py @@ -69,7 +69,7 @@ def cut_off_function_names(apkbuild): return apkbuild -def apkbuild(path): +def apkbuild(args, path): """ Parse relevant information out of the APKBUILD file. This is not meant to be perfect and catch every edge case (for that, a full shell parser @@ -80,6 +80,11 @@ def apkbuild(path): :returns: Relevant variables from the APKBUILD. Arrays get returned as arrays. """ + # Try to get a cached result first (we assume, that the aports don't change + # in one pmbootstrap call) + if path in args.cache["apkbuild"]: + return args.cache["apkbuild"][path] + with open(path, encoding="utf-8") as handle: lines = handle.readlines() @@ -122,4 +127,5 @@ def apkbuild(path): ret = replace_variables(ret) ret = cut_off_function_names(ret) + args.cache["apkbuild"][path] = ret return ret diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index 984703f9..e831a6d0 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ import distutils.version -import glob +import logging import os import tarfile +import pmb.chroot.apk +import pmb.helpers.repo def compare_version(a_str, b_str): @@ -118,8 +120,7 @@ def parse_next_block(args, path, lines, start): return None -def parse_add_block(path, strict, ret, block, pkgname=None, - version_new=None): +def parse_add_block(path, strict, ret, block, pkgname=None): """ Add one block to the return dictionary of parse(). @@ -140,8 +141,6 @@ def parse_add_block(path, strict, ret, block, pkgname=None, # Defaults if not pkgname: pkgname = block["pkgname"] - if not version_new: - version_new = block["version"] # Handle duplicate entries if pkgname in ret: @@ -151,6 +150,7 @@ def parse_add_block(path, strict, ret, block, pkgname=None, # Ignore the block, if the block we already have has a higher # version version_old = ret[pkgname]["version"] + version_new = block["version"] if compare_version(version_old, version_new) == 1: return @@ -207,8 +207,7 @@ def parse(args, path, strict=False): for alias in block["provides"]: split = alias.split("=") if len(split) == 2: - parse_add_block(path, strict, ret, block, split[0], - split[1]) + parse_add_block(path, strict, ret, block, split[0]) # Update the cache args.cache["apkindex"][path] = {"lastmod": lastmod, "ret": ret} @@ -254,12 +253,14 @@ def read_any_index(args, package, arch=None): """ if not arch: arch = args.arch_native - indexes = [args.work + "/packages/" + arch + "/APKINDEX.tar.gz"] - pattern = args.work + "/cache_apk_" + arch + "/APKINDEX.*.tar.gz" - indexes += glob.glob(pattern) - for index in indexes: + # Return first match + for index in pmb.helpers.repo.apkindex_files(args, arch): index_data = read(args, package, index, False) + logging.verbose("Search for " + package + " in " + index + + " - result: " + str(index_data)) if index_data: return index_data + + logging.verbose("No match found in any APKINDEX.tar.gz!") return None diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index d9506482..56fc05ae 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -107,8 +107,8 @@ def arguments(): # Logging parser.add_argument("-l", "--log", dest="log", default=None) parser.add_argument("-v", "--verbose", dest="verbose", - action="store_true", help="output the source file, where the log" - " message originated from with each log message") + action="store_true", help="write even more to the" + "logfile (this may reduce performance)") parser.add_argument("-q", "--quiet", dest="quiet", action="store_true", help="do not output any log messages") @@ -195,23 +195,38 @@ def arguments(): " .apk, or must be named" " APKINDEX.tar.gz.") + # Action: parse_apkindex + parse_apkindex = sub.add_parser("parse_apkindex") + parse_apkindex.add_argument("apkindex_path") + # Use defaults from the user's config file args = parser.parse_args() cfg = pmb.config.load(args) for varname in cfg["pmbootstrap"]: if varname not in args or not getattr(args, varname): - setattr(args, varname, cfg["pmbootstrap"][varname]) + value = cfg["pmbootstrap"][varname] + if varname in pmb.config.defaults: + default = pmb.config.defaults[varname] + if isinstance(default, bool): + value = (value.lower() == "true") + setattr(args, varname, value) # Replace $WORK in variables from user's config for varname in cfg["pmbootstrap"]: old = getattr(args, varname) - setattr(args, varname, old.replace("$WORK", args.work)) + if isinstance(old, str): + setattr(args, varname, old.replace("$WORK", args.work)) # Add convenience shortcuts setattr(args, "arch_native", pmb.parse.arch.alpine_native()) - # Add a caching dict - setattr(args, "cache", {"apkindex": {}, "apk_min_version_checked": []}) + # Add a caching dict (caches parsing of files etc. for the current session) + setattr(args, "cache", {"apkindex": {}, + "apkindex_files": {}, + "apkbuild": {}, + "apk_min_version_checked": [], + "aports_files_out_of_sync_with_git": None, + "find_aport": {}}) # Add and verify the deviceinfo (only after initialization) if args.action != "init": diff --git a/pmb/parse/depends.py b/pmb/parse/depends.py new file mode 100644 index 00000000..878707c0 --- /dev/null +++ b/pmb/parse/depends.py @@ -0,0 +1,107 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import pmb.chroot +import pmb.chroot.apk +import pmb.parse.apkindex + + +def apkindex(args, pkgname, arch): + """ + Non-recursively get the dependencies of one package in any APKINDEX. + """ + index_data = pmb.parse.apkindex.read_any_index(args, pkgname, arch) + if not index_data: + return None + + # Remove operators from the depends list + ret = [] + for depend in index_data["depends"]: + if depend.startswith("!"): + continue + for operator in [">", "="]: + if operator in depend: + depend = depend.split(operator)[0] + if depend not in ret: + ret.append(depend) + return ret + + +def recurse_error_message(pkgname, in_aports, in_apkindexes): + ret = "Could not find package '" + pkgname + "'" + if in_aports: + ret += " aport" + if in_apkindexes: + ret += " and could not find it" + if in_apkindexes: + ret += " in any APKINDEX" + return ret + + +def recurse(args, pkgnames, arch=None, in_apkindexes=True, in_aports=True, + strict=False): + """ + Find all dependencies of the given pkgnames. + + :param in_apkindexes: look through all APKINDEX files (with the specified arch) + :param in_aports: look through the aports folder + :param strict: raise RuntimeError, when a dependency can not be found. + """ + logging.debug("Calculate depends of packages " + str(pkgnames) + + ", arch: " + arch) + logging.verbose("Search in_aports: " + str(in_aports) + ", in_apkindexes: " + + str(in_apkindexes)) + + # Sanity check + if not apkindex and not in_aports: + raise RuntimeError("Set at least one of apkindex or aports to True.") + + todo = list(pkgnames) + ret = [] + while len(todo): + # Skip already passed entries + pkgname = todo.pop(0) + if pkgname in ret: + continue + + # Get depends + logging.verbose("Getting depends of single package: " + pkgname) + depends = None + if in_aports: + aport = pmb.build.find_aport(args, pkgname, False) + if aport: + logging.verbose("-> Found aport: " + aport) + apkbuild = pmb.parse.apkbuild(args, aport + "/APKBUILD") + depends = apkbuild["depends"] + if depends is None and in_apkindexes: + logging.verbose("-> Search through APKINDEX files") + depends = apkindex(args, pkgname, arch) + if depends is None and strict: + raise RuntimeError( + recurse_error_message( + pkgname, + in_aports, + in_apkindexes)) + + # Append to todo/ret + logging.verbose("-> Depends: " + str(depends)) + todo += depends + ret.append(pkgname) + + return ret diff --git a/pmbootstrap.py b/pmbootstrap.py index 8bf31bba..649a05dd 100755 --- a/pmbootstrap.py +++ b/pmbootstrap.py @@ -62,7 +62,7 @@ def main(): if args.action == "aportgen": pmb.aportgen.generate(args, args.package) elif args.action == "build": - pmb.build.package(args, args.package, args.arch, args.force, False, + pmb.build.package(args, args.package, args.arch, args.force, args.buildinfo) elif args.action == "build_init": pmb.build.init(args, args.suffix) @@ -84,8 +84,15 @@ def main(): elif args.action == "menuconfig": pmb.build.menuconfig(args, args.package, args.deviceinfo["arch"]) elif args.action == "parse_apkbuild": - print(json.dumps(pmb.parse.apkbuild(args.aports + "/" + + print(json.dumps(pmb.parse.apkbuild(args, args.aports + "/" + args.package + "/APKBUILD"), indent=4)) + elif args.action == "parse_apkindex": + print( + json.dumps( + pmb.parse.apkindex.parse( + args, + args.apkindex_path), + indent=4)) elif args.action == "shutdown": pmb.chroot.shutdown(args) elif args.action == "stats": diff --git a/test/test_apk_static.py b/test/test_apk_static.py index 49f9a296..8e5e6dc7 100644 --- a/test/test_apk_static.py +++ b/test/test_apk_static.py @@ -27,6 +27,7 @@ pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/..")) sys.path.append(pmb_src) import pmb.chroot.apk_static import pmb.parse.apkindex +import pmb.helpers.logging @pytest.fixture @@ -34,7 +35,8 @@ def args(request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args diff --git a/test/test_aport_in_sync_with_git.py b/test/test_aport_in_sync_with_git.py new file mode 100644 index 00000000..d62c3609 --- /dev/null +++ b/test/test_aport_in_sync_with_git.py @@ -0,0 +1,119 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import sys +import pytest + +# Import from parent directory +sys.path.append(os.path.abspath( + os.path.join(os.path.dirname(__file__) + "/.."))) +import pmb.build.other +import pmb.chroot.apk +import pmb.chroot.root +import pmb.helpers.run +import pmb.helpers.logging +import pmb.helpers.git + + +@pytest.fixture +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +def temp_aports_repo(args): + # Temp folder + temp = "/tmp/test_aport_in_sync_with_git" + temp_outside = args.work + "/chroot_native" + temp + if os.path.exists(temp_outside): + pmb.chroot.root(args, ["rm", "-rf", temp]) + pmb.chroot.user(args, ["mkdir", temp]) + + # Create fake "aports" repo + # For this test to work, we need a git repository cloned from a real upstream + # location. It does not work, when cloned from the same file system. The + # apk-tools repo also gets used in test_versions.py, so we use that. + pmb.chroot.apk.install(args, ["git"]) + pmb.helpers.git.clone(args, "apk-tools") + pmb.chroot.user(args, ["cp", "-r", "/home/user/git/apk-tools", + temp + "/aports"]) + + # Configure git + pmb.chroot.user(args, ["git", "config", "user.email", "user@localhost"], + working_dir=temp + "/aports") + pmb.chroot.user(args, ["git", "config", "user.name", "User"], + working_dir=temp + "/aports") + + # Update args.aports + setattr(args, "aports", temp_outside + "/aports") + return temp + "/aports" + + +def out_of_sync_files(args): + """ + Clear the cache again (because when running pmbootstrap normally, we assume, + that the contents of the aports folder does not change during one run) and + return the files out of sync for the hello-world package. + """ + args.cache["aports_files_out_of_sync_with_git"] = None + return pmb.build.other.aports_files_out_of_sync_with_git(args, + "hello-world") + + +def test_aport_in_sync_with_git(args): + aports = temp_aports_repo(args) + ret_in_sync = [] + ret_out_of_sync = [args.aports + "/hello-world/APKBUILD"] + + # In sync (no files changed) + assert out_of_sync_files(args) == ret_in_sync + + # Out of sync: untracked files + pmb.chroot.user(args, ["mkdir", aports + "/hello-world"]) + pmb.chroot.user(args, ["touch", aports + "/hello-world/APKBUILD"]) + assert out_of_sync_files(args) == ret_out_of_sync + + # Out of sync: tracked files + pmb.chroot.user(args, ["git", "add", aports + "/hello-world/APKBUILD"], + working_dir=aports) + assert out_of_sync_files(args) == ret_out_of_sync + + # Out of sync: comitted files + pmb.chroot.user(args, ["git", "commit", "-m", "test"], working_dir=aports) + assert out_of_sync_files(args) == ret_out_of_sync + + # In sync: undo the commit and check out a new branch + pmb.chroot.user(args, ["git", "reset", "--hard", "origin/master"], + working_dir=aports) + pmb.chroot.user(args, ["git", "checkout", "-b", "pmbootstrap-testbranch"], + working_dir=aports) + assert out_of_sync_files(args) == ret_in_sync + + # In sync: not a git repository + pmb.chroot.user(args, ["rm", "-rf", aports + "/.git"]) + assert out_of_sync_files(args) == ret_in_sync + + # TODO: + # - reinstall git, but rm .git, check again + # - remove temporary folder diff --git a/test/test_aportgen.py b/test/test_aportgen.py index 31f12104..2aaacb7f 100644 --- a/test/test_aportgen.py +++ b/test/test_aportgen.py @@ -26,6 +26,7 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.aportgen import pmb.config +import pmb.helpers.logging @pytest.fixture @@ -33,10 +34,11 @@ def args(tmpdir, request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) setattr(args, "_aports_real", args.aports) args.aports = str(tmpdir) - request.addfinalizer(args.logfd.close) return args diff --git a/test/test_build.py b/test/test_build.py index ca72f353..df4394f2 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -25,6 +25,7 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.aportgen import pmb.config +import pmb.helpers.logging @pytest.fixture @@ -32,7 +33,8 @@ def args(tmpdir, request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args diff --git a/test/test_build_is_necessary.py b/test/test_build_is_necessary.py index 3aad3faa..e1e0af70 100644 --- a/test/test_build_is_necessary.py +++ b/test/test_build_is_necessary.py @@ -24,6 +24,7 @@ import pytest sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.build.other +import pmb.helpers.logging @pytest.fixture @@ -31,12 +32,13 @@ def args(request, tmpdir): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) - request.addfinalizer(args.logfd.close) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) # Create an empty APKINDEX.tar.gz file, so we can use its path and # timestamp to put test information in the cache. + setattr(args, "timestamp_based_rebuild", True) apkindex_path = str(tmpdir) + "/APKINDEX.tar.gz" open(apkindex_path, "a").close() lastmod = os.path.getmtime(apkindex_path) @@ -44,9 +46,39 @@ def args(request, tmpdir): return args +def cache_apkindex(args, version=None, timestamp=None): + """ + Modify the cache of the parsed binary package repository's APKINDEX + for the "hello-world" package. + + The parameters version and timestamp are optional. If specified, they + change the string in the cache to the new value. + """ + apkindex_path = list(args.cache["apkindex"].keys())[0] + + if version is not None: + args.cache["apkindex"][apkindex_path][ + "ret"]["hello-world"]["version"] = version + if timestamp is not None: + args.cache["apkindex"][apkindex_path][ + "ret"]["hello-world"]["timestamp"] = timestamp + + +def cache_files_out_of_sync(args, is_out_of_sync): + """ + Modify the cache, so the function aports_files_out_of_sync_with_git() + returns, that there are files out of sync for the "hello-world" package, + or not. + """ + new = [] + if is_out_of_sync: + new = [os.path.abspath(args.aports + "/hello-world/APKBUILD")] + args.cache["aports_files_out_of_sync_with_git"] = new + + def test_build_is_necessary(args): # Prepare APKBUILD and APKINDEX data - apkbuild = pmb.parse.apkbuild(args.aports + "/hello-world/APKBUILD") + apkbuild = pmb.parse.apkbuild(args, args.aports + "/hello-world/APKBUILD") apkbuild["pkgver"] = "1" apkbuild["pkgrel"] = "2" apkindex_path = list(args.cache["apkindex"].keys())[0] @@ -55,25 +87,47 @@ def test_build_is_necessary(args): } # a) Binary repo has a newer version - args.cache["apkindex"][apkindex_path]["ret"][ - "hello-world"]["version"] = "999-r1" + cache_apkindex(args, version="999-r1") assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False # b) Aports folder has a newer version - args.cache["apkindex"][apkindex_path][ - "ret"]["hello-world"]["version"] = "0-r0" + cache_apkindex(args, version="0-r0") assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True - # c) Same version - args.cache["apkindex"][apkindex_path][ - "ret"]["hello-world"]["version"] = "1-r2" + # c), d) Preparation: same version + cache_apkindex(args, version="1-r2") - # c.1) Newer timestamp in aport (timestamp in repo: 1970-01-01) - args.cache["apkindex"][apkindex_path][ - "ret"]["hello-world"]["timestamp"] = "0" + # c) Out of sync sources, newer sources + cache_files_out_of_sync(args, True) + cache_apkindex(args, timestamp="0") assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True - # c.2) Newer timestamp in binary repo (timestamp in repo: 3000-01-01) - args.cache["apkindex"][apkindex_path]["ret"][ - "hello-world"]["timestamp"] = "32503680000" + # Timestamp based rebuild deactivated + setattr(args, "timestamp_based_rebuild", False) assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False + setattr(args, "timestamp_based_rebuild", True) + + # d1) Out of sync sources, old sources + cache_files_out_of_sync(args, True) + cache_apkindex(args, timestamp="32503680000") + assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False + + # d2) Sources in sync, newer sources + cache_files_out_of_sync(args, False) + cache_apkindex(args, timestamp="0") + assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False + + # d3) Out of sync sources, old sources + cache_files_out_of_sync(args, False) + cache_apkindex(args, timestamp="32503680000") + assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False + + +def test_build_is_necessary_no_binary_available(args): + """ + APKINDEX cache is set up to fake an empty APKINDEX, which means, that the + hello-world package has not been built yet. + """ + apkindex_path = list(args.cache["apkindex"].keys())[0] + apkbuild = pmb.parse.apkbuild(args, args.aports + "/hello-world/APKBUILD") + assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True diff --git a/test/test_challenge_apk.py b/test/test_challenge_apk.py index e7792da2..d72e0a47 100644 --- a/test/test_challenge_apk.py +++ b/test/test_challenge_apk.py @@ -27,6 +27,7 @@ sys.path.append(os.path.abspath( import pmb.challenge.apk_file import pmb.config import pmb.chroot.other +import pmb.helpers.logging @pytest.fixture @@ -34,7 +35,8 @@ def args(request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args diff --git a/test/test_challenge_apkindex.py b/test/test_challenge_apkindex.py index 12039352..2afef4f4 100644 --- a/test/test_challenge_apkindex.py +++ b/test/test_challenge_apkindex.py @@ -25,6 +25,7 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.challenge.apkindex import pmb.config +import pmb.helpers.logging @pytest.fixture @@ -32,7 +33,8 @@ def args(request, tmpdir): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) # Create an empty APKINDEX.tar.gz file, so we can use its path and diff --git a/test/test_challenge_build.py b/test/test_challenge_build.py index e7a03c29..ea835dd2 100644 --- a/test/test_challenge_build.py +++ b/test/test_challenge_build.py @@ -25,8 +25,9 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.build.package import pmb.challenge.build -import pmb.parse import pmb.config +import pmb.helpers.logging +import pmb.parse @pytest.fixture @@ -34,7 +35,8 @@ def args(request, tmpdir): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args @@ -45,7 +47,8 @@ def test_challenge_build(args): pmb.build.package(args, pkgname, None, force=True, buildinfo=True) # Copy it to a temporary path - apkbuild = pmb.parse.apkbuild(args.aports + "/" + pkgname + "/APKBUILD") + apkbuild = pmb.parse.apkbuild(args, args.aports + "/" + pkgname + + "/APKBUILD") version = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"] temp_path = pmb.chroot.other.tempfolder(args, "/tmp/test_challenge_build/" + args.arch_native) diff --git a/test/test_keys.py b/test/test_keys.py index b05d62d0..13fd2742 100644 --- a/test/test_keys.py +++ b/test/test_keys.py @@ -27,6 +27,7 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.parse.apkindex import pmb.helpers.git +import pmb.helpers.logging @pytest.fixture @@ -34,7 +35,8 @@ def args(request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args diff --git a/test/test_repo.py b/test/test_repo.py index b35e3b99..6960f230 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -25,11 +25,24 @@ import time # Import from parent directory pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/..")) sys.path.append(pmb_src) +import pmb.build.package +import pmb.helpers.logging import pmb.helpers.repo @pytest.fixture -def args(request, tmpdir): +def args(request): + import pmb.parse + sys.argv = ["pmbootstrap.py", "chroot"] + args = pmb.parse.arguments() + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) + request.addfinalizer(args.logfd.close) + return args + + +@pytest.fixture +def args_fake_work_dir(request, tmpdir): args = types.SimpleNamespace() args.work = str(tmpdir) return args @@ -45,12 +58,14 @@ def clear_timestamps_from_files(files): files[arch][file] = None -def test_files_empty(args): +def test_files_empty(args_fake_work_dir): + args = args_fake_work_dir os.mkdir(args.work + "/packages") assert pmb.helpers.repo.files(args) == {} -def test_files_not_empty(args): +def test_files_not_empty(args_fake_work_dir): + args = args_fake_work_dir pkgs = args.work + "/packages" for dir in ["", "armhf", "x86_64"]: os.mkdir(pkgs + "/" + dir) @@ -60,7 +75,8 @@ def test_files_not_empty(args): assert files == {"armhf": {}, "x86_64": {"test": None}} -def test_files_diff(args): +def test_files_diff(args_fake_work_dir): + args = args_fake_work_dir # Create x86_64/test, x86_64/test2 pkgs = args.work + "/packages" for dir in ["", "x86_64"]: @@ -85,3 +101,21 @@ def test_files_diff(args): diff = pmb.helpers.repo.diff(args, first) assert diff == ["aarch64/test3", "x86_64/test", "x86_64/test4"] + + +def test_hash(): + url = "https://nl.alpinelinux.org/alpine/edge/testing" + hash = "865a153c" + assert pmb.helpers.repo.hash(url, 8) == hash + + +def test_apkindex_files(args): + # Make sure, that we have a user's APKINDEX.tar.gz + pmb.build.package(args, "hello-world", args.arch_native) + + files = pmb.helpers.repo.apkindex_files(args) + for file in files: + assert os.path.exists(file) + + # Test cache + assert files == pmb.helpers.repo.apkindex_files(args) diff --git a/test/test_shell_escape.py b/test/test_shell_escape.py index 342b4439..881ccffa 100644 --- a/test/test_shell_escape.py +++ b/test/test_shell_escape.py @@ -23,9 +23,10 @@ import pytest # Import from parent directory pmb_src = os.path.abspath(os.path.join(os.path.dirname(__file__) + "/..")) sys.path.append(pmb_src) -import pmb.helpers.run import pmb.chroot.root import pmb.chroot.user +import pmb.helpers.run +import pmb.helpers.logging @pytest.fixture @@ -33,7 +34,8 @@ def args(request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args diff --git a/test/test_version.py b/test/test_version.py index cca921a5..80163234 100644 --- a/test/test_version.py +++ b/test/test_version.py @@ -25,6 +25,7 @@ sys.path.append(os.path.abspath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.parse.apkindex import pmb.helpers.git +import pmb.helpers.logging @pytest.fixture @@ -32,7 +33,8 @@ def args(request): import pmb.parse sys.argv = ["pmbootstrap.py", "chroot"] args = pmb.parse.arguments() - setattr(args, "logfd", open("/dev/null", "a+")) + args.log = args.work + "/log_testsuite.txt" + pmb.helpers.logging.init(args) request.addfinalizer(args.logfd.close) return args