From e8c27795a8f9ed8cb27658255c623603a9c92ac5 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 1 Feb 2018 22:03:21 +0000 Subject: [PATCH] Remove rest of 'pmbootstrap challenge' left overs (#1173) Follow up to #1162. * `pmb.build.buildinfo()`: Used to record the build environment. It is flawed because it scans the repo APKINDEX files instead of using the actually installed packages list. When it was implemented we were not able to do the latter. After this is removed, `pmb.parse.depends` can be simplified (it needs to be rewritten for #1122). * `pmb.helpers.repo.diff()` and `pmb.helpers.repo.files()`: These were used exclusively by `pmb.build.buildinfo()`, to learn about which files have been changed in the local repository folder after a package was built. The idea was, that we could find subpackages that way. But this information is present in the installed package list as well, which is a much cleaner approach. --- pmb/build/_package.py | 18 ++------- pmb/build/buildinfo.py | 74 ------------------------------------ pmb/helpers/frontend.py | 2 +- pmb/helpers/repo.py | 47 ----------------------- pmb/parse/arguments.py | 1 - test/test_build_package.py | 1 - test/test_repo.py | 78 -------------------------------------- 7 files changed, 5 insertions(+), 216 deletions(-) delete mode 100644 pmb/build/buildinfo.py diff --git a/pmb/build/_package.py b/pmb/build/_package.py index b68350ca..b6bf2d41 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -21,7 +21,6 @@ import logging import pmb.build import pmb.build.autodetect -import pmb.build.buildinfo import pmb.chroot import pmb.chroot.apk import pmb.chroot.distccd @@ -272,8 +271,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, return (output, cmd, env) -def finish(args, apkbuild, arch, output, strict=False, suffix="native", - buildinfo=False): +def finish(args, apkbuild, arch, output, strict=False, suffix="native"): """ Various finishing tasks that need to be done after a build. """ @@ -282,13 +280,6 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native", if not os.path.exists(path): raise RuntimeError("Package not found after build: " + path) - # Create .buildinfo.json file (captures the build environment, from the - # reproducible builds approach in #64 that we aren't using anymore, but it - # might still be useful) - if buildinfo: - logging.info("(" + suffix + ") generate " + output + ".buildinfo.json") - pmb.build.buildinfo.write(args, output, arch, suffix, apkbuild) - # Clear APKINDEX cache (we only parse APKINDEX files once per session and # cache the result for faster dependency resolving, but after we built a # package we need to parse it again) @@ -301,15 +292,14 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native", pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build") -def package(args, pkgname, arch=None, force=False, buildinfo=False, - strict=False, skip_init_buildenv=False): +def package(args, pkgname, arch=None, force=False, strict=False, + skip_init_buildenv=False): """ Build a package and its dependencies with Alpine Linux' abuild. :param pkgname: package name to be built, as specified in the APKBUILD :param arch: architecture we're building for (default: native) :param force: even build, if not necessary - :param buildinfo: record the build environment in a .buildinfo.json file :param strict: avoid building with irrelevant dependencies installed by letting abuild install and uninstall all dependencies. :param skip_init_buildenv: can be set to False to avoid initializing the @@ -340,5 +330,5 @@ def package(args, pkgname, arch=None, force=False, buildinfo=False, # Build and finish up (output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross, suffix) - finish(args, apkbuild, arch, output, strict, suffix, buildinfo) + finish(args, apkbuild, arch, output, strict, suffix) return output diff --git a/pmb/build/buildinfo.py b/pmb/build/buildinfo.py deleted file mode 100644 index 68fa3f04..00000000 --- a/pmb/build/buildinfo.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Copyright 2018 Oliver Smith - -This file is part of pmbootstrap. - -pmbootstrap is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -pmbootstrap is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with pmbootstrap. If not, see . -""" -import os -import json -import pmb.chroot -import pmb.chroot.apk -import pmb.parse.apkindex -import pmb.parse.depends - - -def generate(args, apk_path, arch, suffix, apkbuild): - """ - :param apk_path: Path to the .apk file, relative to the packages cache. - :param arch: Architecture, that the package has been built for. - :apkbuild: Return from pmb.parse.apkbuild(). - """ - ret = {"pkgname": apkbuild["pkgname"], - "pkgver": apkbuild["pkgver"], - "pkgrel": apkbuild["pkgrel"], - "arch": arch, - "versions": {}} - - # Add makedepends versions - installed = pmb.chroot.apk.installed(args, suffix) - relevant = (apkbuild["makedepends"] + [apkbuild["pkgname"], "abuild", - "build-base"]) - relevant = pmb.parse.depends.recurse(args, relevant, arch, in_aports=False, - strict=True) - for pkgname in relevant: - if pkgname == apkbuild["pkgname"]: - continue - if pkgname in installed: - ret["versions"][pkgname] = installed[pkgname]["version"] - return ret - - -def write(args, apk_path, arch, suffix, apkbuild): - """ - Write a .buildinfo.json file for a package, right after building it. - It stores all information required to rebuild the package, very similar - to how they do it in Debian (but as JSON file, so it's easier to parse in - Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles - - :param apk_path: Path to the .apk file, relative to the packages cache. - :param arch: Architecture, that the package has been built for. - :apkbuild: Return from pmb.parse.apkbuild(). - """ - # Write to temp - if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"): - pmb.chroot.root(args, ["rm", "/tmp/buildinfo"]) - buildinfo = generate(args, apk_path, arch, suffix, apkbuild) - with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle: - handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n") - - # Move to packages - pmb.chroot.root(args, ["chown", "pmos:pmos", "/tmp/buildinfo"]) - pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/pmos/packages/pmos/" + - apk_path + ".buildinfo.json"]) diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 5ba1ef25..d9958758 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -144,7 +144,7 @@ def build(args): for package in args.packages: arch_package = args.arch or pmb.build.autodetect.arch(args, package) if not pmb.build.package(args, package, arch_package, args.force, - args.buildinfo, args.strict): + args.strict): logging.info("NOTE: Package '" + package + "' is up to date. Use" " 'pmbootstrap build " + package + " --force'" " if needed.") diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 657fc6fc..2811829b 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -16,7 +16,6 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ -import glob import os import hashlib import logging @@ -24,52 +23,6 @@ import pmb.helpers.http import pmb.helpers.run -def files(args): - """ - Returns all files (apk/buildinfo) with their last modification timestamp - inside the package repository, sorted by architecture. - - :returns: {"x86_64": {"first.apk": last_modified_timestamp, ... }, ... } - """ - ret = {} - for arch_folder in glob.glob(args.work + "/packages/*"): - arch = os.path.basename(arch_folder) - ret[arch] = {} - for file in glob.glob(arch_folder + "/*"): - basename = os.path.basename(file) - ret[arch][basename] = os.path.getmtime(file) - return ret - - -def diff(args, files_a, files_b=None): - """ - Returns a list of files, that have been added or modified inside the - package repository. - - :param files_a: return value from pmb.helpers.repo.files() - :param files_b: defaults to creating a new list - :returns: ["x86_64/APKINDEX.tar.gz", "x86_64/package.apk", - "x86_64/package.buildinfo", ...] - """ - if not files_b: - files_b = files(args) - - ret = [] - for arch in files_b.keys(): - for file, timestamp in files_b[arch].items(): - add = False - if arch not in files_a: - add = True - elif file not in files_a[arch]: - add = True - elif timestamp != files_a[arch][file]: - add = True - if add: - ret.append(arch + "/" + file) - - return sorted(ret) - - def hash(url, length=8): """ Generate the hash, that APK adds to the APKINDEX and apk packages diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index c283dab3..c2a31c00 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -312,7 +312,6 @@ def arguments(): " APKBUILD)") build.add_argument("--force", action="store_true", help="even build if not" " necessary") - build.add_argument("--buildinfo", action="store_true") build.add_argument("--strict", action="store_true", help="(slower) zap and install only" " required depends when building, to detect dependency errors") build.add_argument("-i", "--ignore-depends", action="store_true", diff --git a/test/test_build_package.py b/test/test_build_package.py index a39b5758..a95c78e7 100644 --- a/test/test_build_package.py +++ b/test/test_build_package.py @@ -243,7 +243,6 @@ def test_finish(args, monkeypatch): output = pmb.build.package(args, "hello-world", force=True) # Disable effects of functions we don't want to test below - monkeypatch.setattr(pmb.build.buildinfo, "write", return_none) monkeypatch.setattr(pmb.chroot, "user", return_none) # Shortcut and fake apkbuild diff --git a/test/test_repo.py b/test/test_repo.py index 5bad2645..809f477b 100644 --- a/test/test_repo.py +++ b/test/test_repo.py @@ -18,91 +18,13 @@ along with pmbootstrap. If not, see . """ import os import sys -import pytest -import types -import time # Import from parent directory pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/..")) sys.path.append(pmb_src) -import pmb.build -import pmb.helpers.logging import pmb.helpers.repo -@pytest.fixture -def args(request): - import pmb.parse - sys.argv = ["pmbootstrap.py", "chroot"] - args = pmb.parse.arguments() - args.log = args.work + "/log_testsuite.txt" - pmb.helpers.logging.init(args) - request.addfinalizer(args.logfd.close) - return args - - -@pytest.fixture -def args_fake_work_dir(request, tmpdir): - args = types.SimpleNamespace() - args.work = str(tmpdir) - return args - - -def clear_timestamps_from_files(files): - """ - Replace all last modified timestamps from pmb.helpers.repo.files() with - None. The files-parameter gets changed in place. - """ - for arch in files.keys(): - for file in files[arch].keys(): - files[arch][file] = None - - -def test_files_empty(args_fake_work_dir): - args = args_fake_work_dir - os.mkdir(args.work + "/packages") - assert pmb.helpers.repo.files(args) == {} - - -def test_files_not_empty(args_fake_work_dir): - args = args_fake_work_dir - pkgs = args.work + "/packages" - for dir in ["", "armhf", "x86_64"]: - os.mkdir(pkgs + "/" + dir) - open(pkgs + "/x86_64/test", "a").close() - files = pmb.helpers.repo.files(args) - clear_timestamps_from_files(files) - assert files == {"armhf": {}, "x86_64": {"test": None}} - - -def test_files_diff(args_fake_work_dir): - args = args_fake_work_dir - # Create x86_64/test, x86_64/test2 - pkgs = args.work + "/packages" - for dir in ["", "x86_64"]: - os.mkdir(pkgs + "/" + dir) - for file in ["x86_64/test", "x86_64/test2"]: - open(pkgs + "/" + file, "a").close() - - # First snapshot - first = pmb.helpers.repo.files(args) - - # Change: x86_64/test (set the lastmod timestamp 5 seconds in the future) - mtime_old = os.path.getmtime(pkgs + "/x86_64/test") - time_new = time.time() + 5 - os.utime(pkgs + "/x86_64/test", (time_new, time_new)) - mtime_new = os.path.getmtime(pkgs + "/x86_64/test") - assert mtime_old != mtime_new - - # Create: aarch64/test3, x86_64/test4 - os.mkdir(pkgs + "/aarch64") - open(pkgs + "/aarch64/test3", "a").close() - open(pkgs + "/x86_64/test4", "a").close() - - diff = pmb.helpers.repo.diff(args, first) - assert diff == ["aarch64/test3", "x86_64/test", "x86_64/test4"] - - def test_hash(): url = "https://nl.alpinelinux.org/alpine/edge/testing" hash = "865a153c"