Remove rest of 'pmbootstrap challenge' left overs (#1173)
Follow up to #1162. * `pmb.build.buildinfo()`: Used to record the build environment. It is flawed because it scans the repo APKINDEX files instead of using the actually installed packages list. When it was implemented we were not able to do the latter. After this is removed, `pmb.parse.depends` can be simplified (it needs to be rewritten for #1122). * `pmb.helpers.repo.diff()` and `pmb.helpers.repo.files()`: These were used exclusively by `pmb.build.buildinfo()`, to learn about which files have been changed in the local repository folder after a package was built. The idea was, that we could find subpackages that way. But this information is present in the installed package list as well, which is a much cleaner approach.
This commit is contained in:
Oliver Smith
GitHub
parent
987ec7a00a
commit
e8c27795a8
|
|
@ -21,7 +21,6 @@ import logging
|
|||
|
||||
import pmb.build
|
||||
import pmb.build.autodetect
|
||||
import pmb.build.buildinfo
|
||||
import pmb.chroot
|
||||
import pmb.chroot.apk
|
||||
import pmb.chroot.distccd
|
||||
|
|
@ -272,8 +271,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
|
|||
return (output, cmd, env)
|
||||
|
||||
|
||||
def finish(args, apkbuild, arch, output, strict=False, suffix="native",
|
||||
buildinfo=False):
|
||||
def finish(args, apkbuild, arch, output, strict=False, suffix="native"):
|
||||
"""
|
||||
Various finishing tasks that need to be done after a build.
|
||||
"""
|
||||
|
|
@ -282,13 +280,6 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native",
|
|||
if not os.path.exists(path):
|
||||
raise RuntimeError("Package not found after build: " + path)
|
||||
|
||||
# Create .buildinfo.json file (captures the build environment, from the
|
||||
# reproducible builds approach in #64 that we aren't using anymore, but it
|
||||
# might still be useful)
|
||||
if buildinfo:
|
||||
logging.info("(" + suffix + ") generate " + output + ".buildinfo.json")
|
||||
pmb.build.buildinfo.write(args, output, arch, suffix, apkbuild)
|
||||
|
||||
# Clear APKINDEX cache (we only parse APKINDEX files once per session and
|
||||
# cache the result for faster dependency resolving, but after we built a
|
||||
# package we need to parse it again)
|
||||
|
|
@ -301,15 +292,14 @@ def finish(args, apkbuild, arch, output, strict=False, suffix="native",
|
|||
pmb.chroot.user(args, ["abuild", "undeps"], suffix, "/home/pmos/build")
|
||||
|
||||
|
||||
def package(args, pkgname, arch=None, force=False, buildinfo=False,
|
||||
strict=False, skip_init_buildenv=False):
|
||||
def package(args, pkgname, arch=None, force=False, strict=False,
|
||||
skip_init_buildenv=False):
|
||||
"""
|
||||
Build a package and its dependencies with Alpine Linux' abuild.
|
||||
|
||||
:param pkgname: package name to be built, as specified in the APKBUILD
|
||||
:param arch: architecture we're building for (default: native)
|
||||
:param force: even build, if not necessary
|
||||
:param buildinfo: record the build environment in a .buildinfo.json file
|
||||
:param strict: avoid building with irrelevant dependencies installed by
|
||||
letting abuild install and uninstall all dependencies.
|
||||
:param skip_init_buildenv: can be set to False to avoid initializing the
|
||||
|
|
@ -340,5 +330,5 @@ def package(args, pkgname, arch=None, force=False, buildinfo=False,
|
|||
# Build and finish up
|
||||
(output, cmd, env) = run_abuild(args, apkbuild, arch, strict, force, cross,
|
||||
suffix)
|
||||
finish(args, apkbuild, arch, output, strict, suffix, buildinfo)
|
||||
finish(args, apkbuild, arch, output, strict, suffix)
|
||||
return output
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import pmb.chroot
|
||||
import pmb.chroot.apk
|
||||
import pmb.parse.apkindex
|
||||
import pmb.parse.depends
|
||||
|
||||
|
||||
def generate(args, apk_path, arch, suffix, apkbuild):
|
||||
"""
|
||||
:param apk_path: Path to the .apk file, relative to the packages cache.
|
||||
:param arch: Architecture, that the package has been built for.
|
||||
:apkbuild: Return from pmb.parse.apkbuild().
|
||||
"""
|
||||
ret = {"pkgname": apkbuild["pkgname"],
|
||||
"pkgver": apkbuild["pkgver"],
|
||||
"pkgrel": apkbuild["pkgrel"],
|
||||
"arch": arch,
|
||||
"versions": {}}
|
||||
|
||||
# Add makedepends versions
|
||||
installed = pmb.chroot.apk.installed(args, suffix)
|
||||
relevant = (apkbuild["makedepends"] + [apkbuild["pkgname"], "abuild",
|
||||
"build-base"])
|
||||
relevant = pmb.parse.depends.recurse(args, relevant, arch, in_aports=False,
|
||||
strict=True)
|
||||
for pkgname in relevant:
|
||||
if pkgname == apkbuild["pkgname"]:
|
||||
continue
|
||||
if pkgname in installed:
|
||||
ret["versions"][pkgname] = installed[pkgname]["version"]
|
||||
return ret
|
||||
|
||||
|
||||
def write(args, apk_path, arch, suffix, apkbuild):
|
||||
"""
|
||||
Write a .buildinfo.json file for a package, right after building it.
|
||||
It stores all information required to rebuild the package, very similar
|
||||
to how they do it in Debian (but as JSON file, so it's easier to parse in
|
||||
Python): https://wiki.debian.org/ReproducibleBuilds/BuildinfoFiles
|
||||
|
||||
:param apk_path: Path to the .apk file, relative to the packages cache.
|
||||
:param arch: Architecture, that the package has been built for.
|
||||
:apkbuild: Return from pmb.parse.apkbuild().
|
||||
"""
|
||||
# Write to temp
|
||||
if os.path.exists(args.work + "/chroot_native/tmp/buildinfo"):
|
||||
pmb.chroot.root(args, ["rm", "/tmp/buildinfo"])
|
||||
buildinfo = generate(args, apk_path, arch, suffix, apkbuild)
|
||||
with open(args.work + "/chroot_native/tmp/buildinfo", "w") as handle:
|
||||
handle.write(json.dumps(buildinfo, indent=4, sort_keys=True) + "\n")
|
||||
|
||||
# Move to packages
|
||||
pmb.chroot.root(args, ["chown", "pmos:pmos", "/tmp/buildinfo"])
|
||||
pmb.chroot.user(args, ["mv", "/tmp/buildinfo", "/home/pmos/packages/pmos/" +
|
||||
apk_path + ".buildinfo.json"])
|
||||
|
|
@ -144,7 +144,7 @@ def build(args):
|
|||
for package in args.packages:
|
||||
arch_package = args.arch or pmb.build.autodetect.arch(args, package)
|
||||
if not pmb.build.package(args, package, arch_package, args.force,
|
||||
args.buildinfo, args.strict):
|
||||
args.strict):
|
||||
logging.info("NOTE: Package '" + package + "' is up to date. Use"
|
||||
" 'pmbootstrap build " + package + " --force'"
|
||||
" if needed.")
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ GNU General Public License for more details.
|
|||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import hashlib
|
||||
import logging
|
||||
|
|
@ -24,52 +23,6 @@ import pmb.helpers.http
|
|||
import pmb.helpers.run
|
||||
|
||||
|
||||
def files(args):
|
||||
"""
|
||||
Returns all files (apk/buildinfo) with their last modification timestamp
|
||||
inside the package repository, sorted by architecture.
|
||||
|
||||
:returns: {"x86_64": {"first.apk": last_modified_timestamp, ... }, ... }
|
||||
"""
|
||||
ret = {}
|
||||
for arch_folder in glob.glob(args.work + "/packages/*"):
|
||||
arch = os.path.basename(arch_folder)
|
||||
ret[arch] = {}
|
||||
for file in glob.glob(arch_folder + "/*"):
|
||||
basename = os.path.basename(file)
|
||||
ret[arch][basename] = os.path.getmtime(file)
|
||||
return ret
|
||||
|
||||
|
||||
def diff(args, files_a, files_b=None):
|
||||
"""
|
||||
Returns a list of files, that have been added or modified inside the
|
||||
package repository.
|
||||
|
||||
:param files_a: return value from pmb.helpers.repo.files()
|
||||
:param files_b: defaults to creating a new list
|
||||
:returns: ["x86_64/APKINDEX.tar.gz", "x86_64/package.apk",
|
||||
"x86_64/package.buildinfo", ...]
|
||||
"""
|
||||
if not files_b:
|
||||
files_b = files(args)
|
||||
|
||||
ret = []
|
||||
for arch in files_b.keys():
|
||||
for file, timestamp in files_b[arch].items():
|
||||
add = False
|
||||
if arch not in files_a:
|
||||
add = True
|
||||
elif file not in files_a[arch]:
|
||||
add = True
|
||||
elif timestamp != files_a[arch][file]:
|
||||
add = True
|
||||
if add:
|
||||
ret.append(arch + "/" + file)
|
||||
|
||||
return sorted(ret)
|
||||
|
||||
|
||||
def hash(url, length=8):
|
||||
"""
|
||||
Generate the hash, that APK adds to the APKINDEX and apk packages
|
||||
|
|
|
|||
|
|
@ -312,7 +312,6 @@ def arguments():
|
|||
" APKBUILD)")
|
||||
build.add_argument("--force", action="store_true", help="even build if not"
|
||||
" necessary")
|
||||
build.add_argument("--buildinfo", action="store_true")
|
||||
build.add_argument("--strict", action="store_true", help="(slower) zap and install only"
|
||||
" required depends when building, to detect dependency errors")
|
||||
build.add_argument("-i", "--ignore-depends", action="store_true",
|
||||
|
|
|
|||
|
|
@ -243,7 +243,6 @@ def test_finish(args, monkeypatch):
|
|||
output = pmb.build.package(args, "hello-world", force=True)
|
||||
|
||||
# Disable effects of functions we don't want to test below
|
||||
monkeypatch.setattr(pmb.build.buildinfo, "write", return_none)
|
||||
monkeypatch.setattr(pmb.chroot, "user", return_none)
|
||||
|
||||
# Shortcut and fake apkbuild
|
||||
|
|
|
|||
|
|
@ -18,91 +18,13 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
import types
|
||||
import time
|
||||
|
||||
# Import from parent directory
|
||||
pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
|
||||
sys.path.append(pmb_src)
|
||||
import pmb.build
|
||||
import pmb.helpers.logging
|
||||
import pmb.helpers.repo
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args(request):
|
||||
import pmb.parse
|
||||
sys.argv = ["pmbootstrap.py", "chroot"]
|
||||
args = pmb.parse.arguments()
|
||||
args.log = args.work + "/log_testsuite.txt"
|
||||
pmb.helpers.logging.init(args)
|
||||
request.addfinalizer(args.logfd.close)
|
||||
return args
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args_fake_work_dir(request, tmpdir):
|
||||
args = types.SimpleNamespace()
|
||||
args.work = str(tmpdir)
|
||||
return args
|
||||
|
||||
|
||||
def clear_timestamps_from_files(files):
|
||||
"""
|
||||
Replace all last modified timestamps from pmb.helpers.repo.files() with
|
||||
None. The files-parameter gets changed in place.
|
||||
"""
|
||||
for arch in files.keys():
|
||||
for file in files[arch].keys():
|
||||
files[arch][file] = None
|
||||
|
||||
|
||||
def test_files_empty(args_fake_work_dir):
|
||||
args = args_fake_work_dir
|
||||
os.mkdir(args.work + "/packages")
|
||||
assert pmb.helpers.repo.files(args) == {}
|
||||
|
||||
|
||||
def test_files_not_empty(args_fake_work_dir):
|
||||
args = args_fake_work_dir
|
||||
pkgs = args.work + "/packages"
|
||||
for dir in ["", "armhf", "x86_64"]:
|
||||
os.mkdir(pkgs + "/" + dir)
|
||||
open(pkgs + "/x86_64/test", "a").close()
|
||||
files = pmb.helpers.repo.files(args)
|
||||
clear_timestamps_from_files(files)
|
||||
assert files == {"armhf": {}, "x86_64": {"test": None}}
|
||||
|
||||
|
||||
def test_files_diff(args_fake_work_dir):
|
||||
args = args_fake_work_dir
|
||||
# Create x86_64/test, x86_64/test2
|
||||
pkgs = args.work + "/packages"
|
||||
for dir in ["", "x86_64"]:
|
||||
os.mkdir(pkgs + "/" + dir)
|
||||
for file in ["x86_64/test", "x86_64/test2"]:
|
||||
open(pkgs + "/" + file, "a").close()
|
||||
|
||||
# First snapshot
|
||||
first = pmb.helpers.repo.files(args)
|
||||
|
||||
# Change: x86_64/test (set the lastmod timestamp 5 seconds in the future)
|
||||
mtime_old = os.path.getmtime(pkgs + "/x86_64/test")
|
||||
time_new = time.time() + 5
|
||||
os.utime(pkgs + "/x86_64/test", (time_new, time_new))
|
||||
mtime_new = os.path.getmtime(pkgs + "/x86_64/test")
|
||||
assert mtime_old != mtime_new
|
||||
|
||||
# Create: aarch64/test3, x86_64/test4
|
||||
os.mkdir(pkgs + "/aarch64")
|
||||
open(pkgs + "/aarch64/test3", "a").close()
|
||||
open(pkgs + "/x86_64/test4", "a").close()
|
||||
|
||||
diff = pmb.helpers.repo.diff(args, first)
|
||||
assert diff == ["aarch64/test3", "x86_64/test", "x86_64/test4"]
|
||||
|
||||
|
||||
def test_hash():
|
||||
url = "https://nl.alpinelinux.org/alpine/edge/testing"
|
||||
hash = "865a153c"
|
||||
|
|
|
|||
Loading…
Reference in New Issue