From 112dc5e70c7edf5594357fcba3282c4a45f6c3a6 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Sat, 17 Jun 2017 17:42:28 +0200 Subject: [PATCH] Move challenge code to own folder (#64) ...and add a stub for 'pmbootstrap challenge APKINDEX.tar.gz'. --- pmb/build/__init__.py | 1 - pmb/challenge/__init__.py | 23 ++++++ pmb/{build/challenge.py => challenge/apk.py} | 76 ++--------------- pmb/challenge/apkindex.py | 24 ++++++ pmb/challenge/build.py | 85 ++++++++++++++++++++ pmb/challenge/frontend.py | 31 +++++++ pmb/parse/arguments.py | 9 ++- pmbootstrap.py | 3 +- 8 files changed, 178 insertions(+), 74 deletions(-) create mode 100644 pmb/challenge/__init__.py rename pmb/{build/challenge.py => challenge/apk.py} (59%) create mode 100644 pmb/challenge/apkindex.py create mode 100644 pmb/challenge/build.py create mode 100644 pmb/challenge/frontend.py diff --git a/pmb/build/__init__.py b/pmb/build/__init__.py index 5b6baa8b..c89d01c7 100644 --- a/pmb/build/__init__.py +++ b/pmb/build/__init__.py @@ -23,4 +23,3 @@ from pmb.build.other import copy_to_buildpath, is_necessary, \ symlink_noarch_package, find_aport, ccache_stats, index_repo from pmb.build.package import package from pmb.build.menuconfig import menuconfig -from pmb.build.challenge import challenge diff --git a/pmb/challenge/__init__.py b/pmb/challenge/__init__.py new file mode 100644 index 00000000..a218f26f --- /dev/null +++ b/pmb/challenge/__init__.py @@ -0,0 +1,23 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +# Exported functions +from pmb.challenge.apk import apk +from pmb.challenge.apkindex import apkindex +from pmb.challenge.build import build +from pmb.challenge.frontend import frontend diff --git a/pmb/build/challenge.py b/pmb/challenge/apk.py similarity index 59% rename from pmb/build/challenge.py rename to pmb/challenge/apk.py index b01000c6..2624ba23 100644 --- a/pmb/build/challenge.py +++ b/pmb/challenge/apk.py @@ -17,19 +17,14 @@ You should have received a copy of the GNU General Public License along with pmbootstrap. If not, see . """ import logging -import json import os import tarfile import tempfile import filecmp import shutil -import pmb.build -import pmb.parse.apkbuild -import pmb.parse.other -import pmb.helpers.repo -def diff_files(tar_a, tar_b, member_a, member_b, name): +def contents_diff(tar_a, tar_b, member_a, member_b, name): # Extract both files tars = [tar_a, tar_b] members = [member_a, member_b] @@ -51,7 +46,7 @@ def diff_files(tar_a, tar_b, member_a, member_b, name): raise RuntimeError("File '" + name + "' is different!") -def tar_getnames_without_signature(tar, tar_name): +def contents_without_signature(tar, tar_name): """ The signature file name is always different. This function raises an exception, when the number of signature @@ -79,12 +74,12 @@ def tar_getnames_without_signature(tar, tar_name): return sorted(ret) -def diff(args, apk_a, apk_b): +def apk(args, apk_a, apk_b): with tarfile.open(apk_a, "r:gz") as tar_a: with tarfile.open(apk_b, "r:gz") as tar_b: # List of files must be the same - list_a = tar_getnames_without_signature(tar_a, apk_a) - list_b = tar_getnames_without_signature(tar_b, apk_b) + list_a = contents_without_signature(tar_a, apk_a) + list_b = contents_without_signature(tar_b, apk_b) if list_a != list_b: logging.info("Files in " + apk_a + ":" + str(list_a)) logging.info("Files in " + apk_b + ":" + str(list_b)) @@ -111,7 +106,7 @@ def diff(args, apk_a, apk_b): if member_a.isdir(): logging.debug("=> Skipping: directory") elif member_a.isfile(): - diff_files(tar_a, tar_b, member_a, member_b, name) + contents_diff(tar_a, tar_b, member_a, member_b, name) elif member_a.issym() or member_a.islnk(): if member_a.linkname == member_b.linkname: logging.debug( @@ -127,62 +122,3 @@ def diff(args, apk_a, apk_b): success = False if not success: raise RuntimeError("Challenge failed (see errors above)") - - -def challenge(args, apk_path): - # Parse buildinfo - buildinfo_path = apk_path + ".buildinfo.json" - if not os.path.exists(buildinfo_path): - logging.info("NOTE: To create a .buildinfo.json file, use the" - " --buildinfo command while building: 'pmbootstrap build" - " --buildinfo '") - raise RuntimeError("Missing file: " + buildinfo_path) - with open(buildinfo_path) as handle: - buildinfo = json.load(handle) - - # Parse and install all packages listed in versions - versions = {} - for package in buildinfo["versions"]: - split = pmb.parse.other.package_split(package) - pkgname = split["pkgname"] - versions[pkgname] = split - pmb.chroot.apk.install(args, versions.keys()) - - # Verify the installed versions - installed = pmb.chroot.apk.installed(args) - for pkgname, split in versions.items(): - package_installed = installed[pkgname]["package"] - package_buildinfo = split["package"] - if package_installed != package_buildinfo: - raise RuntimeError("Dependency " + pkgname + " version is different" - " (installed: " + package_installed + "," - " buildinfo: " + package_buildinfo + ")!") - # Build the package - repo_before = pmb.helpers.repo.files(args) - pmb.build.package(args, buildinfo["pkgname"], buildinfo["arch"], - force=True, buildinfo=True) - repo_diff = pmb.helpers.repo.diff(args, repo_before) - - # Diff the apk contents - staging_path = os.path.abspath(os.path.dirname(apk_path) + "/../") - for file in repo_diff: - file_staging = staging_path + "/" + file - file_work = args.work + "/packages/" + file - - if file.endswith(".apk"): - logging.info("Verify " + file) - diff(args, file_staging, file_work) - elif (file.endswith("/APKINDEX.tar.gz") or - file.endswith(".apk.buildinfo.json")): - # We only verify the apk file (see above). The APKINDEX can - # be verified separately. - continue - else: - raise RuntimeError("Unknown file type changed in the" - " package repository folder: " + file) - - # Output the changed files from the repository - if args.output_repo_changes: - with open(args.output_repo_changes, "w") as handler: - for file in repo_diff: - handler.write(file + "\n") diff --git a/pmb/challenge/apkindex.py b/pmb/challenge/apkindex.py new file mode 100644 index 00000000..9f56e277 --- /dev/null +++ b/pmb/challenge/apkindex.py @@ -0,0 +1,24 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" + + +def apkindex(args, apkindex_path): + raise NotImplementedError("Challenge for APKINDEX.tar.gz is not" + " implemented yet (see issue #64 for more" + " information).") diff --git a/pmb/challenge/build.py b/pmb/challenge/build.py new file mode 100644 index 00000000..b9ff352f --- /dev/null +++ b/pmb/challenge/build.py @@ -0,0 +1,85 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import logging +import json +import os +import pmb.build +import pmb.parse.apkbuild +import pmb.parse.other +import pmb.helpers.repo +import pmb.challenge.apk + + +def build(args, apk_path): + # Parse buildinfo + buildinfo_path = apk_path + ".buildinfo.json" + if not os.path.exists(buildinfo_path): + logging.info("NOTE: To create a .buildinfo.json file, use the" + " --buildinfo command while building: 'pmbootstrap build" + " --buildinfo '") + raise RuntimeError("Missing file: " + buildinfo_path) + with open(buildinfo_path) as handle: + buildinfo = json.load(handle) + + # Parse and install all packages listed in versions + versions = {} + for package in buildinfo["versions"]: + split = pmb.parse.other.package_split(package) + pkgname = split["pkgname"] + versions[pkgname] = split + pmb.chroot.apk.install(args, versions.keys()) + + # Verify the installed versions + installed = pmb.chroot.apk.installed(args) + for pkgname, split in versions.items(): + package_installed = installed[pkgname]["package"] + package_buildinfo = split["package"] + if package_installed != package_buildinfo: + raise RuntimeError("Dependency " + pkgname + " version is different" + " (installed: " + package_installed + "," + " buildinfo: " + package_buildinfo + ")!") + # Build the package + repo_before = pmb.helpers.repo.files(args) + pmb.build.package(args, buildinfo["pkgname"], buildinfo["arch"], + force=True, buildinfo=True) + repo_diff = pmb.helpers.repo.diff(args, repo_before) + + # Diff the apk contents + staging_path = os.path.abspath(os.path.dirname(apk_path) + "/../") + for file in repo_diff: + file_staging = staging_path + "/" + file + file_work = args.work + "/packages/" + file + + if file.endswith(".apk"): + logging.info("Verify " + file) + pmb.challenge.apk(args, file_staging, file_work) + elif (file.endswith("/APKINDEX.tar.gz") or + file.endswith(".apk.buildinfo.json")): + # We only verify the apk file (see above). The APKINDEX can + # be verified separately. + continue + else: + raise RuntimeError("Unknown file type changed in the" + " package repository folder: " + file) + + # Output the changed files from the repository + if args.output_repo_changes: + with open(args.output_repo_changes, "w") as handler: + for file in repo_diff: + handler.write(file + "\n") diff --git a/pmb/challenge/frontend.py b/pmb/challenge/frontend.py new file mode 100644 index 00000000..46852d3a --- /dev/null +++ b/pmb/challenge/frontend.py @@ -0,0 +1,31 @@ +""" +Copyright 2017 Oliver Smith + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import os +import pmb.challenge + + +def frontend(args): + path = args.challenge_file + if path.endswith(".apk"): + pmb.challenge.build(args, path) + elif os.path.basename(path) == "APKINDEX.tar.gz": + pmb.challenge.apkindex(args, path) + else: + raise ValueError("It is only possible to challenge files ending" + " in .apk or files named APKINDEX.tar.gz.") diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 8e9ce001..6909d8cb 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -172,12 +172,17 @@ def arguments(): # Action: challenge challenge = sub.add_parser("challenge", - help="rebuild a package and diff its contents") + help="verify, that an apk file or" + " APKINDEX has been built" + " generated.") challenge.add_argument("--output-repo-changes", dest="output_repo_changes", help="pass the path to a file here, to store a list" " of apk- and APKINDEX-files that have been" " changed during the build", default=None) - challenge.add_argument("apk") + challenge.add_argument("challenge_file", + help="the file to be verified. must end in" + " .apk, or must be named" + " APKINDEX.tar.gz.") # Use defaults from the user's config file args = parser.parse_args() diff --git a/pmbootstrap.py b/pmbootstrap.py index d4b0e8f0..1366ca79 100755 --- a/pmbootstrap.py +++ b/pmbootstrap.py @@ -28,6 +28,7 @@ import traceback import pmb.aportgen import pmb.build import pmb.config +import pmb.challenge import pmb.chroot import pmb.chroot.initfs import pmb.chroot.other @@ -61,7 +62,7 @@ def main(): elif args.action == "build_init": pmb.build.init(args, args.suffix) elif args.action == "challenge": - pmb.build.challenge(args, args.apk) + pmb.challenge.frontend(args) elif args.action == "checksum": pmb.build.checksum(args, args.package) elif args.action == "chroot":