Remove timestamp based rebuilds (#1174)
If you want to build a package without changing the version number,
please use `--force` from now on. For example:
pmbootstrap build --force hello-world
Prior to this commit, changes were detected automatically (timestamp
based rebuilds). However, that feature does not work as expected with
the binary package repository we have now, and depending on how you use
git, it has never worked. Close #1167, close #1156, close #1023 and
close #985. This commit also mentions --force when a package is up to date,
but the user requested to build it.
This commit is contained in:
Oliver Smith
GitHub
parent
bdeec7a255
commit
3c59126bc1
|
|
@ -11,7 +11,6 @@ install: "pip install flake8 pytest-cov python-coveralls"
|
|||
script:
|
||||
- test/static_code_analysis.sh
|
||||
- yes "" | ./pmbootstrap.py init
|
||||
- ./pmbootstrap.py config timestamp_based_rebuild False
|
||||
- ./pmbootstrap.py kconfig_check
|
||||
- test/testcases_fast.sh
|
||||
- test/check_checksums.py
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|||
import os
|
||||
import logging
|
||||
import glob
|
||||
import shutil
|
||||
|
||||
import pmb.build.other
|
||||
import pmb.chroot
|
||||
|
|
@ -92,83 +91,6 @@ def copy_to_buildpath(args, package, suffix="native"):
|
|||
"/home/pmos/build"], suffix=suffix)
|
||||
|
||||
|
||||
def aports_files_out_of_sync_with_git(args, package=None):
|
||||
"""
|
||||
Get a list of files, about which git says, that they have changed in
|
||||
comparison to upstream. We need this for the timestamp based rebuild check,
|
||||
where it does not only rely on the APKBUILD pkgver and pkgrel, but also on
|
||||
the file's last modified date to decide if it needs to be rebuilt. Git sets
|
||||
the last modified timestamp to the last checkout date, so we must ignore
|
||||
all files, that have not been modified, or else we would trigger rebuilds
|
||||
for all packages, from the pmOS binary repository.
|
||||
|
||||
:returns: list of absolute paths to all files not in sync with upstream
|
||||
"""
|
||||
|
||||
# Filter out a specific package
|
||||
if package:
|
||||
ret = []
|
||||
prefix = os.path.realpath(
|
||||
pmb.build.other.find_aport(
|
||||
args, package)) + "/"
|
||||
for file in aports_files_out_of_sync_with_git(args):
|
||||
if file.startswith(prefix):
|
||||
ret.append(file)
|
||||
return ret
|
||||
|
||||
# Use cached result if possible
|
||||
if args.cache["aports_files_out_of_sync_with_git"] is not None:
|
||||
return args.cache["aports_files_out_of_sync_with_git"]
|
||||
|
||||
# Get the aport's git repository folder
|
||||
git_root = None
|
||||
if shutil.which("git"):
|
||||
git_root = pmb.helpers.run.user(args, ["git", "rev-parse",
|
||||
"--show-toplevel"],
|
||||
working_dir=args.aports,
|
||||
return_stdout=True,
|
||||
check=False)
|
||||
if git_root:
|
||||
git_root = git_root.rstrip()
|
||||
ret = []
|
||||
if git_root and os.path.exists(git_root):
|
||||
# Find all out of sync files
|
||||
tracked = pmb.helpers.git.find_out_of_sync_files_tracked(
|
||||
args, git_root)
|
||||
untracked = pmb.helpers.run.user(
|
||||
args, ["git", "ls-files", "--others", "--exclude-standard"],
|
||||
working_dir=git_root, return_stdout=True)
|
||||
|
||||
# Set absolute path, filter out aports files
|
||||
aports_absolute = os.path.realpath(args.aports)
|
||||
files = tracked.rstrip().split("\n") + untracked.rstrip().split("\n")
|
||||
for file in files:
|
||||
file = os.path.realpath(git_root + "/" + file)
|
||||
if file.startswith(aports_absolute):
|
||||
ret.append(file)
|
||||
else:
|
||||
logging.warning("WARNING: Can not determine, which aport-files have been"
|
||||
" changed from upstream!")
|
||||
logging.info("* Aports-folder is not a git repository or git is not"
|
||||
" installed")
|
||||
logging.info("* You can turn timestamp-based rebuilds off in"
|
||||
" 'pmbootstrap init'")
|
||||
|
||||
# Save cache
|
||||
args.cache["aports_files_out_of_sync_with_git"] = ret
|
||||
return ret
|
||||
|
||||
|
||||
def sources_newer_than_binary_package(args, package, index_data):
|
||||
path_sources = []
|
||||
for file in glob.glob(args.aports + "/*/" + package + "/*"):
|
||||
path_sources.append(file)
|
||||
|
||||
lastmod_target = float(index_data["timestamp"])
|
||||
return not pmb.helpers.file.is_up_to_date(path_sources,
|
||||
lastmod_target=lastmod_target)
|
||||
|
||||
|
||||
def is_necessary(args, arch, apkbuild, apkindex_path=None):
|
||||
"""
|
||||
Check if the package has already been built. Compared to abuild's check,
|
||||
|
|
@ -212,26 +134,7 @@ def is_necessary(args, arch, apkbuild, apkindex_path=None):
|
|||
return True
|
||||
|
||||
# Aports and binary repo have the same version.
|
||||
if not args.timestamp_based_rebuild:
|
||||
return False
|
||||
|
||||
# c) Same version, source files out of sync with upstream, source
|
||||
# files newer than binary package
|
||||
files_out_of_sync = aports_files_out_of_sync_with_git(args, package)
|
||||
sources_newer = sources_newer_than_binary_package(
|
||||
args, package, index_data)
|
||||
if len(files_out_of_sync) and sources_newer:
|
||||
logging.debug(msg + "Binary package and aport have the same pkgver and"
|
||||
" pkgrel, but there are aport source files out of sync"
|
||||
" with the upstream git repository *and* these source"
|
||||
" files have a more recent 'last modified' timestamp than"
|
||||
" the binary package's build timestamp.")
|
||||
return True
|
||||
|
||||
# d) Same version, source files *in sync* with upstream *or* source
|
||||
# files *older* than binary package
|
||||
else:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def index_repo(args, arch=None):
|
||||
|
|
|
|||
|
|
@ -46,8 +46,7 @@ work_version = "1"
|
|||
|
||||
# Only save keys to the config file, which we ask for in 'pmbootstrap init'.
|
||||
config_keys = ["ccache_size", "device", "extra_packages", "jobs", "keymap",
|
||||
"qemu_native_mesa_driver", "timestamp_based_rebuild", "timezone",
|
||||
"ui", "user", "work"]
|
||||
"qemu_native_mesa_driver", "timezone", "ui", "user", "work"]
|
||||
|
||||
# Config file/commandline default values
|
||||
# $WORK gets replaced with the actual value for args.work (which may be
|
||||
|
|
@ -73,7 +72,6 @@ defaults = {
|
|||
"mirror_postmarketos": "http://postmarketos.brixit.nl",
|
||||
"port_distccd": "33632",
|
||||
"qemu_native_mesa_driver": "dri-virtio",
|
||||
"timestamp_based_rebuild": True,
|
||||
"timezone": "GMT",
|
||||
"ui": "weston",
|
||||
"user": "user",
|
||||
|
|
|
|||
|
|
@ -168,10 +168,8 @@ def ask_for_qemu_native_mesa_driver(args, device, arch_native):
|
|||
|
||||
def ask_for_build_options(args, cfg):
|
||||
# Allow to skip build options
|
||||
ts_rebuild = "True" if args.timestamp_based_rebuild else "False"
|
||||
logging.info("Build options: Parallel jobs: " + args.jobs +
|
||||
", ccache per arch: " + args.ccache_size +
|
||||
", timestamp based rebuilds: " + ts_rebuild)
|
||||
", ccache per arch: " + args.ccache_size)
|
||||
|
||||
if not pmb.helpers.cli.confirm(args, "Change them?",
|
||||
default=False):
|
||||
|
|
@ -194,14 +192,6 @@ def ask_for_build_options(args, cfg):
|
|||
lowercase_answer=False, validation_regex=regex)
|
||||
cfg["pmbootstrap"]["ccache_size"] = answer
|
||||
|
||||
# Timestamp based rebuilds
|
||||
logging.info("Rebuild packages, when the last modified timestamp changed,"
|
||||
" even if the version did not change?"
|
||||
" This makes pmbootstrap behave more like 'make'.")
|
||||
answer = pmb.helpers.cli.confirm(args, "Timestamp based rebuilds",
|
||||
default=args.timestamp_based_rebuild)
|
||||
cfg["pmbootstrap"]["timestamp_based_rebuild"] = str(answer)
|
||||
|
||||
|
||||
def frontend(args):
|
||||
cfg = pmb.config.load(args)
|
||||
|
|
|
|||
|
|
@ -143,8 +143,11 @@ def build(args):
|
|||
# Build all packages
|
||||
for package in args.packages:
|
||||
arch_package = args.arch or pmb.build.autodetect.arch(args, package)
|
||||
pmb.build.package(args, package, arch_package, args.force,
|
||||
args.buildinfo, args.strict)
|
||||
if not pmb.build.package(args, package, arch_package, args.force,
|
||||
args.buildinfo, args.strict):
|
||||
logging.info("NOTE: Package '" + package + "' is up to date. Use"
|
||||
" 'pmbootstrap build " + package + " --force'"
|
||||
" if needed.")
|
||||
|
||||
|
||||
def build_init(args):
|
||||
|
|
|
|||
|
|
@ -45,34 +45,3 @@ def rev_parse(args, revision="HEAD"):
|
|||
logging.warning("WARNING: Failed to determine revision of git repository at " + args.aports)
|
||||
return ""
|
||||
return rev.rstrip()
|
||||
|
||||
|
||||
def find_out_of_sync_files_tracked(args, git_root):
|
||||
"""
|
||||
Find all files tracked by git, that are are out of sync with origin/HEAD.
|
||||
|
||||
In some cases (when you rename a remote or add it manually), origin/HEAD
|
||||
does not exist. We check for that to provide a meaningful error message
|
||||
instead of a confusing crash (see #151).
|
||||
See also: <https://stackoverflow.com/a/17639471>
|
||||
"""
|
||||
# Return changed files compared to origin/HEAD when it exists
|
||||
ret = pmb.helpers.run.user(args, ["git", "show-ref",
|
||||
"refs/remotes/origin/HEAD"],
|
||||
working_dir=git_root, return_stdout=True,
|
||||
check=False)
|
||||
if ret and "refs/remotes/origin/HEAD" in ret:
|
||||
return pmb.helpers.run.user(args, ["git", "diff", "--name-only",
|
||||
"origin"], working_dir=git_root,
|
||||
return_stdout=True)
|
||||
|
||||
# Meaningful error
|
||||
logging.debug("Output of 'git diff --name-only origin': " + str(ret))
|
||||
logging.info("See also: <https://github.com/postmarketOS/pmbootstrap/"
|
||||
"issues/151>")
|
||||
fix_cmds = ("git symbolic-ref refs/remotes/origin/HEAD refs/remotes/"
|
||||
"origin/master; git fetch")
|
||||
raise RuntimeError("Your aports repository does not have the"
|
||||
" 'origin/HEAD' reference. Please add it by"
|
||||
" running the following commands inside " +
|
||||
git_root + ": " + fix_cmds)
|
||||
|
|
|
|||
|
|
@ -366,7 +366,6 @@ def arguments():
|
|||
"apkbuild": {},
|
||||
"apk_min_version_checked": [],
|
||||
"apk_repository_list_updated": [],
|
||||
"aports_files_out_of_sync_with_git": None,
|
||||
"built": {},
|
||||
"find_aport": {}})
|
||||
|
||||
|
|
|
|||
|
|
@ -1,136 +0,0 @@
|
|||
"""
|
||||
Copyright 2018 Oliver Smith
|
||||
|
||||
This file is part of pmbootstrap.
|
||||
|
||||
pmbootstrap is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
pmbootstrap is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import pytest
|
||||
|
||||
# Import from parent directory
|
||||
sys.path.append(os.path.realpath(
|
||||
os.path.join(os.path.dirname(__file__) + "/..")))
|
||||
import pmb.build.other
|
||||
import pmb.chroot.apk
|
||||
import pmb.chroot.root
|
||||
import pmb.helpers.run
|
||||
import pmb.helpers.logging
|
||||
import pmb.helpers.git
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def args(request):
|
||||
import pmb.parse
|
||||
sys.argv = ["pmbootstrap.py", "chroot"]
|
||||
args = pmb.parse.arguments()
|
||||
args.log = args.work + "/log_testsuite.txt"
|
||||
pmb.helpers.logging.init(args)
|
||||
request.addfinalizer(args.logfd.close)
|
||||
return args
|
||||
|
||||
|
||||
def temp_aports_repo(args):
|
||||
# Temp folder
|
||||
temp = "/tmp/test_aport_in_sync_with_git"
|
||||
temp_outside = args.work + "/chroot_native" + temp
|
||||
if os.path.exists(temp_outside):
|
||||
pmb.chroot.root(args, ["rm", "-rf", temp])
|
||||
pmb.chroot.user(args, ["mkdir", temp])
|
||||
|
||||
# Create fake "aports" repo
|
||||
# For this test to work, we need a git repository cloned from a real upstream
|
||||
# location. It does not work, when cloned from the same file system. The
|
||||
# aports_upstream repo also gets used in test_aportgen.py, so we use that.
|
||||
pmb.chroot.apk.install(args, ["git"])
|
||||
pmb.helpers.git.clone(args, "aports_upstream")
|
||||
pmb.chroot.user(args, ["cp", "-r", "/home/pmos/git/aports_upstream",
|
||||
temp + "/aports"])
|
||||
|
||||
# Configure git
|
||||
pmb.chroot.user(args, ["git", "config", "user.email", "user@localhost"],
|
||||
working_dir=temp + "/aports")
|
||||
pmb.chroot.user(args, ["git", "config", "user.name", "User"],
|
||||
working_dir=temp + "/aports")
|
||||
|
||||
# Update args.aports
|
||||
setattr(args, "aports", temp_outside + "/aports")
|
||||
return temp + "/aports"
|
||||
|
||||
|
||||
def out_of_sync_files(args):
|
||||
"""
|
||||
Clear the cache again (because when running pmbootstrap normally, we assume,
|
||||
that the contents of the aports folder does not change during one run) and
|
||||
return the files out of sync for the hello-world package.
|
||||
"""
|
||||
args.cache["aports_files_out_of_sync_with_git"] = None
|
||||
return pmb.build.other.aports_files_out_of_sync_with_git(args,
|
||||
"alpine-base")
|
||||
|
||||
|
||||
def test_aport_in_sync_with_git(args):
|
||||
aports = temp_aports_repo(args)
|
||||
ret_in_sync = []
|
||||
ret_out_of_sync = [os.path.realpath(args.aports + "/main/alpine-base/APKBUILD")]
|
||||
|
||||
# In sync (no files changed)
|
||||
assert out_of_sync_files(args) == ret_in_sync
|
||||
|
||||
# Out of sync: untracked files
|
||||
pmb.chroot.user(args, ["sh -c 'echo test >> " + aports +
|
||||
"/main/alpine-base/APKBUILD'"])
|
||||
assert out_of_sync_files(args) == ret_out_of_sync
|
||||
|
||||
# Out of sync: tracked files
|
||||
pmb.chroot.user(args, ["git", "add", aports + "/main/alpine-base/APKBUILD"],
|
||||
working_dir=aports)
|
||||
assert out_of_sync_files(args) == ret_out_of_sync
|
||||
|
||||
# Out of sync: comitted files
|
||||
pmb.chroot.user(args, ["git", "commit", "-m", "test"], working_dir=aports)
|
||||
assert out_of_sync_files(args) == ret_out_of_sync
|
||||
|
||||
# In sync: undo the commit and check out a new branch
|
||||
pmb.chroot.user(args, ["git", "reset", "--hard", "origin/master"],
|
||||
working_dir=aports)
|
||||
pmb.chroot.user(args, ["git", "checkout", "-b", "pmbootstrap-testbranch"],
|
||||
working_dir=aports)
|
||||
assert out_of_sync_files(args) == ret_in_sync
|
||||
|
||||
# In sync: not a git repository
|
||||
pmb.chroot.user(args, ["rm", "-rf", aports + "/.git"])
|
||||
assert out_of_sync_files(args) == ret_in_sync
|
||||
|
||||
# TODO:
|
||||
# - reinstall git, but rm .git, check again
|
||||
# - remove temporary folder
|
||||
|
||||
|
||||
def test_ambigious_argument(args):
|
||||
"""
|
||||
Testcase for #151, forces "fatal: ambiguous argument" in git.
|
||||
See also: https://stackoverflow.com/a/17639471
|
||||
"""
|
||||
|
||||
# Delete origin/HEAD
|
||||
aports = temp_aports_repo(args)
|
||||
pmb.chroot.user(args, ["git", "update-ref", "-d", "refs/remotes/origin/HEAD"],
|
||||
working_dir=aports)
|
||||
|
||||
# Check for exception
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
out_of_sync_files(args)
|
||||
assert "'origin/HEAD' reference" in str(e.value)
|
||||
|
|
@ -38,7 +38,6 @@ def args(request, tmpdir):
|
|||
|
||||
# Create an empty APKINDEX.tar.gz file, so we can use its path and
|
||||
# timestamp to put test information in the cache.
|
||||
setattr(args, "timestamp_based_rebuild", True)
|
||||
apkindex_path = str(tmpdir) + "/APKINDEX.tar.gz"
|
||||
open(apkindex_path, "a").close()
|
||||
lastmod = os.path.getmtime(apkindex_path)
|
||||
|
|
@ -46,35 +45,15 @@ def args(request, tmpdir):
|
|||
return args
|
||||
|
||||
|
||||
def cache_apkindex(args, version=None, timestamp=None):
|
||||
def cache_apkindex(args, version):
|
||||
"""
|
||||
Modify the cache of the parsed binary package repository's APKINDEX
|
||||
for the "hello-world" package.
|
||||
|
||||
The parameters version and timestamp are optional. If specified, they
|
||||
change the string in the cache to the new value.
|
||||
:param version: full version string, includes pkgver and pkgrl (e.g. 1-r2)
|
||||
"""
|
||||
apkindex_path = list(args.cache["apkindex"].keys())[0]
|
||||
|
||||
if version is not None:
|
||||
args.cache["apkindex"][apkindex_path][
|
||||
"ret"]["hello-world"]["version"] = version
|
||||
if timestamp is not None:
|
||||
args.cache["apkindex"][apkindex_path][
|
||||
"ret"]["hello-world"]["timestamp"] = timestamp
|
||||
|
||||
|
||||
def cache_files_out_of_sync(args, is_out_of_sync):
|
||||
"""
|
||||
Modify the cache, so the function aports_files_out_of_sync_with_git()
|
||||
returns, that there are files out of sync for the "hello-world" package,
|
||||
or not.
|
||||
"""
|
||||
new = []
|
||||
if is_out_of_sync:
|
||||
aport = pmb.build.other.find_aport(args, "hello-world")
|
||||
new = [os.path.realpath(aport + "/APKBUILD")]
|
||||
args.cache["aports_files_out_of_sync_with_git"] = new
|
||||
args.cache["apkindex"][apkindex_path]["ret"]["hello-world"]["version"] = version
|
||||
|
||||
|
||||
def test_build_is_necessary(args):
|
||||
|
|
@ -88,40 +67,16 @@ def test_build_is_necessary(args):
|
|||
"hello-world": {"pkgname": "hello-world", "version": "1-r2"}
|
||||
}
|
||||
|
||||
# a) Binary repo has a newer version
|
||||
cache_apkindex(args, version="999-r1")
|
||||
# Binary repo has a newer version
|
||||
cache_apkindex(args, "999-r1")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
|
||||
|
||||
# b) Aports folder has a newer version
|
||||
cache_apkindex(args, version="0-r0")
|
||||
# Aports folder has a newer version
|
||||
cache_apkindex(args, "0-r0")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
|
||||
|
||||
# c), d) Preparation: same version
|
||||
cache_apkindex(args, version="1-r2")
|
||||
|
||||
# c) Out of sync sources, newer sources
|
||||
cache_files_out_of_sync(args, True)
|
||||
cache_apkindex(args, timestamp="0")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is True
|
||||
|
||||
# Timestamp based rebuild deactivated
|
||||
setattr(args, "timestamp_based_rebuild", False)
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
|
||||
setattr(args, "timestamp_based_rebuild", True)
|
||||
|
||||
# d1) Out of sync sources, old sources
|
||||
cache_files_out_of_sync(args, True)
|
||||
cache_apkindex(args, timestamp="32503680000")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
|
||||
|
||||
# d2) Sources in sync, newer sources
|
||||
cache_files_out_of_sync(args, False)
|
||||
cache_apkindex(args, timestamp="0")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
|
||||
|
||||
# d3) Out of sync sources, old sources
|
||||
cache_files_out_of_sync(args, False)
|
||||
cache_apkindex(args, timestamp="32503680000")
|
||||
# Same version
|
||||
cache_apkindex(args, "1-r2")
|
||||
assert pmb.build.is_necessary(args, None, apkbuild, apkindex_path) is False
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -66,13 +66,6 @@ def test_config_user(args, tmpdir, monkeypatch):
|
|||
argv = ["pmbootstrap.py", "-c", path_config, "config"]
|
||||
args_default = args_patched(monkeypatch, argv)
|
||||
assert args_default.work == path_work
|
||||
assert args_default.timestamp_based_rebuild is True
|
||||
|
||||
# Modify timestamp_based_rebuild
|
||||
change_config(monkeypatch, path_config, "timestamp_based_rebuild", "false")
|
||||
assert args_patched(monkeypatch, argv).timestamp_based_rebuild is False
|
||||
change_config(monkeypatch, path_config, "timestamp_based_rebuild", "true")
|
||||
assert args_patched(monkeypatch, argv).timestamp_based_rebuild is True
|
||||
|
||||
# Modify jobs count
|
||||
change_config(monkeypatch, path_config, "jobs", "9000")
|
||||
|
|
|
|||
|
|
@ -110,7 +110,6 @@ def setup_work(args, tmpdir):
|
|||
# Copy over the pmbootstrap config, disable timestamp based rebuilds
|
||||
pmb.helpers.run.user(args, ["cp", args.config, tmpdir +
|
||||
"/_pmbootstrap.cfg"])
|
||||
pmbootstrap(args, tmpdir, ["config", "timestamp_based_rebuild", "false"])
|
||||
|
||||
|
||||
def verify_pkgrels(args, tmpdir, pkgrel_testlib, pkgrel_testapp):
|
||||
|
|
|
|||
|
|
@ -171,5 +171,4 @@ def test_questions(args, monkeypatch, tmpdir):
|
|||
answers = ["y", "5", "2G", "n"]
|
||||
func(args, cfg)
|
||||
assert cfg == {"pmbootstrap": {"jobs": "5",
|
||||
"ccache_size": "2G",
|
||||
"timestamp_based_rebuild": "False"}}
|
||||
"ccache_size": "2G"}}
|
||||
|
|
|
|||
Loading…
Reference in New Issue