2017-05-26 20:35:21 +00:00
|
|
|
"""
|
|
|
|
Copyright 2017 Oliver Smith
|
|
|
|
|
|
|
|
This file is part of pmbootstrap.
|
|
|
|
|
|
|
|
pmbootstrap is free software: you can redistribute it and/or modify
|
|
|
|
it under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
|
|
(at your option) any later version.
|
|
|
|
|
|
|
|
pmbootstrap is distributed in the hope that it will be useful,
|
|
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
GNU General Public License for more details.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
|
|
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
"""
|
|
|
|
import os
|
|
|
|
import logging
|
|
|
|
import glob
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
import shutil
|
2017-05-26 20:35:21 +00:00
|
|
|
|
2017-07-28 22:34:40 +00:00
|
|
|
import pmb.build.other
|
2017-05-26 20:35:21 +00:00
|
|
|
import pmb.chroot
|
|
|
|
import pmb.helpers.run
|
2017-06-19 18:33:56 +00:00
|
|
|
import pmb.helpers.file
|
2017-05-26 20:35:21 +00:00
|
|
|
import pmb.parse.apkindex
|
2017-08-12 14:03:40 +00:00
|
|
|
import pmb.parse.version
|
2017-05-26 20:35:21 +00:00
|
|
|
|
|
|
|
|
|
|
|
def find_aport(args, package, must_exist=True):
|
|
|
|
"""
|
|
|
|
Find the aport, that provides a certain subpackage.
|
|
|
|
|
|
|
|
:param must_exist: Raise an exception, when not found
|
|
|
|
:returns: the full path to the aport folder
|
|
|
|
"""
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
# Try to get a cached result first (we assume, that the aports don't change
|
|
|
|
# in one pmbootstrap call)
|
|
|
|
ret = None
|
2017-07-28 22:34:40 +00:00
|
|
|
if package in args.cache["find_aport"]:
|
|
|
|
ret = args.cache["find_aport"][package]
|
|
|
|
else:
|
|
|
|
# Search in packages
|
|
|
|
paths = glob.glob(args.aports + "/*/" + package)
|
|
|
|
if len(paths) > 2:
|
|
|
|
raise RuntimeError("Package " + package + " found in multiple"
|
|
|
|
" aports subfolders. Please put it only in one"
|
|
|
|
" folder.")
|
|
|
|
elif len(paths) == 1:
|
|
|
|
ret = paths[0]
|
|
|
|
else:
|
|
|
|
# Search in subpackages
|
|
|
|
for path_current in glob.glob(args.aports + "/*/*/APKBUILD"):
|
|
|
|
apkbuild = pmb.parse.apkbuild(args, path_current)
|
|
|
|
if package in apkbuild["subpackages"]:
|
|
|
|
ret = os.path.dirname(path_current)
|
|
|
|
break
|
|
|
|
|
|
|
|
# Crash when necessary
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
if ret is None and must_exist:
|
2017-05-26 20:35:21 +00:00
|
|
|
raise RuntimeError("Could not find aport for package: " +
|
|
|
|
package)
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
|
2017-07-28 22:34:40 +00:00
|
|
|
# Save result in cache
|
|
|
|
args.cache["find_aport"][package] = ret
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
return ret
|
2017-05-26 20:35:21 +00:00
|
|
|
|
|
|
|
|
|
|
|
def copy_to_buildpath(args, package, suffix="native"):
|
|
|
|
# Sanity check
|
2017-07-28 22:34:40 +00:00
|
|
|
aport = find_aport(args, package)
|
2017-05-26 20:35:21 +00:00
|
|
|
if not os.path.exists(aport + "/APKBUILD"):
|
|
|
|
raise ValueError("Path does not contain an APKBUILD file:" +
|
|
|
|
aport)
|
|
|
|
|
|
|
|
# Clean up folder
|
|
|
|
build = args.work + "/chroot_" + suffix + "/home/user/build"
|
|
|
|
if os.path.exists(build):
|
|
|
|
pmb.chroot.root(args, ["rm", "-rf", "/home/user/build"],
|
|
|
|
suffix=suffix)
|
|
|
|
|
|
|
|
# Copy aport contents
|
|
|
|
pmb.helpers.run.root(args, ["cp", "-r", aport + "/", build])
|
|
|
|
pmb.chroot.root(args, ["chown", "-R", "user:user",
|
|
|
|
"/home/user/build"], suffix=suffix)
|
|
|
|
|
|
|
|
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
def aports_files_out_of_sync_with_git(args, package=None):
|
|
|
|
"""
|
|
|
|
Get a list of files, about which git says, that they have changed in
|
|
|
|
comparison to upstream. We need this for the timestamp based rebuild check,
|
|
|
|
where it does not only rely on the APKBUILD pkgver and pkgrel, but also on
|
|
|
|
the file's last modified date to decide if it needs to be rebuilt. Git sets
|
|
|
|
the last modified timestamp to the last checkout date, so we must ignore
|
|
|
|
all files, that have not been modified, or else we would trigger rebuilds
|
|
|
|
for all packages, from the pmOS binary repository.
|
|
|
|
|
|
|
|
:returns: list of absolute paths to all files not in sync with upstream
|
|
|
|
"""
|
|
|
|
|
|
|
|
# Filter out a specific package
|
|
|
|
if package:
|
|
|
|
ret = []
|
2017-08-15 14:08:48 +00:00
|
|
|
prefix = os.path.realpath(
|
2017-07-28 22:34:40 +00:00
|
|
|
pmb.build.other.find_aport(
|
|
|
|
args, package)) + "/"
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
for file in aports_files_out_of_sync_with_git(args):
|
|
|
|
if file.startswith(prefix):
|
|
|
|
ret.append(file)
|
|
|
|
return ret
|
|
|
|
|
|
|
|
# Use cached result if possible
|
|
|
|
if args.cache["aports_files_out_of_sync_with_git"] is not None:
|
|
|
|
return args.cache["aports_files_out_of_sync_with_git"]
|
|
|
|
|
|
|
|
# Get the aport's git repository folder
|
|
|
|
git_root = None
|
|
|
|
if shutil.which("git"):
|
|
|
|
git_root = pmb.helpers.run.user(args, ["git", "rev-parse",
|
|
|
|
"--show-toplevel"],
|
|
|
|
working_dir=args.aports,
|
|
|
|
return_stdout=True,
|
|
|
|
check=False)
|
|
|
|
if git_root:
|
|
|
|
git_root = git_root.rstrip()
|
|
|
|
ret = []
|
|
|
|
if git_root and os.path.exists(git_root):
|
|
|
|
# Find tracked files out of sync with upstream
|
|
|
|
tracked = pmb.helpers.run.user(args, ["git", "diff", "--name-only", "origin"],
|
|
|
|
working_dir=git_root, return_stdout=True)
|
|
|
|
|
|
|
|
# Find all untracked files
|
|
|
|
untracked = pmb.helpers.run.user(
|
|
|
|
args, ["git", "ls-files", "--others", "--exclude-standard"],
|
|
|
|
working_dir=git_root, return_stdout=True)
|
|
|
|
|
|
|
|
# Set absolute path, filter out aports files
|
2017-08-15 14:08:48 +00:00
|
|
|
aports_absolute = os.path.realpath(args.aports)
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
files = tracked.rstrip().split("\n") + untracked.rstrip().split("\n")
|
|
|
|
for file in files:
|
2017-08-15 14:08:48 +00:00
|
|
|
file = os.path.realpath(git_root + "/" + file)
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
if file.startswith(aports_absolute):
|
|
|
|
ret.append(file)
|
|
|
|
else:
|
|
|
|
logging.warning("WARNING: Can not determine, which aport-files have been"
|
|
|
|
" changed from upstream!")
|
|
|
|
logging.info("* Aports-folder is not a git repository or git is not"
|
|
|
|
" installed")
|
|
|
|
logging.info("* You can turn timestamp-based rebuilds off in"
|
|
|
|
" 'pmbootstrap init'")
|
|
|
|
|
|
|
|
# Save cache
|
|
|
|
args.cache["aports_files_out_of_sync_with_git"] = ret
|
|
|
|
return ret
|
|
|
|
|
|
|
|
|
|
|
|
def sources_newer_than_binary_package(args, package, index_data):
|
|
|
|
path_sources = []
|
2017-07-28 22:34:40 +00:00
|
|
|
for file in glob.glob(args.aports + "/*/" + package + "/*"):
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
path_sources.append(file)
|
|
|
|
|
|
|
|
lastmod_target = float(index_data["timestamp"])
|
|
|
|
return not pmb.helpers.file.is_up_to_date(path_sources,
|
|
|
|
lastmod_target=lastmod_target)
|
|
|
|
|
|
|
|
|
2017-06-15 17:53:48 +00:00
|
|
|
def is_necessary(args, arch, apkbuild, apkindex_path=None):
|
2017-05-26 20:35:21 +00:00
|
|
|
"""
|
2017-06-19 18:33:56 +00:00
|
|
|
Check if the package has already been built. Compared to abuild's check,
|
|
|
|
this check also works for different architectures, and it recognizes
|
|
|
|
changed files in an aport folder, even if the pkgver and pkgrel did not
|
|
|
|
change.
|
2017-05-26 20:35:21 +00:00
|
|
|
|
2017-06-11 12:45:00 +00:00
|
|
|
:param arch: package target architecture
|
|
|
|
:param apkbuild: from pmb.parse.apkbuild()
|
2017-06-15 17:53:48 +00:00
|
|
|
:param apkindex_path: override the APKINDEX.tar.gz path
|
2017-06-11 12:45:00 +00:00
|
|
|
:returns: boolean
|
2017-05-26 20:35:21 +00:00
|
|
|
"""
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
# Get package name, version, define start of debug message
|
2017-05-26 20:35:21 +00:00
|
|
|
package = apkbuild["pkgname"]
|
|
|
|
version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
|
2017-07-19 18:05:34 +00:00
|
|
|
msg = "Build is necessary for package '" + package + "': "
|
2017-05-26 20:35:21 +00:00
|
|
|
|
|
|
|
# Get old version from APKINDEX
|
2017-06-20 18:13:05 +00:00
|
|
|
if apkindex_path:
|
|
|
|
index_data = pmb.parse.apkindex.read(
|
|
|
|
args, package, apkindex_path, False)
|
|
|
|
else:
|
|
|
|
index_data = pmb.parse.apkindex.read_any_index(args, package, arch)
|
|
|
|
if not index_data:
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
logging.debug(msg + "No binary package available")
|
2017-06-20 18:13:05 +00:00
|
|
|
return True
|
2017-05-26 20:35:21 +00:00
|
|
|
|
2017-06-20 18:13:05 +00:00
|
|
|
# a) Binary repo has a newer version
|
|
|
|
version_old = index_data["version"]
|
2017-08-12 14:03:40 +00:00
|
|
|
if pmb.parse.version.compare(version_old, version_new) == 1:
|
2017-06-20 18:13:05 +00:00
|
|
|
logging.warning("WARNING: Package '" + package + "' in your aports folder"
|
2017-06-27 15:56:15 +00:00
|
|
|
" has version " + version_new + ", but the binary package"
|
|
|
|
" repositories already have version " + version_old + "!")
|
2017-06-20 18:13:05 +00:00
|
|
|
return False
|
2017-06-19 18:33:56 +00:00
|
|
|
|
2017-06-20 18:13:05 +00:00
|
|
|
# b) Aports folder has a newer version
|
2017-06-19 18:33:56 +00:00
|
|
|
if version_new != version_old:
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
logging.debug(msg + "Binary package out of date (binary: " + version_old +
|
|
|
|
", aport: " + version_new + ")")
|
2017-06-19 18:33:56 +00:00
|
|
|
return True
|
|
|
|
|
Properly rebuild/install packages when something changed (Fix #120, #108, #131) (#129)
TLDR: Always rebuild/install packages when something changed when executing "pmbootstrap install/initfs/flash", more speed in dependency resolution.
---
pmbootstrap has already gotten some support for "timestamp based rebuilds", which modifies the logic for when packages should be rebuilt. It doesn't only consider packages outdated with old pkgver/pkgrel combinations, but also packages, where a source file has a newer timestamp, than the built package has.
I've found out, that this can lead to more rebuilds than expected. For example, when you check out the pmbootstrap git repository again into another folder, although you have already built packages. Then all files have the timestamp of the checkout, and the packages will appear to be outdated. While this is not largely a concern now, this will become a problem once we have a binary package repository, because then the packages from the binary repo will always seem to be outdated, if you just freshly checked out the repository.
To combat this, git gets asked if the files from the aport we're looking at are in sync with upstream, or not. Only when the files are not in sync with upstream and the timestamps of the sources are newer, a rebuild gets triggered from now on.
In case this logic should fail, I've added an option during "pmbootstrap init" where you can enable or disable the "timestamp based rebuilds" option.
In addition to that, this commit also works on fixing #120: packages do not get updated in "pmbootstrap install" after they have been rebuilt. For this to work, we specify all packages explicitly for abuild, instead of letting abuild do the resolving. This feature will also work with the "timestamp based rebuilds".
This commit also fixes the working_dir argument in pmb.helpers.run.user, which was simply ignored before.
Finally, the performance of the dependency resolution is faster again (when compared to the current version in master), because the parsed apkbuilds and finding the aport by pkgname gets cached during one pmbootstrap call (in args.cache, which also makes it easy to put fake data there in testcases).
The new dependency resolution code can output lots of verbose messages for debugging by specifying the `-v` parameter. The meaning of that changed, it used to output the file names where log messages come from, but no one seemed to use that anyway.
2017-07-10 15:23:43 +00:00
|
|
|
# Aports and binary repo have the same version.
|
|
|
|
if not args.timestamp_based_rebuild:
|
|
|
|
return False
|
|
|
|
|
|
|
|
# c) Same version, source files out of sync with upstream, source
|
|
|
|
# files newer than binary package
|
|
|
|
files_out_of_sync = aports_files_out_of_sync_with_git(args, package)
|
|
|
|
sources_newer = sources_newer_than_binary_package(
|
|
|
|
args, package, index_data)
|
|
|
|
if len(files_out_of_sync) and sources_newer:
|
|
|
|
logging.debug(msg + "Binary package and aport have the same pkgver and"
|
|
|
|
" pkgrel, but there are aport source files out of sync"
|
|
|
|
" with the upstream git repository *and* these source"
|
|
|
|
" files have a more recent 'last modified' timestamp than"
|
|
|
|
" the binary package's build timestamp.")
|
|
|
|
return True
|
|
|
|
|
|
|
|
# d) Same version, source files *in sync* with upstream *or* source
|
|
|
|
# files *older* than binary package
|
|
|
|
else:
|
2017-05-26 20:35:21 +00:00
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
def index_repo(args, arch=None):
|
2017-06-08 16:19:17 +00:00
|
|
|
"""
|
|
|
|
:param arch: when not defined, re-index all repos
|
|
|
|
"""
|
|
|
|
pmb.build.init(args)
|
|
|
|
|
2017-05-26 20:35:21 +00:00
|
|
|
if arch:
|
|
|
|
paths = [args.work + "/packages/" + arch]
|
|
|
|
else:
|
|
|
|
paths = glob.glob(args.work + "/packages/*")
|
|
|
|
|
|
|
|
for path in paths:
|
|
|
|
path_arch = os.path.basename(path)
|
|
|
|
path_repo_chroot = "/home/user/packages/user/" + path_arch
|
|
|
|
logging.info("(native) index " + path_arch + " repository")
|
|
|
|
commands = [
|
|
|
|
["apk", "index", "--output", "APKINDEX.tar.gz_",
|
|
|
|
"--rewrite-arch", path_arch, "*.apk"],
|
|
|
|
["abuild-sign", "APKINDEX.tar.gz_"],
|
|
|
|
["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
|
|
|
|
]
|
|
|
|
for command in commands:
|
|
|
|
pmb.chroot.user(args, command, working_dir=path_repo_chroot)
|
|
|
|
|
|
|
|
|
|
|
|
def symlink_noarch_package(args, arch_apk):
|
|
|
|
"""
|
|
|
|
:param arch_apk: for example: x86_64/mypackage-1.2.3-r0.apk
|
|
|
|
"""
|
|
|
|
|
2017-06-15 00:26:32 +00:00
|
|
|
for arch in pmb.config.build_device_architectures:
|
|
|
|
# Create the arch folder
|
|
|
|
arch_folder = "/home/user/packages/user/" + arch
|
|
|
|
arch_folder_outside = args.work + "/packages/" + arch
|
|
|
|
if not os.path.exists(arch_folder_outside):
|
|
|
|
pmb.chroot.user(args, ["mkdir", "-p", arch_folder])
|
|
|
|
|
|
|
|
# Add symlink, rewrite index
|
|
|
|
pmb.chroot.user(args, ["ln", "-sf", "../" + arch_apk, "."],
|
|
|
|
working_dir=arch_folder)
|
|
|
|
index_repo(args, arch)
|
2017-05-26 20:35:21 +00:00
|
|
|
|
|
|
|
|
|
|
|
def ccache_stats(args, arch):
|
|
|
|
suffix = "native"
|
|
|
|
if args.arch:
|
|
|
|
suffix = "buildroot_" + arch
|
|
|
|
pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False)
|
|
|
|
|
|
|
|
|
|
|
|
# set the correct JOBS count in abuild.conf
|
|
|
|
def configure_abuild(args, suffix, verify=False):
|
|
|
|
path = args.work + "/chroot_" + suffix + "/etc/abuild.conf"
|
|
|
|
prefix = "export JOBS="
|
|
|
|
with open(path, encoding="utf-8") as handle:
|
|
|
|
for line in handle:
|
|
|
|
if not line.startswith(prefix):
|
|
|
|
continue
|
|
|
|
if line != (prefix + args.jobs + "\n"):
|
|
|
|
if verify:
|
|
|
|
raise RuntimeError("Failed to configure abuild: " + path +
|
|
|
|
"\nTry to delete the file (or zap the chroot).")
|
|
|
|
pmb.chroot.root(args, ["sed", "-i", "-e",
|
|
|
|
"s/^" + prefix + ".*/" + prefix + args.jobs + "/",
|
|
|
|
"/etc/abuild.conf"], suffix)
|
|
|
|
configure_abuild(args, suffix, True)
|
|
|
|
return
|
|
|
|
raise RuntimeError("Could not find " + prefix + " line in " + path)
|