289 lines
11 KiB
Python
289 lines
11 KiB
Python
"""
|
|
Copyright 2017 Oliver Smith
|
|
|
|
This file is part of pmbootstrap.
|
|
|
|
pmbootstrap is free software: you can redistribute it and/or modify
|
|
it under the terms of the GNU General Public License as published by
|
|
the Free Software Foundation, either version 3 of the License, or
|
|
(at your option) any later version.
|
|
|
|
pmbootstrap is distributed in the hope that it will be useful,
|
|
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
GNU General Public License for more details.
|
|
|
|
You should have received a copy of the GNU General Public License
|
|
along with pmbootstrap. If not, see <http://www.gnu.org/licenses/>.
|
|
"""
|
|
import os
|
|
import logging
|
|
import glob
|
|
import shutil
|
|
|
|
import pmb.chroot
|
|
import pmb.helpers.run
|
|
import pmb.helpers.file
|
|
import pmb.parse.apkindex
|
|
|
|
|
|
def find_aport(args, package, must_exist=True):
|
|
"""
|
|
Find the aport, that provides a certain subpackage.
|
|
|
|
:param must_exist: Raise an exception, when not found
|
|
:returns: the full path to the aport folder
|
|
"""
|
|
path = args.aports + "/" + package
|
|
if os.path.exists(path):
|
|
return path
|
|
|
|
# Try to get a cached result first (we assume, that the aports don't change
|
|
# in one pmbootstrap call)
|
|
if path in args.cache["find_aport"]:
|
|
return args.cache["find_aport"][path]
|
|
|
|
ret = None
|
|
for path_current in glob.glob(args.aports + "/*/APKBUILD"):
|
|
apkbuild = pmb.parse.apkbuild(args, path_current)
|
|
if package in apkbuild["subpackages"]:
|
|
ret = os.path.dirname(path_current)
|
|
break
|
|
if ret is None and must_exist:
|
|
raise RuntimeError("Could not find aport for package: " +
|
|
package)
|
|
|
|
args.cache["find_aport"][path] = ret
|
|
return ret
|
|
|
|
|
|
def copy_to_buildpath(args, package, suffix="native"):
|
|
# Sanity check
|
|
aport = args.aports + "/" + package
|
|
if not os.path.exists(aport + "/APKBUILD"):
|
|
raise ValueError("Path does not contain an APKBUILD file:" +
|
|
aport)
|
|
|
|
# Clean up folder
|
|
build = args.work + "/chroot_" + suffix + "/home/user/build"
|
|
if os.path.exists(build):
|
|
pmb.chroot.root(args, ["rm", "-rf", "/home/user/build"],
|
|
suffix=suffix)
|
|
|
|
# Copy aport contents
|
|
pmb.helpers.run.root(args, ["cp", "-r", aport + "/", build])
|
|
pmb.chroot.root(args, ["chown", "-R", "user:user",
|
|
"/home/user/build"], suffix=suffix)
|
|
|
|
|
|
def aports_files_out_of_sync_with_git(args, package=None):
|
|
"""
|
|
Get a list of files, about which git says, that they have changed in
|
|
comparison to upstream. We need this for the timestamp based rebuild check,
|
|
where it does not only rely on the APKBUILD pkgver and pkgrel, but also on
|
|
the file's last modified date to decide if it needs to be rebuilt. Git sets
|
|
the last modified timestamp to the last checkout date, so we must ignore
|
|
all files, that have not been modified, or else we would trigger rebuilds
|
|
for all packages, from the pmOS binary repository.
|
|
|
|
:returns: list of absolute paths to all files not in sync with upstream
|
|
"""
|
|
|
|
# Filter out a specific package
|
|
if package:
|
|
ret = []
|
|
prefix = os.path.abspath(args.aports + "/" + package + "/")
|
|
for file in aports_files_out_of_sync_with_git(args):
|
|
if file.startswith(prefix):
|
|
ret.append(file)
|
|
return ret
|
|
|
|
# Use cached result if possible
|
|
if args.cache["aports_files_out_of_sync_with_git"] is not None:
|
|
return args.cache["aports_files_out_of_sync_with_git"]
|
|
|
|
# Get the aport's git repository folder
|
|
git_root = None
|
|
if shutil.which("git"):
|
|
git_root = pmb.helpers.run.user(args, ["git", "rev-parse",
|
|
"--show-toplevel"],
|
|
working_dir=args.aports,
|
|
return_stdout=True,
|
|
check=False)
|
|
if git_root:
|
|
git_root = git_root.rstrip()
|
|
ret = []
|
|
if git_root and os.path.exists(git_root):
|
|
# Find tracked files out of sync with upstream
|
|
tracked = pmb.helpers.run.user(args, ["git", "diff", "--name-only", "origin"],
|
|
working_dir=git_root, return_stdout=True)
|
|
|
|
# Find all untracked files
|
|
untracked = pmb.helpers.run.user(
|
|
args, ["git", "ls-files", "--others", "--exclude-standard"],
|
|
working_dir=git_root, return_stdout=True)
|
|
|
|
# Set absolute path, filter out aports files
|
|
aports_absolute = os.path.abspath(args.aports)
|
|
files = tracked.rstrip().split("\n") + untracked.rstrip().split("\n")
|
|
for file in files:
|
|
file = os.path.abspath(git_root + "/" + file)
|
|
if file.startswith(aports_absolute):
|
|
ret.append(file)
|
|
else:
|
|
logging.warning("WARNING: Can not determine, which aport-files have been"
|
|
" changed from upstream!")
|
|
logging.info("* Aports-folder is not a git repository or git is not"
|
|
" installed")
|
|
logging.info("* You can turn timestamp-based rebuilds off in"
|
|
" 'pmbootstrap init'")
|
|
|
|
# Save cache
|
|
args.cache["aports_files_out_of_sync_with_git"] = ret
|
|
return ret
|
|
|
|
|
|
def sources_newer_than_binary_package(args, package, index_data):
|
|
path_sources = []
|
|
for file in glob.glob(args.aports + "/" + package + "/*"):
|
|
path_sources.append(file)
|
|
|
|
lastmod_target = float(index_data["timestamp"])
|
|
return not pmb.helpers.file.is_up_to_date(path_sources,
|
|
lastmod_target=lastmod_target)
|
|
|
|
|
|
def is_necessary(args, arch, apkbuild, apkindex_path=None):
|
|
"""
|
|
Check if the package has already been built. Compared to abuild's check,
|
|
this check also works for different architectures, and it recognizes
|
|
changed files in an aport folder, even if the pkgver and pkgrel did not
|
|
change.
|
|
|
|
:param arch: package target architecture
|
|
:param apkbuild: from pmb.parse.apkbuild()
|
|
:param apkindex_path: override the APKINDEX.tar.gz path
|
|
:returns: boolean
|
|
"""
|
|
# Get package name, version, define start of debug message
|
|
package = apkbuild["pkgname"]
|
|
version_new = apkbuild["pkgver"] + "-r" + apkbuild["pkgrel"]
|
|
msg = "Build is neccessary for package '" + package + "': "
|
|
|
|
# Get old version from APKINDEX
|
|
if apkindex_path:
|
|
index_data = pmb.parse.apkindex.read(
|
|
args, package, apkindex_path, False)
|
|
else:
|
|
index_data = pmb.parse.apkindex.read_any_index(args, package, arch)
|
|
if not index_data:
|
|
logging.debug(msg + "No binary package available")
|
|
return True
|
|
|
|
# a) Binary repo has a newer version
|
|
version_old = index_data["version"]
|
|
if pmb.parse.apkindex.compare_version(version_old,
|
|
version_new) == 1:
|
|
logging.warning("WARNING: Package '" + package + "' in your aports folder"
|
|
" has version " + version_new + ", but the binary package"
|
|
" repositories already have version " + version_old + "!")
|
|
return False
|
|
|
|
# b) Aports folder has a newer version
|
|
if version_new != version_old:
|
|
logging.debug(msg + "Binary package out of date (binary: " + version_old +
|
|
", aport: " + version_new + ")")
|
|
return True
|
|
|
|
# Aports and binary repo have the same version.
|
|
if not args.timestamp_based_rebuild:
|
|
return False
|
|
|
|
# c) Same version, source files out of sync with upstream, source
|
|
# files newer than binary package
|
|
files_out_of_sync = aports_files_out_of_sync_with_git(args, package)
|
|
sources_newer = sources_newer_than_binary_package(
|
|
args, package, index_data)
|
|
if len(files_out_of_sync) and sources_newer:
|
|
logging.debug(msg + "Binary package and aport have the same pkgver and"
|
|
" pkgrel, but there are aport source files out of sync"
|
|
" with the upstream git repository *and* these source"
|
|
" files have a more recent 'last modified' timestamp than"
|
|
" the binary package's build timestamp.")
|
|
return True
|
|
|
|
# d) Same version, source files *in sync* with upstream *or* source
|
|
# files *older* than binary package
|
|
else:
|
|
return False
|
|
|
|
|
|
def index_repo(args, arch=None):
|
|
"""
|
|
:param arch: when not defined, re-index all repos
|
|
"""
|
|
pmb.build.init(args)
|
|
|
|
if arch:
|
|
paths = [args.work + "/packages/" + arch]
|
|
else:
|
|
paths = glob.glob(args.work + "/packages/*")
|
|
|
|
for path in paths:
|
|
path_arch = os.path.basename(path)
|
|
path_repo_chroot = "/home/user/packages/user/" + path_arch
|
|
logging.info("(native) index " + path_arch + " repository")
|
|
commands = [
|
|
["apk", "index", "--output", "APKINDEX.tar.gz_",
|
|
"--rewrite-arch", path_arch, "*.apk"],
|
|
["abuild-sign", "APKINDEX.tar.gz_"],
|
|
["mv", "APKINDEX.tar.gz_", "APKINDEX.tar.gz"]
|
|
]
|
|
for command in commands:
|
|
pmb.chroot.user(args, command, working_dir=path_repo_chroot)
|
|
|
|
|
|
def symlink_noarch_package(args, arch_apk):
|
|
"""
|
|
:param arch_apk: for example: x86_64/mypackage-1.2.3-r0.apk
|
|
"""
|
|
|
|
for arch in pmb.config.build_device_architectures:
|
|
# Create the arch folder
|
|
arch_folder = "/home/user/packages/user/" + arch
|
|
arch_folder_outside = args.work + "/packages/" + arch
|
|
if not os.path.exists(arch_folder_outside):
|
|
pmb.chroot.user(args, ["mkdir", "-p", arch_folder])
|
|
|
|
# Add symlink, rewrite index
|
|
pmb.chroot.user(args, ["ln", "-sf", "../" + arch_apk, "."],
|
|
working_dir=arch_folder)
|
|
index_repo(args, arch)
|
|
|
|
|
|
def ccache_stats(args, arch):
|
|
suffix = "native"
|
|
if args.arch:
|
|
suffix = "buildroot_" + arch
|
|
pmb.chroot.user(args, ["ccache", "-s"], suffix, log=False)
|
|
|
|
|
|
# set the correct JOBS count in abuild.conf
|
|
def configure_abuild(args, suffix, verify=False):
|
|
path = args.work + "/chroot_" + suffix + "/etc/abuild.conf"
|
|
prefix = "export JOBS="
|
|
with open(path, encoding="utf-8") as handle:
|
|
for line in handle:
|
|
if not line.startswith(prefix):
|
|
continue
|
|
if line != (prefix + args.jobs + "\n"):
|
|
if verify:
|
|
raise RuntimeError("Failed to configure abuild: " + path +
|
|
"\nTry to delete the file (or zap the chroot).")
|
|
pmb.chroot.root(args, ["sed", "-i", "-e",
|
|
"s/^" + prefix + ".*/" + prefix + args.jobs + "/",
|
|
"/etc/abuild.conf"], suffix)
|
|
configure_abuild(args, suffix, True)
|
|
return
|
|
raise RuntimeError("Could not find " + prefix + " line in " + path)
|