From 33e3553cdd41c550a480de18cdcb229b48b48f33 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Sat, 9 Feb 2019 17:58:54 +0100 Subject: [PATCH] aportupgrade command for upgrading APKBUILDs (!1752) The gist of this action is upgrading the specified aport to the latest version. There are implementations for both stable packages (which check via the release-monitoring.org API for new versions) and git packages (which check the GitLab/GitHub API for new commits on the main branch). There's also the possibility to pass --all, --all-stable & --all-git to the action which either loops through all packages, or just stable or git packages and upgrades them. The --dry argument is also respected. Note, that the implementation does update the variables pkgver, pkgrel and _commit but it doesn't update the checksums because that would slow down the process a lot, and is potentially undesirable. --- pmb/config/__init__.py | 11 ++ pmb/helpers/aportupgrade.py | 265 ++++++++++++++++++++++++++++++++++++ pmb/helpers/frontend.py | 16 ++- pmb/parse/arguments.py | 15 ++ 4 files changed, 306 insertions(+), 1 deletion(-) create mode 100644 pmb/helpers/aportupgrade.py diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 0f36a83d..658f9619 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -244,6 +244,11 @@ apkbuild_attributes = { # Overridden packages "_pkgver": {"array": False}, + "_pkgname": {"array": False}, + + # git commit + "_commit": {"array": False}, + "source": {"array": True}, } # Variables from deviceinfo. Reference: @@ -491,3 +496,9 @@ newapkbuild_arguments_switches_other = [ ["-s", "sourceforge", "use sourceforge source URL"], ["-c", "copy_samples", "copy a sample init.d, conf.d and install script"], ] + +# +# UPGRADE +# +# Patterns of package names to ignore for automatic pmaport upgrading ("pmbootstrap aportupgrade --all") +upgrade_ignore = ["device-*", "firmware-*", "linux-*", "postmarketos-*", "*-aarch64", "*-armhf", "*-armv7"] diff --git a/pmb/helpers/aportupgrade.py b/pmb/helpers/aportupgrade.py new file mode 100644 index 00000000..52c82091 --- /dev/null +++ b/pmb/helpers/aportupgrade.py @@ -0,0 +1,265 @@ +""" +Copyright 2020 Luca Weiss + +This file is part of pmbootstrap. + +pmbootstrap is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +pmbootstrap is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with pmbootstrap. If not, see . +""" +import datetime +import fnmatch +import logging +import os +import re +import urllib + +import pmb.helpers.file +import pmb.helpers.http +import pmb.helpers.pmaports + +req_headers = None +req_headers_github = None + +ANITYA_API_BASE = "https://release-monitoring.org/api/v2" +GITHUB_API_BASE = "https://api.github.com" +GITLAB_HOSTS = [ + "https://gitlab.com", + "https://invent.kde.org", + "https://source.puri.sm", + "https://gitlab.freedesktop.org", +] + + +def init_req_headers() -> None: + global req_headers + global req_headers_github + # Only initialize them once + if req_headers is not None and req_headers_github is not None: + return + # Generic request headers + req_headers = {'User-Agent': 'pmbootstrap/{} aportupgrade'.format(pmb.config.version)} + + # Request headers specific to GitHub + req_headers_github = dict(req_headers) + if os.getenv("GITHUB_TOKEN") is not None: + req_headers_github['Authorization'] = 'token ' + os.getenv("GITHUB_TOKEN") + else: + logging.info("NOTE: Consider using a GITHUB_TOKEN environment variable to increase your rate limit") + + +def get_github_branch_arg(repo: str) -> str: + """ + Get the branch to query for the latest commit + :param repo: the repository name + :returns: e.g. "?sha=bionic" or "" + """ + if "ubports" not in repo: + return "" + # Get a list of branches to see if a 'bionic' branch exists + branches = pmb.helpers.http.retrieve_json(GITHUB_API_BASE + "/repos/" + repo + "/branches", + headers=req_headers_github) + for branch_o in branches: + if branch_o["name"] == "bionic": + return "?sha=bionic" + # Return no branch if 'bionic' does not exist + return "" + + +def get_package_version_info_github(repo_name: str): + logging.debug("Trying GitHub repository: {}".format(repo_name)) + + # Special case for ubports Unity 8 repos, we want to use the 'bionic' branch (where available) + branch = get_github_branch_arg(repo_name) + + # Get the commits for the repository + commits = pmb.helpers.http.retrieve_json(GITHUB_API_BASE + "/repos/" + repo_name + "/commits" + branch, + headers=req_headers_github) + latest_commit = commits[0] + commit_date = latest_commit["commit"]["committer"]["date"] + # Extract the time from the field + date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%SZ") + return { + "sha": latest_commit["sha"], + "date": date, + } + + +def get_package_version_info_gitlab(gitlab_host: str, repo_name: str): + logging.debug("Trying GitLab repository: {}".format(repo_name)) + + # Get the commits for the repository + commits = pmb.helpers.http.retrieve_json( + gitlab_host + "/api/v4/projects/" + urllib.parse.quote(repo_name, safe='') + "/repository/commits", + headers=req_headers) + latest_commit = commits[0] + commit_date = latest_commit["committed_date"] + # Extract the time from the field + # 2019-10-14T09:32:00.000Z / 2019-12-27T07:58:53.000-05:00 + date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%S.000%z") + return { + "sha": latest_commit["id"], + "date": date, + } + + +def upgrade_git_package(args, pkgname: str, package) -> bool: + """ + Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if args.dry is set). + :param pkgname: the package name + :param package: a dict containing package information + :returns: if something (would have) been changed + """ + # Get the wanted source line + source = package["source"][0] + source = re.split(r"::", source) + if 1 <= len(source) <= 2: + source = source[-1] + else: + raise RuntimeError("Unhandled number of source elements. Please open a bug report: {}".format(source)) + + verinfo = None + + github_match = re.match(r"https://github\.com/(.+)/(?:archive|releases)", source) + gitlab_match = re.match(r"(" + '|'.join(GITLAB_HOSTS) + ")/(.+)/-/archive/", source) + if github_match: + verinfo = get_package_version_info_github(github_match.group(1)) + elif gitlab_match: + verinfo = get_package_version_info_gitlab(gitlab_match.group(1), gitlab_match.group(2)) + + if verinfo is None: + # ignore for now + logging.warning("{}: source not handled: {}".format(pkgname, source)) + return False + + # Get the new commit sha + sha = package["_commit"] + sha_new = verinfo["sha"] + + # Format the new pkgver, keep the value before _git the same + pkgver = package["pkgver"] + pkgver_match = re.match(r"([\d.]+)_git", pkgver) + date_pkgver = verinfo["date"].strftime("%Y%m%d") + pkgver_new = pkgver_match.group(1) + "_git" + date_pkgver + + # pkgrel will be zero + pkgrel = int(package["pkgrel"]) + pkgrel_new = 0 + + if sha == sha_new: + logging.info("{}: up-to-date".format(pkgname)) + return False + + logging.info("{}: upgrading pmaport".format(pkgname)) + if args.dry: + logging.info(" Would change _commit from {} to {}".format(sha, sha_new)) + logging.info(" Would change pkgver from {} to {}".format(pkgver, pkgver_new)) + logging.info(" Would change pkgrel from {} to {}".format(pkgrel, pkgrel_new)) + return True + + pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new) + pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new) + pmb.helpers.file.replace_apkbuild(args, pkgname, "_commit", sha_new, True) + return True + + +def upgrade_stable_package(args, pkgname: str, package) -> bool: + """ + Update _commit/pkgver/pkgrel in an APKBUILD (or pretend to do it if args.dry is set). + + :param pkgname: the package name + :param package: a dict containing package information + :returns: if something (would have) been changed + """ + projects = pmb.helpers.http.retrieve_json(ANITYA_API_BASE + "/projects/?name=" + pkgname, headers=req_headers) + if projects["total_items"] < 1: + # There is no Anitya project with the package name. + # Looking up if there's a custom mapping from postmarketOS package name to Anitya project name. + mappings = pmb.helpers.http.retrieve_json( + ANITYA_API_BASE + "/packages/?distribution=postmarketOS&name=" + pkgname, headers=req_headers) + if mappings["total_items"] < 1: + logging.warning("{}: failed to get Anitya project".format(pkgname)) + return False + project_name = mappings["items"][0]["project"] + projects = pmb.helpers.http.retrieve_json( + ANITYA_API_BASE + "/projects/?name=" + project_name, headers=req_headers) + + # Get the first, best-matching item + project = projects["items"][0] + + # Check that we got a version number + if project["version"] is None: + logging.warning("{}: got no version number, ignoring".format(pkgname)) + return False + + # Compare the pmaports version with the project version + if package["pkgver"] == project["version"]: + logging.info("{}: up-to-date".format(pkgname)) + return False + + pkgver = package["pkgver"] + pkgver_new = project["version"] + + pkgrel = package["pkgrel"] + pkgrel_new = 0 + + if not pmb.parse.version.validate(pkgver_new): + logging.warning("{}: would upgrade to invalid pkgver: {}, ignoring".format(pkgname, pkgver_new)) + return False + + logging.info("{}: upgrading pmaport".format(pkgname)) + if args.dry: + logging.info(" Would change pkgver from {} to {}".format(pkgver, pkgver_new)) + logging.info(" Would change pkgrel from {} to {}".format(pkgrel, pkgrel_new)) + return True + + pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new) + pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new) + return True + + +def upgrade(args, pkgname, git=True, stable=True) -> bool: + """ + Find new versions of a single package and upgrade it. + + :param pkgname: the name of the package + :param git: True if git packages should be upgraded + :param stable: True if stable packages should be upgraded + :returns: if something (would have) been changed + """ + # Initialize request headers + init_req_headers() + + package = pmb.helpers.pmaports.get(args, pkgname) + # Run the correct function + if "_git" in package["pkgver"]: + if git: + return upgrade_git_package(args, pkgname, package) + else: + if stable: + return upgrade_stable_package(args, pkgname, package) + + +def upgrade_all(args) -> None: + """ + Upgrade all packages, based on args.all, args.all_git and args.all_stable. + """ + for pkgname in pmb.helpers.pmaports.get_list(args): + # Always ignore postmarketOS-specific packages that have no upstream source + skip = False + for pattern in pmb.config.upgrade_ignore: + if fnmatch.fnmatch(pkgname, pattern): + skip = True + if skip: + continue + + upgrade(args, pkgname, args.all or args.all_git, args.all or args.all_stable) diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index d08ebf91..3227aa3c 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -28,10 +28,10 @@ import sys import pmb.aportgen import pmb.build import pmb.build.autodetect -import pmb.config import pmb.chroot import pmb.chroot.initfs import pmb.chroot.other +import pmb.config import pmb.export import pmb.flasher import pmb.helpers.git @@ -42,6 +42,7 @@ import pmb.helpers.pmaports import pmb.helpers.repo import pmb.helpers.repo_missing import pmb.helpers.run +import pmb.helpers.aportupgrade import pmb.install import pmb.parse import pmb.qemu @@ -340,6 +341,19 @@ def pkgrel_bump(args): sys.exit(1) +def aportupgrade(args): + if args.all or args.all_stable or args.all_git: + pmb.helpers.aportupgrade.upgrade_all(args) + else: + # Each package must exist + for package in args.packages: + pmb.helpers.pmaports.find(args, package) + + # Check each package for a new version + for package in args.packages: + pmb.helpers.aportupgrade.upgrade(args, package) + + def qemu(args): pmb.qemu.run(args) diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index ceb7c3a3..0ddeecc7 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -178,6 +178,20 @@ def arguments_pkgrel_bump(subparser): return ret +def arguments_aportupgrade(subparser): + ret = subparser.add_parser("aportupgrade") + ret.add_argument("--dry", action="store_true", help="instead of modifying APKBUILDs," + " print the changes that would be made") + + # Mutually exclusive: "--all" or package names + mode = ret.add_mutually_exclusive_group(required=True) + mode.add_argument("--all", action="store_true", help="iterate through all packages") + mode.add_argument("--all-stable", action="store_true", help="iterate through all non-git packages") + mode.add_argument("--all-git", action="store_true", help="iterate through all git packages") + mode.add_argument("packages", nargs="*", default=[]) + return ret + + def arguments_newapkbuild(subparser): """ Wrapper for Alpine's "newapkbuild" command. @@ -386,6 +400,7 @@ def arguments(): arguments_initfs(sub) arguments_qemu(sub) arguments_pkgrel_bump(sub) + arguments_aportupgrade(sub) arguments_newapkbuild(sub) arguments_lint(sub)