292 lines
9.6 KiB
Python
292 lines
9.6 KiB
Python
# Copyright 2023 Luca Weiss
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
import datetime
|
|
import fnmatch
|
|
import logging
|
|
import os
|
|
import re
|
|
import urllib
|
|
from typing import Optional
|
|
|
|
import pmb.helpers.file
|
|
import pmb.helpers.http
|
|
import pmb.helpers.pmaports
|
|
|
|
req_headers = None
|
|
req_headers_github = None
|
|
|
|
ANITYA_API_BASE = "https://release-monitoring.org/api/v2"
|
|
GITHUB_API_BASE = "https://api.github.com"
|
|
GITLAB_HOSTS = [
|
|
"https://gitlab.com",
|
|
"https://gitlab.freedesktop.org",
|
|
"https://gitlab.gnome.org",
|
|
"https://invent.kde.org",
|
|
"https://source.puri.sm",
|
|
]
|
|
|
|
|
|
def init_req_headers() -> None:
|
|
global req_headers
|
|
global req_headers_github
|
|
# Only initialize them once
|
|
if req_headers is not None and req_headers_github is not None:
|
|
return
|
|
# Generic request headers
|
|
req_headers = {
|
|
'User-Agent': f'pmbootstrap/{pmb.__version__} aportupgrade'}
|
|
|
|
# Request headers specific to GitHub
|
|
req_headers_github = dict(req_headers)
|
|
if os.getenv("GITHUB_TOKEN") is not None:
|
|
token = os.getenv("GITHUB_TOKEN")
|
|
req_headers_github['Authorization'] = f'token {token}'
|
|
else:
|
|
logging.info("NOTE: Consider using a GITHUB_TOKEN environment variable"
|
|
" to increase your rate limit")
|
|
|
|
|
|
def get_package_version_info_github(repo_name: str, ref: Optional[str]):
|
|
logging.debug("Trying GitHub repository: {}".format(repo_name))
|
|
|
|
# Get the URL argument to request a special ref, if needed
|
|
ref_arg = ""
|
|
if ref is not None:
|
|
ref_arg = f"?sha={ref}"
|
|
|
|
# Get the commits for the repository
|
|
commits = pmb.helpers.http.retrieve_json(
|
|
f"{GITHUB_API_BASE}/repos/{repo_name}/commits{ref_arg}",
|
|
headers=req_headers_github)
|
|
latest_commit = commits[0]
|
|
commit_date = latest_commit["commit"]["committer"]["date"]
|
|
# Extract the time from the field
|
|
date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%SZ")
|
|
return {
|
|
"sha": latest_commit["sha"],
|
|
"date": date,
|
|
}
|
|
|
|
|
|
def get_package_version_info_gitlab(gitlab_host: str, repo_name: str,
|
|
ref: Optional[str]):
|
|
logging.debug("Trying GitLab repository: {}".format(repo_name))
|
|
|
|
repo_name_safe = urllib.parse.quote(repo_name, safe='')
|
|
|
|
# Get the URL argument to request a special ref, if needed
|
|
ref_arg = ""
|
|
if ref is not None:
|
|
ref_arg = f"?ref_name={ref}"
|
|
|
|
# Get the commits for the repository
|
|
commits = pmb.helpers.http.retrieve_json(
|
|
f"{gitlab_host}/api/v4/projects/{repo_name_safe}/repository"
|
|
f"/commits{ref_arg}",
|
|
headers=req_headers)
|
|
latest_commit = commits[0]
|
|
commit_date = latest_commit["committed_date"]
|
|
# Extract the time from the field
|
|
# 2019-10-14T09:32:00.000Z / 2019-12-27T07:58:53.000-05:00
|
|
date = datetime.datetime.strptime(commit_date, "%Y-%m-%dT%H:%M:%S.000%z")
|
|
return {
|
|
"sha": latest_commit["id"],
|
|
"date": date,
|
|
}
|
|
|
|
|
|
def upgrade_git_package(args, pkgname: str, package) -> bool:
|
|
"""
|
|
Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if
|
|
args.dry is set).
|
|
:param pkgname: the package name
|
|
:param package: a dict containing package information
|
|
:returns: if something (would have) been changed
|
|
"""
|
|
# Get the wanted source line
|
|
source = package["source"][0]
|
|
source = re.split(r"::", source)
|
|
if 1 <= len(source) <= 2:
|
|
source = source[-1]
|
|
else:
|
|
raise RuntimeError("Unhandled number of source elements. Please open"
|
|
f" a bug report: {source}")
|
|
|
|
verinfo = None
|
|
|
|
github_match = re.match(
|
|
r"https://github\.com/(.+)/(?:archive|releases)", source)
|
|
gitlab_match = re.match(
|
|
fr"({'|'.join(GITLAB_HOSTS)})/(.+)/-/archive/", source)
|
|
if github_match:
|
|
verinfo = get_package_version_info_github(
|
|
github_match.group(1), args.ref)
|
|
elif gitlab_match:
|
|
verinfo = get_package_version_info_gitlab(
|
|
gitlab_match.group(1), gitlab_match.group(2), args.ref)
|
|
|
|
if verinfo is None:
|
|
# ignore for now
|
|
logging.warning("{}: source not handled: {}".format(pkgname, source))
|
|
return False
|
|
|
|
# Get the new commit sha
|
|
sha = package["_commit"]
|
|
sha_new = verinfo["sha"]
|
|
|
|
# Format the new pkgver, keep the value before _git the same
|
|
if package["pkgver"] == "9999":
|
|
pkgver = package["_pkgver"]
|
|
else:
|
|
pkgver = package["pkgver"]
|
|
|
|
pkgver_match = re.match(r"([\d.]+)_git", pkgver)
|
|
date_pkgver = verinfo["date"].strftime("%Y%m%d")
|
|
pkgver_new = f"{pkgver_match.group(1)}_git{date_pkgver}"
|
|
|
|
# pkgrel will be zero
|
|
pkgrel = int(package["pkgrel"])
|
|
pkgrel_new = 0
|
|
|
|
if sha == sha_new:
|
|
logging.info("{}: up-to-date".format(pkgname))
|
|
return False
|
|
|
|
logging.info("{}: upgrading pmaport".format(pkgname))
|
|
if args.dry:
|
|
logging.info(f" Would change _commit from {sha} to {sha_new}")
|
|
logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}")
|
|
logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}")
|
|
return True
|
|
|
|
if package["pkgver"] == "9999":
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
|
|
else:
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_commit", sha_new, True)
|
|
return True
|
|
|
|
|
|
def upgrade_stable_package(args, pkgname: str, package) -> bool:
|
|
"""
|
|
Update _commit/pkgver/pkgrel in an APKBUILD (or pretend to do it if
|
|
args.dry is set).
|
|
|
|
:param pkgname: the package name
|
|
:param package: a dict containing package information
|
|
:returns: if something (would have) been changed
|
|
"""
|
|
|
|
# Looking up if there's a custom mapping from postmarketOS package name
|
|
# to Anitya project name.
|
|
mappings = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/packages/?distribution=postmarketOS"
|
|
f"&name={pkgname}", headers=req_headers)
|
|
if mappings["total_items"] < 1:
|
|
projects = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/projects/?name={pkgname}", headers=req_headers)
|
|
if projects["total_items"] < 1:
|
|
logging.warning(f"{pkgname}: failed to get Anitya project")
|
|
return False
|
|
else:
|
|
project_name = mappings["items"][0]["project"]
|
|
ecosystem = mappings["items"][0]["ecosystem"]
|
|
projects = pmb.helpers.http.retrieve_json(
|
|
f"{ANITYA_API_BASE}/projects/?name={project_name}&"
|
|
f"ecosystem={ecosystem}",
|
|
headers=req_headers)
|
|
|
|
if projects["total_items"] < 1:
|
|
logging.warning(f"{pkgname}: didn't find any projects, can't upgrade!")
|
|
return False
|
|
if projects["total_items"] > 1:
|
|
logging.warning(f"{pkgname}: found more than one project, can't "
|
|
f"upgrade! Please create an explicit mapping of "
|
|
f"\"project\" to the package name.")
|
|
return False
|
|
|
|
# Get the first, best-matching item
|
|
project = projects["items"][0]
|
|
|
|
# Check that we got a version number
|
|
if len(project["stable_versions"]) < 1:
|
|
logging.warning("{}: got no version number, ignoring".format(pkgname))
|
|
return False
|
|
|
|
version = project["stable_versions"][0]
|
|
|
|
# Compare the pmaports version with the project version
|
|
if package["pkgver"] == version:
|
|
logging.info("{}: up-to-date".format(pkgname))
|
|
return False
|
|
|
|
if package["pkgver"] == "9999":
|
|
pkgver = package["_pkgver"]
|
|
else:
|
|
pkgver = package["pkgver"]
|
|
|
|
pkgver_new = version
|
|
|
|
pkgrel = package["pkgrel"]
|
|
pkgrel_new = 0
|
|
|
|
if not pmb.parse.version.validate(pkgver_new):
|
|
logging.warning(f"{pkgname}: would upgrade to invalid pkgver:"
|
|
f" {pkgver_new}, ignoring")
|
|
return False
|
|
|
|
logging.info("{}: upgrading pmaport".format(pkgname))
|
|
if args.dry:
|
|
logging.info(f" Would change pkgver from {pkgver} to {pkgver_new}")
|
|
logging.info(f" Would change pkgrel from {pkgrel} to {pkgrel_new}")
|
|
return True
|
|
|
|
if package["pkgver"] == "9999":
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "_pkgver", pkgver_new)
|
|
else:
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgver", pkgver_new)
|
|
|
|
pmb.helpers.file.replace_apkbuild(args, pkgname, "pkgrel", pkgrel_new)
|
|
return True
|
|
|
|
|
|
def upgrade(args, pkgname, git=True, stable=True) -> bool:
|
|
"""
|
|
Find new versions of a single package and upgrade it.
|
|
|
|
:param pkgname: the name of the package
|
|
:param git: True if git packages should be upgraded
|
|
:param stable: True if stable packages should be upgraded
|
|
:returns: if something (would have) been changed
|
|
"""
|
|
# Initialize request headers
|
|
init_req_headers()
|
|
|
|
package = pmb.helpers.pmaports.get(args, pkgname)
|
|
# Run the correct function
|
|
if "_git" in package["pkgver"]:
|
|
if git:
|
|
return upgrade_git_package(args, pkgname, package)
|
|
else:
|
|
if stable:
|
|
return upgrade_stable_package(args, pkgname, package)
|
|
|
|
|
|
def upgrade_all(args) -> None:
|
|
"""
|
|
Upgrade all packages, based on args.all, args.all_git and args.all_stable.
|
|
"""
|
|
for pkgname in pmb.helpers.pmaports.get_list(args):
|
|
# Always ignore postmarketOS-specific packages that have no upstream
|
|
# source
|
|
skip = False
|
|
for pattern in pmb.config.upgrade_ignore:
|
|
if fnmatch.fnmatch(pkgname, pattern):
|
|
skip = True
|
|
if skip:
|
|
continue
|
|
|
|
upgrade(args, pkgname, args.all or args.all_git,
|
|
args.all or args.all_stable)
|