From ce433ae012112650145ca0d73ea5eea3cdfb5a56 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Fri, 10 Apr 2020 10:56:34 +0200 Subject: [PATCH] pmb.helpers.repo.urls: use channels.cfg mirrordirs (MR 1912) Use mirrordir_pmos and mirrordir_alpine from channels.cfg to generate the mirror URLs for postmarketOS and Alpine, which get written to /etc/apk/repositories and which postmarketOS uses to download the APKINDEX files. Remove hardcoded "master" at the end of the postmarketOS mirror and use mirrordir_pmos instead (which is "master" for the edge channel). Let the postmarketOS mirror end in a '/' for consistency with the Alpine mirror in pmb/config/__init__.py. Remove obsolete --alpine-version. To experiment with a different Alpine version, one should pass a custom --config-channels from now on. --- pmb/config/__init__.py | 3 +- pmb/config/pmaports.py | 31 +++++++++++++++++++++ pmb/helpers/repo.py | 25 ++++++++++++++--- pmb/parse/arguments.py | 2 -- test/test_aportgen.py | 5 +++- test/test_aportgen_device_wizard.py | 8 +++++- test/test_config_pmaports.py | 23 +++++++++++++++ test/test_helpers_repo.py | 43 ++++++++++++++++++++++++++++- test/testdata/pmaports.cfg | 5 ++++ 9 files changed, 134 insertions(+), 11 deletions(-) create mode 100644 test/testdata/pmaports.cfg diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 1146c679..1736ecc6 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -59,7 +59,6 @@ config_keys = ["aports", # $WORK gets replaced with the actual value for args.work (which may be # overridden on the commandline) defaults = { - "alpine_version": "edge", # alternatively: latest-stable "aports": "$WORK/cache_git/pmaports", "ccache_size": "5G", "is_default_channel": True, @@ -80,7 +79,7 @@ defaults = { "keymap": "", "log": "$WORK/log.txt", "mirror_alpine": "http://dl-cdn.alpinelinux.org/alpine/", - "mirrors_postmarketos": ["http://postmarketos1.brixit.nl/postmarketos/master"], + "mirrors_postmarketos": ["http://postmarketos1.brixit.nl/postmarketos/"], "nonfree_firmware": True, "nonfree_userland": False, "port_distccd": "33632", diff --git a/pmb/config/pmaports.py b/pmb/config/pmaports.py index 989f1cc7..d2b0b9b4 100644 --- a/pmb/config/pmaports.py +++ b/pmb/config/pmaports.py @@ -118,6 +118,37 @@ def read_config(args): return ret +def read_config_channel(args): + """ Get the properties of the currently active channel in pmaports.git, + as specified in channels.cfg (https://postmarketos.org/channels.cfg). + :returns: {"description: ..., + "branch_pmaports": ..., + "branch_aports": ..., + "mirrordir_alpine": ...} """ + channel = read_config(args)["channel"] + channels_cfg = pmb.helpers.git.parse_channels_cfg(args) + + if channel in channels_cfg["channels"]: + return channels_cfg["channels"][channel] + + # Channel not in channels.cfg, try to be helpful + branch = pmb.helpers.git.rev_parse(args, args.aports, + extra_args=["--abbrev-ref"]) + branches_official = pmb.helpers.git.get_branches_official(args, "pmaports") + branches_official = ", ".join(branches_official) + remote = pmb.helpers.git.get_upstream_remote(args, "pmaports") + logging.info("NOTE: fix the error by rebasing or cherry picking relevant" + " commits from this branch onto a branch that is on a" + f" supported channel: {branches_official}") + logging.info("NOTE: as workaround, you may pass --config-channels with a" + " custom channels.cfg. Reference:" + " https://postmarketos.org/channels.cfg") + raise RuntimeError(f"Current branch '{branch}' of pmaports.git is on" + f" channel '{channel}', but this channel was not" + f" found in channels.cfg (of {remote}/master" + " branch). Looks like a very old branch.") + + def init(args): check_legacy_folder() if not os.path.exists(args.aports): diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index e94de055..0e6e1800 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -8,6 +8,7 @@ Functions that work with binary package repos. See also: import os import hashlib import logging +import pmb.config.pmaports import pmb.helpers.http import pmb.helpers.run @@ -44,6 +45,13 @@ def urls(args, user_repository=True, postmarketos_mirror=True): Get a list of repository URLs, as they are in /etc/apk/repositories. """ ret = [] + + # Get mirrordirs from channels.cfg (postmarketOS mirrordir is the same as + # the pmaports branch of the channel, no need to make it more complicated) + channel_cfg = pmb.config.pmaports.read_config_channel(args) + mirrordir_pmos = channel_cfg["branch_pmaports"] + mirrordir_alpine = channel_cfg["mirrordir_alpine"] + # Local user repository (for packages compiled with pmbootstrap) if user_repository: ret.append("/mnt/pmbootstrap-packages") @@ -51,14 +59,22 @@ def urls(args, user_repository=True, postmarketos_mirror=True): # Upstream postmarketOS binary repository if postmarketos_mirror: for mirror in args.mirrors_postmarketos: - ret.append(mirror) + # Remove "master" mirrordir to avoid breakage until bpo is adjusted + # (build.postmarketos.org#63) and to give potential other users of + # this flag a heads up. + if mirror.endswith("/master"): + logging.warning("WARNING: 'master' at the end of" + " --mirror-pmOS is deprecated, the branch gets" + " added automatically now!") + mirror = mirror[:-1 * len("master")] + ret.append(f"{mirror}{mirrordir_pmos}") # Upstream Alpine Linux repositories directories = ["main", "community"] - if args.alpine_version == "edge": + if mirrordir_alpine == "edge": directories.append("testing") for dir in directories: - ret.append(args.mirror_alpine + args.alpine_version + "/" + dir) + ret.append(f"{args.mirror_alpine}{mirrordir_alpine}/{dir}") return ret @@ -181,6 +197,7 @@ def alpine_apkindex_path(args, repo="main", arch=None): update(args, arch) # Find it on disk - repo_link = args.mirror_alpine + args.alpine_version + "/" + repo + channel_cfg = pmb.config.pmaports.read_config_channel(args) + repo_link = f"{args.mirror_alpine}{channel_cfg['mirrordir_alpine']}/{repo}" cache_folder = args.work + "/cache_apk_" + arch return cache_folder + "/APKINDEX." + hash(repo_link) + ".tar.gz" diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index d932d781..4b91e47a 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -336,8 +336,6 @@ def arguments(): # Other parser.add_argument("-V", "--version", action="version", version=pmb.config.version) - parser.add_argument("-a", "--alpine-version", dest="alpine_version", - help="examples: edge, latest-stable, v3.5") parser.add_argument("-c", "--config", dest="config", default=pmb.config.defaults["config"], help="path to pmbootstrap.cfg file (default in" diff --git a/test/test_aportgen.py b/test/test_aportgen.py index 8e32e162..f6921815 100644 --- a/test/test_aportgen.py +++ b/test/test_aportgen.py @@ -17,7 +17,8 @@ import pmb.helpers.logging @pytest.fixture def args(tmpdir, request): import pmb.parse - sys.argv = ["pmbootstrap.py", "chroot"] + cfg = f"{pmb_test.const.testdata}/channels.cfg" + sys.argv = ["pmbootstrap.py", "--config-channels", cfg, "chroot"] args = pmb.parse.arguments() args.log = args.work + "/log_testsuite.txt" args.fork_alpine = False @@ -74,9 +75,11 @@ def test_aportgen_fork_alpine_compare_output(args, tmpdir, monkeypatch): def test_aportgen(args, tmpdir): # Fake aports folder in tmpdir + testdata = pmb_test.const.testdata tmpdir = str(tmpdir) shutil.copytree(args.aports + "/.git", tmpdir + "/.git") args.aports = tmpdir + shutil.copy(f"{testdata}/pmaports.cfg", args.aports) os.mkdir(tmpdir + "/cross") # Create aportgen folder -> code path where it still exists diff --git a/test/test_aportgen_device_wizard.py b/test/test_aportgen_device_wizard.py index 26a2ab15..e17381ec 100644 --- a/test/test_aportgen_device_wizard.py +++ b/test/test_aportgen_device_wizard.py @@ -6,6 +6,7 @@ import sys import shutil import pmb_test # noqa +import pmb_test.const import pmb.aportgen import pmb.config import pmb.helpers.logging @@ -14,7 +15,9 @@ import pmb.parse @pytest.fixture def args(tmpdir, request): - sys.argv = ["pmbootstrap.py", "build", "-i", "device-testsuite-testdevice"] + cfg = f"{pmb_test.const.testdata}/channels.cfg" + sys.argv = ["pmbootstrap.py", "--config-channels", cfg, "build", "-i", + "device-testsuite-testdevice"] args = pmb.parse.arguments() args.log = args.work + "/log_testsuite.txt" pmb.helpers.logging.init(args) @@ -35,6 +38,9 @@ def args(tmpdir, request): pmb.helpers.run.user(args, ["mkdir", "-p", tmpdir + "/device/testing"]) path_mako = args._aports_real + "/device/testing/linux-lg-mako" pmb.helpers.run.user(args, ["cp", "-r", path_mako, tmpdir + "/device/testing"]) + + # Copy pmaports.cfg + shutil.copy(f"{pmb_test.const.testdata}/pmaports.cfg", args.aports) return args diff --git a/test/test_config_pmaports.py b/test/test_config_pmaports.py index 3f6d7432..a9ff7f25 100644 --- a/test/test_config_pmaports.py +++ b/test/test_config_pmaports.py @@ -48,3 +48,26 @@ def test_switch_to_channel_branch(args, monkeypatch, tmpdir): assert func(args, "stable") is True branch = pmb.helpers.git.rev_parse(args, path, extra_args=["--abbrev-ref"]) assert branch == "v20.05" + + +def test_read_config_channel(args, monkeypatch): + channel = "edge" + + # Pretend to have a certain channel in pmaports.cfg + def read_config(args): + return {"channel": channel} + monkeypatch.setattr(pmb.config.pmaports, "read_config", read_config) + + # Channel found + func = pmb.config.pmaports.read_config_channel + exp = {"description": "Rolling release channel", + "branch_pmaports": "master", + "branch_aports": "master", + "mirrordir_alpine": "edge"} + assert func(args) == exp + + # Channel not found + channel = "non-existing" + with pytest.raises(RuntimeError) as e: + func(args) + assert "channel was not found in channels.cfg" in str(e.value) diff --git a/test/test_helpers_repo.py b/test/test_helpers_repo.py index ab40cee9..bfeac2a9 100644 --- a/test/test_helpers_repo.py +++ b/test/test_helpers_repo.py @@ -5,13 +5,15 @@ import pytest import sys import pmb_test # noqa +import pmb_test.const import pmb.helpers.repo @pytest.fixture def args(tmpdir, request): import pmb.parse - sys.argv = ["pmbootstrap.py", "chroot"] + cfg = f"{pmb_test.const.testdata}/channels.cfg" + sys.argv = ["pmbootstrap.py", "--config-channels", cfg, "chroot"] args = pmb.parse.arguments() args.log = args.work + "/log_testsuite.txt" pmb.helpers.logging.init(args) @@ -30,3 +32,42 @@ def test_alpine_apkindex_path(args): args.mirror_alpine = "http://dl-cdn.alpinelinux.org/alpine/" ret = args.work + "/cache_apk_armhf/APKINDEX.30e6f5af.tar.gz" assert func(args, "testing", "armhf") == ret + + +def test_urls(args, monkeypatch): + func = pmb.helpers.repo.urls + channel = "stable" + args.mirror_alpine = "http://localhost/alpine/" + + # Second mirror with /master at the end is legacy, gets fixed by func. + # Note that bpo uses multiple postmarketOS mirrors at the same time, so it + # can use its WIP repository together with the final repository. + args.mirrors_postmarketos = ["http://localhost/pmos1/", + "http://localhost/pmos2/master"] + + # Pretend to have a certain channel in pmaports.cfg + def read_config(args): + return {"channel": channel} + monkeypatch.setattr(pmb.config.pmaports, "read_config", read_config) + + # Channel: stable + assert func(args) == ["/mnt/pmbootstrap-packages", + "http://localhost/pmos1/v20.05", + "http://localhost/pmos2/v20.05", + "http://localhost/alpine/v3.11/main", + "http://localhost/alpine/v3.11/community"] + + # Channel: edge (has Alpine's testing) + channel = "edge" + assert func(args) == ["/mnt/pmbootstrap-packages", + "http://localhost/pmos1/master", + "http://localhost/pmos2/master", + "http://localhost/alpine/edge/main", + "http://localhost/alpine/edge/community", + "http://localhost/alpine/edge/testing"] + + # Only Alpine's URLs + exp = ["http://localhost/alpine/edge/main", + "http://localhost/alpine/edge/community", + "http://localhost/alpine/edge/testing"] + assert func(args, False, False) == exp diff --git a/test/testdata/pmaports.cfg b/test/testdata/pmaports.cfg new file mode 100644 index 00000000..249c65a9 --- /dev/null +++ b/test/testdata/pmaports.cfg @@ -0,0 +1,5 @@ +# Reference: https://postmarketos.org/pmaports.cfg +[pmaports] +version=9999 +pmbootstrap_min_version=1.18.0 +channel=edge