From e04712a636e52c214d27626b67190fc8838c99b4 Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Mon, 6 Jan 2020 06:39:57 +0100 Subject: [PATCH] pmbootstrap pull: new action (!1848) Add a shortcut for "git pull --ff-only" in all repositories cloned by pmbootstrap (currently pmaports and aports_upstream, new pmdevices repository coming soon). 'pmbootstrap pull' will only update the repositories, if: * they are on an officially supported branch (e.g. master) * the history is not conflicting (fast-forward is possible) * the git workdirs are clean Otherwise it shows the user a descriptive message about what to do. The list of supported branches is only "master" right now, and will be extended in later commits, so we can have a stable branch for pmaports based on Alpine's releases. More about that in the project direction 2020 issue. Closes: #1858 --- pmb/config/pmaports.py | 3 +- pmb/helpers/frontend.py | 27 +++++++ pmb/helpers/git.py | 94 +++++++++++++++++++++++ pmb/parse/arguments.py | 4 + test/test_helpers_git.py | 158 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 283 insertions(+), 3 deletions(-) diff --git a/pmb/config/pmaports.py b/pmb/config/pmaports.py index d9259af3..f1be7111 100644 --- a/pmb/config/pmaports.py +++ b/pmb/config/pmaports.py @@ -75,8 +75,7 @@ def check_version_pmaports(args): # Outated error logging.info("NOTE: your pmaports folder has version " + real + ", but" + " version " + min + " is required.") - raise RuntimeError("Please update your local pmaports repository. Usually" - " with: 'git -C \"" + args.aports + "\" pull'") + raise RuntimeError("Run 'pmbootstrap pull' to update your pmaports.") def check_version_pmbootstrap(args): diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 1cf65282..bfdcbbee 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -34,6 +34,7 @@ import pmb.chroot.initfs import pmb.chroot.other import pmb.export import pmb.flasher +import pmb.helpers.git import pmb.helpers.logging import pmb.helpers.pkgrel_bump import pmb.helpers.pmaports @@ -388,3 +389,29 @@ def bootimg_analyze(args): for line in pmb.aportgen.device.generate_deviceinfo_fastboot_content(args, bootimg).split("\n"): tmp_output += "\n" + line.lstrip() logging.info(tmp_output) + + +def pull(args): + failed = [] + for repo in pmb.config.git_repos.keys(): + if pmb.helpers.git.pull(args, repo) < 0: + failed.append(repo) + + if not failed: + return True + + logging.info("---") + logging.info("WARNING: failed to update: " + ", ".join(failed)) + logging.info("") + logging.info("'pmbootstrap pull' will only update the repositories, if:") + logging.info("* they are on an officially supported branch (e.g. master)") + logging.info("* the history is not conflicting (fast-forward is possible)") + logging.info("* the git workdirs are clean") + logging.info("You have changed mentioned repositories, so they don't meet") + logging.info("these conditions anymore.") + logging.info("") + logging.info("Fix and try again:") + for name_repo in failed: + logging.info("* " + pmb.helpers.git.get_path(args, name_repo)) + logging.info("---") + return False diff --git a/pmb/helpers/git.py b/pmb/helpers/git.py index 504bc10c..4de42251 100644 --- a/pmb/helpers/git.py +++ b/pmb/helpers/git.py @@ -78,3 +78,97 @@ def rev_parse(args, path, revision="HEAD", extra_args: list = []): command = ["git", "rev-parse"] + extra_args + [revision] rev = pmb.helpers.run.user(args, command, path, output_return=True) return rev.rstrip() + + +def can_fast_forward(args, path, branch_upstream, branch="HEAD"): + command = ["git", "merge-base", "--is-ancestor", branch, branch_upstream] + ret = pmb.helpers.run.user(args, command, path, check=False) + if ret == 0: + return True + elif ret == 1: + return False + else: + raise RuntimeError("Unexpected exit code from git: " + str(ret)) + + +def clean_worktree(args, path): + """ Check if there are not any modified files in the git dir. """ + command = ["git", "status", "--porcelain"] + return pmb.helpers.run.user(args, command, path, output_return=True) == "" + + +def get_upstream_remote(args, name_repo): + """ Find the remote, which matches the git URL from the config. Usually + "origin", but the user may have set up their git repository + differently. """ + url = pmb.config.git_repos[name_repo] + path = get_path(args, name_repo) + command = ["git", "remote", "-v"] + output = pmb.helpers.run.user(args, command, path, output_return=True) + for line in output.split("\n"): + if url in line: + return line.split("\t", 1)[0] + raise RuntimeError("{}: could not find remote name for URL '{}' in git" + " repository: {}".format(name_repo, url, path)) + + +def pull(args, name_repo): + """ Check if on official branch and essentially try 'git pull --ff-only'. + Instead of really doing 'git pull --ff-only', do it in multiple steps + (fetch, merge --ff-only), so we can display useful messages depending + on which part fails. + + :returns: integer, >= 0 on success, < 0 on error """ + branches_official = ["master"] + + # Skip if repo wasn't cloned + path = get_path(args, name_repo) + if not os.path.exists(path): + logging.debug(name_repo + ": repo was not cloned, skipping pull!") + return 1 + + # Skip if not on official branch + branch = rev_parse(args, path, extra_args=["--abbrev-ref"]) + msg_start = "{} (branch: {}):".format(name_repo, branch) + if branch not in branches_official: + logging.warning("{} not on one of the official branches ({}), skipping" + " pull!" + "".format(msg_start, ", ".join(branches_official))) + return -1 + + # Skip if workdir is not clean + if not clean_worktree(args, path): + logging.warning(msg_start + " workdir is not clean, skipping pull!") + return -2 + + # Skip if branch is tracking different remote + branch_upstream = get_upstream_remote(args, name_repo) + "/" + branch + remote_ref = rev_parse(args, path, branch + "@{u}", ["--abbrev-ref"]) + if remote_ref != branch_upstream: + logging.warning("{} is tracking unexpected remote branch '{}' instead" + " of '{}'".format(msg_start, remote_ref, + branch_upstream)) + return -3 + + # Fetch (exception on failure, meaning connection to server broke) + logging.info(msg_start + " git pull --ff-only") + if not args.offline: + pmb.helpers.run.user(args, ["git", "fetch"], path) + + # Skip if already up to date + if rev_parse(args, path, branch) == rev_parse(args, path, branch_upstream): + logging.info(msg_start + " already up to date") + return 2 + + # Skip if we can't fast-forward + if not can_fast_forward(args, path, branch_upstream): + logging.warning("{} can't fast-forward to {}, looks like you changed" + " the git history of your local branch. Skipping pull!" + "".format(msg_start, branch_upstream)) + return -4 + + # Fast-forward now (should not fail due to checks above, so it's fine to + # throw an exception on error) + command = ["git", "merge", "--ff-only", branch_upstream] + pmb.helpers.run.user(args, command, path, "stdout") + return 0 diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 799601ab..275f4584 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -552,6 +552,10 @@ def arguments(): help="force even if the file seems to be" " invalid") + # Action: pull + sub.add_parser("pull", help="update all git repositories that pmbootstrap" + " cloned (pmaports, etc.)") + if argcomplete: argcomplete.autocomplete(parser, always_complete_options="long") diff --git a/test/test_helpers_git.py b/test/test_helpers_git.py index d5a9ab35..d4c95d6d 100644 --- a/test/test_helpers_git.py +++ b/test/test_helpers_git.py @@ -1,5 +1,5 @@ """ -Copyright 2019 Oliver Smith +Copyright 2020 Oliver Smith This file is part of pmbootstrap. @@ -20,12 +20,14 @@ along with pmbootstrap. If not, see . import os import sys import pytest +import shutil # Import from parent directory sys.path.insert(0, os.path.realpath( os.path.join(os.path.dirname(__file__) + "/.."))) import pmb.helpers.git import pmb.helpers.logging +import pmb.helpers.run @pytest.fixture @@ -46,3 +48,157 @@ def test_get_path(args): assert func(args, "aports_upstream") == "/wrk/cache_git/aports_upstream" assert func(args, "pmaports") == "/tmp/pmaports" + + +def test_can_fast_forward(args, tmpdir): + tmpdir = str(tmpdir) + func = pmb.helpers.git.can_fast_forward + branch_origin = "fake-branch-origin" + + def run_git(git_args): + pmb.helpers.run.user(args, ["git"] + git_args, tmpdir, "stdout") + + # Create test git repo + run_git(["init", "."]) + run_git(["commit", "--allow-empty", "-m", "commit on master"]) + run_git(["checkout", "-b", branch_origin]) + run_git(["commit", "--allow-empty", "-m", "commit on branch_origin"]) + run_git(["checkout", "master"]) + + # Can fast-forward + assert func(args, tmpdir, branch_origin) is True + + # Can't fast-forward + run_git(["commit", "--allow-empty", "-m", "commit on master #2"]) + assert func(args, tmpdir, branch_origin) is False + + # Git command fails + with pytest.raises(RuntimeError) as e: + func(args, tmpdir, "invalid-branch") + assert str(e.value).startswith("Unexpected exit code") + + +def test_clean_worktree(args, tmpdir): + tmpdir = str(tmpdir) + func = pmb.helpers.git.clean_worktree + + def run_git(git_args): + pmb.helpers.run.user(args, ["git"] + git_args, tmpdir, "stdout") + + # Create test git repo + run_git(["init", "."]) + run_git(["commit", "--allow-empty", "-m", "commit on master"]) + + assert func(args, tmpdir) is True + pmb.helpers.run.user(args, ["touch", "test"], tmpdir) + assert func(args, tmpdir) is False + + +def test_get_upstream_remote(args, monkeypatch, tmpdir): + tmpdir = str(tmpdir) + func = pmb.helpers.git.get_upstream_remote + name_repo = "test" + + # Override get_path() + def get_path(args, name_repo): + return tmpdir + monkeypatch.setattr(pmb.helpers.git, "get_path", get_path) + + # Override pmb.config.git_repos + url = "https://postmarketos.org/get-upstream-remote-test.git" + git_repos = {"test": url} + monkeypatch.setattr(pmb.config, "git_repos", git_repos) + + def run_git(git_args): + pmb.helpers.run.user(args, ["git"] + git_args, tmpdir, "stdout") + + # Create git repo + run_git(["init", "."]) + run_git(["commit", "--allow-empty", "-m", "commit on master"]) + + # No upstream remote + with pytest.raises(RuntimeError) as e: + func(args, name_repo) + assert "could not find remote name for URL" in str(e.value) + + run_git(["remote", "add", "hello", url]) + assert func(args, name_repo) == "hello" + + +def test_pull_non_existing(args): + assert pmb.helpers.git.pull(args, "non-existing-repo-name") == 1 + + +def test_pull(args, monkeypatch, tmpdir): + """ Test pmb.helpers.git.pull """ + # --- PREPARATION: git repos --- + # Prepare three git repos: + # * local: like local clone of pmaports.git + # * remote: emulate a remote repository, that we can add to "local", so we + # can pass the tracking-remote tests in pmb.helpers.git.pull + # * remote2: unexpected remote, that pmbootstrap can complain about + path_local = str(tmpdir) + "/local" + path_remote = str(tmpdir) + "/remote" + path_remote2 = str(tmpdir) + "/remote2" + os.makedirs(path_local) + os.makedirs(path_remote) + os.makedirs(path_remote2) + + def run_git(git_args, path=path_local): + pmb.helpers.run.user(args, ["git"] + git_args, path, "stdout") + + # Remote repos + run_git(["init", "."], path_remote) + run_git(["commit", "--allow-empty", "-m", "commit: remote"], path_remote) + run_git(["init", "."], path_remote2) + run_git(["commit", "--allow-empty", "-m", "commit: remote2"], path_remote2) + + # Local repo (with master -> origin2/master) + run_git(["init", "."]) + run_git(["remote", "add", "-f", "origin", path_remote]) + run_git(["remote", "add", "-f", "origin2", path_remote2]) + run_git(["checkout", "-b", "master", "--track", "origin2/master"]) + + # --- PREPARATION: function overrides --- + # get_path() + def get_path(args, name_repo): + return path_local + monkeypatch.setattr(pmb.helpers.git, "get_path", get_path) + + # get_upstream_remote() + def get_u_r(args, name_repo): + return "origin" + monkeypatch.setattr(pmb.helpers.git, "get_upstream_remote", get_u_r) + + # --- TEST RETURN VALUES --- + # Not on official branch + func = pmb.helpers.git.pull + name_repo = "test" + run_git(["checkout", "-b", "inofficial-branch"]) + assert func(args, name_repo) == -1 + + # Workdir is not clean + run_git(["checkout", "master"]) + shutil.copy(__file__, path_local + "/test.py") + assert func(args, name_repo) == -2 + os.unlink(path_local + "/test.py") + + # Tracking different remote + assert func(args, name_repo) == -3 + + # Let master track origin/master + run_git(["checkout", "-b", "temp"]) + run_git(["branch", "-D", "master"]) + run_git(["checkout", "-b", "master", "--track", "origin/master"]) + + # Already up to date + assert func(args, name_repo) == 2 + + # Can't fast-forward + run_git(["commit", "--allow-empty", "-m", "test"]) + assert func(args, name_repo) == -4 + + # Fast-forward successfully + run_git(["reset", "--hard", "origin/master"]) + run_git(["commit", "--allow-empty", "-m", "new"], path_remote) + assert func(args, name_repo) == 0