From 20a0ccf36f1fb248bd1b261caa432d6abfe5c26c Mon Sep 17 00:00:00 2001 From: Oliver Smith Date: Thu, 13 Jul 2023 20:19:47 +0200 Subject: [PATCH] pmb.helpers.run_core: move flat_cmd here Move pmb.helpers.run.flat_cmd to run_core, as it will be used by pmb.helpers.run_core.core in the next patch. Link: https://lists.sr.ht/~postmarketos/pmbootstrap-devel/%3C20230713182337.6185-2-ollieparanoid@postmarketos.org%3E --- pmb/chroot/root.py | 4 ++-- pmb/chroot/user.py | 3 ++- pmb/helpers/apk.py | 3 ++- pmb/helpers/run.py | 33 ++------------------------------- pmb/helpers/run_core.py | 32 +++++++++++++++++++++++++++++++- pmb/sideload/__init__.py | 7 ++++--- test/test_config_user.py | 11 ++++++----- test/test_shell_escape.py | 9 +++++---- 8 files changed, 54 insertions(+), 48 deletions(-) diff --git a/pmb/chroot/root.py b/pmb/chroot/root.py index d1e70013..4555638d 100644 --- a/pmb/chroot/root.py +++ b/pmb/chroot/root.py @@ -71,10 +71,10 @@ def root(args, cmd, suffix="native", working_dir="/", output="log", # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."] executables = executables_absolute_path() cmd_chroot = [executables["chroot"], chroot, "/bin/sh", "-c", - pmb.helpers.run.flat_cmd(cmd, working_dir)] + pmb.helpers.run_core.flat_cmd(cmd, working_dir)] cmd_sudo = pmb.config.sudo([ "env", "-i", executables["sh"], "-c", - pmb.helpers.run.flat_cmd(cmd_chroot, env=env_all)] + pmb.helpers.run_core.flat_cmd(cmd_chroot, env=env_all)] ) return pmb.helpers.run_core.core(args, msg, cmd_sudo, None, output, output_return, check, True, diff --git a/pmb/chroot/user.py b/pmb/chroot/user.py index 925f3124..9bad1e9b 100644 --- a/pmb/chroot/user.py +++ b/pmb/chroot/user.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import pmb.chroot.root import pmb.helpers.run +import pmb.helpers.run_core def user(args, cmd, suffix="native", working_dir="/", output="log", @@ -21,7 +22,7 @@ def user(args, cmd, suffix="native", working_dir="/", output="log", if "HOME" not in env: env["HOME"] = "/home/pmos" - flat_cmd = pmb.helpers.run.flat_cmd(cmd, env=env) + flat_cmd = pmb.helpers.run_core.flat_cmd(cmd, env=env) cmd = ["busybox", "su", "pmos", "-c", flat_cmd] return pmb.chroot.root(args, cmd, suffix, working_dir, output, output_return, check, {}, auto_init) diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py index 2833c1c9..774ca0b9 100644 --- a/pmb/helpers/apk.py +++ b/pmb/helpers/apk.py @@ -6,6 +6,7 @@ import pmb.chroot.root import pmb.config.pmaports import pmb.helpers.cli import pmb.helpers.run +import pmb.helpers.run_core import pmb.parse.version @@ -62,7 +63,7 @@ def _create_command_with_progress(command, fifo): """ flags = ["--no-progress", "--progress-fd", "3"] command_full = [command[0]] + flags + command[1:] - command_flat = pmb.helpers.run.flat_cmd(command_full) + command_flat = pmb.helpers.run_core.flat_cmd(command_full) command_flat = f"exec 3>{fifo}; {command_flat}" return ["sh", "-c", command_flat] diff --git a/pmb/helpers/run.py b/pmb/helpers/run.py index 9835434a..efded208 100644 --- a/pmb/helpers/run.py +++ b/pmb/helpers/run.py @@ -4,35 +4,6 @@ import shlex import pmb.helpers.run_core -def flat_cmd(cmd, working_dir=None, env={}): - """ - Convert a shell command passed as list into a flat shell string with - proper escaping. - - :param cmd: command as list, e.g. ["echo", "string with spaces"] - :param working_dir: when set, prepend "cd ...;" to execute the command - in the given working directory - :param env: dict of environment variables to be passed to the command, e.g. - {"JOBS": "5"} - :returns: the flat string, e.g. - echo 'string with spaces' - cd /home/pmos;echo 'string with spaces' - """ - # Merge env and cmd into escaped list - escaped = [] - for key, value in env.items(): - escaped.append(key + "=" + shlex.quote(value)) - for i in range(len(cmd)): - escaped.append(shlex.quote(cmd[i])) - - # Prepend working dir - ret = " ".join(escaped) - if working_dir: - ret = "cd " + shlex.quote(working_dir) + ";" + ret - - return ret - - def user(args, cmd, working_dir=None, output="log", output_return=False, check=None, env={}, sudo=False): """ @@ -54,7 +25,7 @@ def user(args, cmd, working_dir=None, output="log", output_return=False, # Add environment variables and run if env: - cmd = ["sh", "-c", flat_cmd(cmd, env=env)] + cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)] return pmb.helpers.run_core.core(args, msg, cmd, working_dir, output, output_return, check, sudo) @@ -71,7 +42,7 @@ def root(args, cmd, working_dir=None, output="log", output_return=False, arguments and the return value. """ if env: - cmd = ["sh", "-c", flat_cmd(cmd, env=env)] + cmd = ["sh", "-c", pmb.helpers.run_core.flat_cmd(cmd, env=env)] cmd = pmb.config.sudo(cmd) return user(args, cmd, working_dir, output, output_return, check, env, diff --git a/pmb/helpers/run_core.py b/pmb/helpers/run_core.py index c7412948..37d4db34 100644 --- a/pmb/helpers/run_core.py +++ b/pmb/helpers/run_core.py @@ -2,12 +2,13 @@ # SPDX-License-Identifier: GPL-3.0-or-later import fcntl import logging +import os import selectors +import shlex import subprocess import sys import threading import time -import os import pmb.helpers.run """ For a detailed description of all output modes, read the description of @@ -15,6 +16,35 @@ import pmb.helpers.run called by core(). """ +def flat_cmd(cmd, working_dir=None, env={}): + """ + Convert a shell command passed as list into a flat shell string with + proper escaping. + + :param cmd: command as list, e.g. ["echo", "string with spaces"] + :param working_dir: when set, prepend "cd ...;" to execute the command + in the given working directory + :param env: dict of environment variables to be passed to the command, e.g. + {"JOBS": "5"} + :returns: the flat string, e.g. + echo 'string with spaces' + cd /home/pmos;echo 'string with spaces' + """ + # Merge env and cmd into escaped list + escaped = [] + for key, value in env.items(): + escaped.append(key + "=" + shlex.quote(value)) + for i in range(len(cmd)): + escaped.append(shlex.quote(cmd[i])) + + # Prepend working dir + ret = " ".join(escaped) + if working_dir: + ret = "cd " + shlex.quote(working_dir) + ";" + ret + + return ret + + def sanity_checks(output="log", output_return=False, check=None): """ Raise an exception if the parameters passed to core() don't make sense diff --git a/pmb/sideload/__init__.py b/pmb/sideload/__init__.py index f46882d1..2d11c6df 100644 --- a/pmb/sideload/__init__.py +++ b/pmb/sideload/__init__.py @@ -5,6 +5,7 @@ import os import logging import pmb.helpers.run +import pmb.helpers.run_core import pmb.parse.apkindex import pmb.config.pmaports import pmb.build @@ -29,7 +30,7 @@ def scp_abuild_key(args, user, host, port): keyname = os.path.join("/tmp", os.path.basename(key)) remote_cmd = ['sudo', '-p', pmb.config.sideload_sudo_prompt, '-S', 'mv', '-n', keyname, "/etc/apk/keys/"] - remote_cmd = pmb.helpers.run.flat_cmd(remote_cmd) + remote_cmd = pmb.helpers.run_core.flat_cmd(remote_cmd) command = ['ssh', '-t', '-p', port, f'{user}@{host}', remote_cmd] pmb.helpers.run.user(args, command, output="tui") @@ -53,8 +54,8 @@ def ssh_install_apks(args, user, host, port, paths): logging.info(f"Installing packages at {user}@{host}") add_cmd = ['sudo', '-p', pmb.config.sideload_sudo_prompt, '-S', 'apk', '--wait', '30', 'add'] + remote_paths - add_cmd = pmb.helpers.run.flat_cmd(add_cmd) - clean_cmd = pmb.helpers.run.flat_cmd(['rm'] + remote_paths) + add_cmd = pmb.helpers.run_core.flat_cmd(add_cmd) + clean_cmd = pmb.helpers.run_core.flat_cmd(['rm'] + remote_paths) command = ['ssh', '-t', '-p', port, f'{user}@{host}', f'{add_cmd}; rc=$?; {clean_cmd}; exit $rc'] pmb.helpers.run.user(args, command, output="tui") diff --git a/test/test_config_user.py b/test/test_config_user.py index d11899ac..af4c349c 100644 --- a/test/test_config_user.py +++ b/test/test_config_user.py @@ -9,6 +9,7 @@ import pmb.config import pmb.helpers.frontend import pmb.helpers.logging import pmb.helpers.run +import pmb.helpers.run_core @pytest.fixture @@ -40,11 +41,11 @@ def test_config_user(args, tmpdir, monkeypatch): path_config = tmpdir + "/pmbootstrap.cfg" # Generate default config (only uses tmpdir) - cmd = pmb.helpers.run.flat_cmd(["./pmbootstrap.py", - "-c", path_config, - "-w", path_work, - "--aports", args.aports, - "init"]) + cmd = pmb.helpers.run_core.flat_cmd(["./pmbootstrap.py", + "-c", path_config, + "-w", path_work, + "--aports", args.aports, + "init"]) pmb.helpers.run.user(args, ["sh", "-c", "yes '' | " + cmd], pmb.config.pmb_src) diff --git a/test/test_shell_escape.py b/test/test_shell_escape.py index 22288d9b..d2c57ee9 100644 --- a/test/test_shell_escape.py +++ b/test/test_shell_escape.py @@ -7,6 +7,7 @@ import pmb_test # noqa import pmb.chroot.root import pmb.chroot.user import pmb.helpers.run +import pmb.helpers.run_core import pmb.helpers.logging @@ -79,7 +80,7 @@ def test_shell_escape_env(args): def test_flat_cmd_simple(): - func = pmb.helpers.run.flat_cmd + func = pmb.helpers.run_core.flat_cmd cmd = ["echo", "test"] working_dir = None ret = "echo test" @@ -88,7 +89,7 @@ def test_flat_cmd_simple(): def test_flat_cmd_wrap_shell_string_with_spaces(): - func = pmb.helpers.run.flat_cmd + func = pmb.helpers.run_core.flat_cmd cmd = ["echo", "string with spaces"] working_dir = None ret = "echo 'string with spaces'" @@ -97,7 +98,7 @@ def test_flat_cmd_wrap_shell_string_with_spaces(): def test_flat_cmd_wrap_env_simple(): - func = pmb.helpers.run.flat_cmd + func = pmb.helpers.run_core.flat_cmd cmd = ["echo", "test"] working_dir = None ret = "JOBS=5 echo test" @@ -106,7 +107,7 @@ def test_flat_cmd_wrap_env_simple(): def test_flat_cmd_wrap_env_spaces(): - func = pmb.helpers.run.flat_cmd + func = pmb.helpers.run_core.flat_cmd cmd = ["echo", "test"] working_dir = None ret = "JOBS=5 TEST='spaces string' echo test"