diff --git a/README.md b/README.md index 2bbb46cf..224a9000 100644 --- a/README.md +++ b/README.md @@ -255,11 +255,6 @@ $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hel $ pmbootstrap stats --arch=armhf ``` -`distccd` log: -``` -$ pmbootstrap log_distccd -``` - ### Use alternative sudo pmbootstrap supports `doas` and `sudo`. diff --git a/pmb/build/_package.py b/pmb/build/_package.py index e800ca33..884c931f 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -8,7 +8,6 @@ import pmb.build import pmb.build.autodetect import pmb.chroot import pmb.chroot.apk -import pmb.chroot.distccd import pmb.helpers.pmaports import pmb.helpers.repo import pmb.parse @@ -190,7 +189,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None, just initialized the build environment for nothing) and then setup the whole build environment (abuild, gcc, dependencies, cross-compiler). - :param cross: None, "native", "distcc", or "crossdirect" + :param cross: None, "native", or "crossdirect" :param skip_init_buildenv: can be set to False to avoid initializing the build environment. Use this when building something during initialization of the build @@ -223,27 +222,12 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None, # Cross-compiler init if cross: pmb.build.init_compiler(args, depends, cross, arch) - if cross == "distcc": - pmb.chroot.distccd.start(args, arch) if cross == "crossdirect": pmb.chroot.mount_native_into_foreign(args, suffix) return True -def get_gcc_version(args, arch): - """ - Get the GCC version for a specific arch from parsing the right APKINDEX. - We feed this to ccache, so it knows the right GCC version, when - cross-compiling in a foreign arch chroot with distcc. See the "using - ccache with other compiler wrappers" section of their man page: - - :returns: a string like "6.4.0-r5" - """ - return pmb.parse.apkindex.package(args, "gcc-" + arch, - pmb.config.arch_native)["version"] - - def get_pkgver(original_pkgver, original_source=False, now=None): """ Get the original pkgver when using the original source. Otherwise, get the @@ -381,7 +365,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, depending on the cross-compiler method and target architecture), copy the aport to the chroot and execute abuild. - :param cross: None, "native", "distcc", or "crossdirect" + :param cross: None, "native", or "crossdirect" :param src: override source used to build the package with a local folder :returns: (output, cmd, env), output is the destination apk path relative to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are @@ -410,18 +394,6 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, hostspec = pmb.parse.arch.alpine_to_hostspec(arch) env["CROSS_COMPILE"] = hostspec + "-" env["CC"] = hostspec + "-gcc" - if cross == "distcc": - env["CCACHE_PREFIX"] = "distcc" - env["CCACHE_PATH"] = f"/usr/lib/arch-bin-masquerade/{arch}:/usr/bin" - env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch) - env["DISTCC_HOSTS"] = "@127.0.0.1:/home/pmos/.distcc-sshd/distccd" - env["DISTCC_SSH"] = ("ssh -o StrictHostKeyChecking=no -p" + - args.port_distccd) - env["DISTCC_BACKOFF_PERIOD"] = "0" - if not args.distcc_fallback: - env["DISTCC_FALLBACK"] = "0" - if args.verbose: - env["DISTCC_VERBOSE"] = "1" if cross == "crossdirect": env["PATH"] = ":".join(["/native/usr/lib/crossdirect/" + arch, pmb.config.chroot_path]) diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py index f6edc063..953f0062 100644 --- a/pmb/build/autodetect.py +++ b/pmb/build/autodetect.py @@ -82,7 +82,7 @@ def suffix(apkbuild, arch): def crosscompile(args, apkbuild, arch, suffix): """ - :returns: None, "native", "crossdirect" or "distcc" + :returns: None, "native", "crossdirect" """ if not args.cross: return None @@ -91,5 +91,5 @@ def crosscompile(args, apkbuild, arch, suffix): if suffix == "native": return "native" if "!pmb:crossdirect" in apkbuild["options"]: - return "distcc" + return None return "crossdirect" diff --git a/pmb/chroot/distccd.py b/pmb/chroot/distccd.py deleted file mode 100644 index be27dad0..00000000 --- a/pmb/chroot/distccd.py +++ /dev/null @@ -1,250 +0,0 @@ -# Copyright 2023 Oliver Smith -# SPDX-License-Identifier: GPL-3.0-or-later -import errno -import json -import logging -import os -import pmb.chroot -import pmb.config -import pmb.chroot.apk - -""" Packages for foreign architectures (e.g. armhf) get built in chroots - running with QEMU. While this works, it is painfully slow. So we speed it - up by using distcc to let cross compilers running in the native chroots do - the heavy lifting. - - This file sets up an SSH server in the native chroot, which will then be - used by the foreign arch chroot to communicate with the distcc daemon. We - make sure that only the foreign arch chroot can connect to the sshd by only - listening on localhost, as well as generating dedicated ssh keys. - - Using the SSH server instead of running distccd directly is a security - measure. Distccd does not authenticate its clients and would therefore - allow any process of the host system (not related to pmbootstrap) to - execute compilers in the native chroot. By modifying the compiler's options - or sending malicious data to the compiler, it is likely that the process - can gain remote code execution [1]. That way, a compromised, but sandboxed - process could gain privilege escalation. - - [1]: -""" - - -def init_server(args): - """ - Install dependencies and generate keys for the server. - """ - # Install dependencies - pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc", - "openssh-server"]) - - # Config folder (nothing to do if existing) - dir = "/home/pmos/.distcc-sshd" - dir_outside = args.work + "/chroot_native" + dir - if os.path.exists(dir_outside): - return - - # Generate keys - logging.info("(native) generate distcc-sshd server keys") - pmb.chroot.user(args, ["mkdir", "-p", dir + "/etc/ssh"]) - pmb.chroot.user(args, ["ssh-keygen", "-A", "-f", dir]) - - -def init_client(args, suffix): - """ - Install dependencies and generate keys for the client. - """ - # Install dependencies - pmb.chroot.apk.install(args, ["arch-bin-masquerade", "distcc", - "openssh-client"], suffix) - - # Public key path (nothing to do if existing) - pub = "/home/pmos/id_ed25519.pub" - pub_outside = args.work + "/chroot_" + suffix + pub - if os.path.exists(pub_outside): - return - - # Generate keys - logging.info("(" + suffix + ") generate distcc-sshd client keys") - pmb.chroot.user(args, ["ssh-keygen", "-t", "ed25519", "-N", "", - "-f", "/home/pmos/.ssh/id_ed25519"], suffix) - pmb.chroot.user(args, ["cp", "/home/pmos/.ssh/id_ed25519.pub", pub], - suffix) - - -def configure_authorized_keys(args, suffix): - """ - Exclusively allow one foreign arch chroot to access the sshd. - """ - auth = "/home/pmos/.distcc-sshd/authorized_keys" - auth_outside = args.work + "/chroot_native/" + auth - pub = "/home/pmos/id_ed25519.pub" - pub_outside = args.work + "/chroot_" + suffix + pub - pmb.helpers.run.root(args, ["cp", pub_outside, auth_outside]) - - -def configure_cmdlist(args, arch): - """ - Create a whitelist of all the cross compiler wrappers. - - Distcc 3.3 and above requires such a whitelist, or else it will only run - with the --make-me-a-botnet parameter (even in ssh mode). - """ - dir = "/home/pmos/.distcc-sshd" - with open(args.work + "/chroot_native/tmp/cmdlist", "w") as handle: - for cmd in ["c++", "cc", "cpp", "g++", "gcc"]: - cmd_full = "/usr/lib/arch-bin-masquerade/" + arch + "/" + cmd - handle.write(cmd_full + "\n") - pmb.chroot.root(args, ["mv", "/tmp/cmdlist", dir + "/cmdlist"]) - pmb.chroot.user(args, ["cat", dir + "/cmdlist"]) - - -def configure_distccd_wrapper(args): - """ - Wrap distccd in a shell script, so we can pass the compiler whitelist and - set the verbose flag (when pmbootstrap is running with --verbose). - """ - dir = "/home/pmos/.distcc-sshd" - with open(args.work + "/chroot_native/tmp/wrapper", "w") as handle: - handle.write("#!/bin/sh\n" - "export DISTCC_CMDLIST='" + dir + "/cmdlist'\n" - "distccd --log-file /home/pmos/distccd.log --nice 19") - if args.verbose: - handle.write(" --verbose") - handle.write(" \"$@\"\n") - pmb.chroot.root(args, ["mv", "/tmp/wrapper", dir + "/distccd"]) - pmb.chroot.user(args, ["cat", dir + "/distccd"]) - pmb.chroot.root(args, ["chmod", "+x", dir + "/distccd"]) - - -def configure_sshd(args): - """ - Configure the SSH daemon in the native chroot. - """ - dir = "/home/pmos/.distcc-sshd" - config = """AllowAgentForwarding no - AllowTcpForwarding no - AuthorizedKeysFile /home/pmos/.distcc-sshd/authorized_keys - HostKey /home/pmos/.distcc-sshd/etc/ssh/ssh_host_ed25519_key - ListenAddress 127.0.0.1 - PasswordAuthentication no - PidFile /home/pmos/.distcc-sshd/sshd.pid - Port """ + args.port_distccd + """ - X11Forwarding no""" - - with open(args.work + "/chroot_native/tmp/cfg", "w") as handle: - for line in config.split("\n"): - handle.write(line.lstrip() + "\n") - pmb.chroot.root(args, ["mv", "/tmp/cfg", dir + "/sshd_config"]) - pmb.chroot.user(args, ["cat", dir + "/sshd_config"]) - - -def get_running_pid(args): - """ - :returns: the running distcc-sshd's pid as integer or None - """ - # PID file must exist - pidfile = "/home/pmos/.distcc-sshd/sshd.pid" - pidfile_outside = args.work + "/chroot_native" + pidfile - if not os.path.exists(pidfile_outside): - return None - - # Verify, if it still exists by sending a kill signal - with open(pidfile_outside, "r") as handle: - pid = int(handle.read()[:-1]) - try: - os.kill(pid, 0) - except OSError as err: - if err.errno == errno.ESRCH: # no such process - pmb.helpers.run.root(args, ["rm", pidfile_outside]) - return None - return pid - - -def get_running_parameters(args): - """ - Get the parameters of the currently running distcc-sshd instance. - - :returns: a dictionary in the form of - {"arch": "armhf", "port": 1234, "verbose": False} - If the information can not be read, "arch" is set to "unknown" - """ - # Return defaults - path = args.work + "/chroot_native/tmp/distcc_sshd_parameters" - if not os.path.exists(path): - return {"arch": "unknown", "port": 0, "verbose": False} - - # Parse the file as JSON - with open(path, "r") as handle: - return json.loads(handle.read()) - - -def set_running_parameters(args, arch): - """ - Set the parameters of the currently running distcc-sshd instance. - """ - parameters = {"arch": arch, - "port": args.port_distccd, - "verbose": args.verbose} - - path = args.work + "/chroot_native/tmp/distcc_sshd_parameters" - with open(path, "w") as handle: - json.dump(parameters, handle) - - -def is_running_with_same_parameters(args, arch): - """ - Check whether we can use the already running distcc-sshd instance with our - current set of parameters. In case we can use it directly, we save some - time, otherwise we need to stop it, configure it again, and start it once - more. - """ - if not get_running_pid(args): - return False - - parameters = get_running_parameters(args) - return (parameters["arch"] == arch and - parameters["port"] == args.port_distccd and - parameters["verbose"] == args.verbose) - - -def stop(args): - """ - Kill the sshd process (by using its pid). - """ - pid = get_running_pid(args) - if not pid: - return - - parameters = get_running_parameters(args) - logging.info("(native) stop distcc-sshd (" + parameters["arch"] + ")") - pmb.chroot.user(args, ["kill", str(pid)]) - - -def start(args, arch): - """ - Set up a new distcc-sshd instance or use an already running one. - """ - if is_running_with_same_parameters(args, arch): - return - stop(args) - - # Initialize server and client - suffix = "buildroot_" + arch - init_server(args) - init_client(args, suffix) - - logging.info("(native) start distcc-sshd (" + arch + ") on 127.0.0.1:" + - args.port_distccd) - - # Configure server parameters (arch, port, verbose) - configure_authorized_keys(args, suffix) - configure_distccd_wrapper(args) - configure_cmdlist(args, arch) - configure_sshd(args) - - # Run - dir = "/home/pmos/.distcc-sshd" - pmb.chroot.user(args, ["/usr/sbin/sshd", "-f", dir + "/sshd_config", - "-E", dir + "/log.txt"]) - set_running_parameters(args, arch) diff --git a/pmb/chroot/shutdown.py b/pmb/chroot/shutdown.py index 633560fe..23d115ad 100644 --- a/pmb/chroot/shutdown.py +++ b/pmb/chroot/shutdown.py @@ -7,7 +7,6 @@ import socket from contextlib import closing import pmb.chroot -import pmb.chroot.distccd import pmb.helpers.mount import pmb.install.losetup import pmb.parse.arch @@ -49,8 +48,6 @@ def shutdown_cryptsetup_device(args, name): def shutdown(args, only_install_related=False): - pmb.chroot.distccd.stop(args) - # Stop adb server kill_adb(args) diff --git a/pmb/chroot/zap.py b/pmb/chroot/zap.py index c91ccfa8..caf6f9f3 100644 --- a/pmb/chroot/zap.py +++ b/pmb/chroot/zap.py @@ -17,7 +17,7 @@ def zap(args, confirm=True, dry=False, pkgs_local=False, http=False, pkgs_local_mismatch=False, pkgs_online_mismatch=False, distfiles=False, rust=False, netboot=False): """ - Shutdown everything inside the chroots (e.g. distccd, adb), umount + Shutdown everything inside the chroots (e.g. adb), umount everything and then safely remove folders from the work-directory. :param dry: Only show what would be deleted, do not delete for real diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 227ab411..6aa52788 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -130,7 +130,6 @@ defaults = { "mirrors_postmarketos": "http://mirror.postmarketos.org/postmarketos/", "nonfree_firmware": True, "nonfree_userland": False, - "port_distccd": "33632", "ssh_keys": False, "ssh_key_glob": "~/.ssh/id_*.pub", "timezone": "GMT", diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index d8d80bff..903476f7 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -552,14 +552,6 @@ def log(args): pmb.helpers.run.user(args, cmd, output="tui") -def log_distccd(args): - logpath = "/home/pmos/distccd.log" - if args.clear_log: - pmb.chroot.user(args, ["truncate", "-s", "0", logpath]) - pmb.chroot.user(args, ["tail", "-n", args.lines, "-f", logpath], - output="tui") - - def zap(args): pmb.chroot.zap(args, dry=args.dry, http=args.http, distfiles=args.distfiles, pkgs_local=args.pkgs_local, diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 9454483c..40d870a8 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -611,7 +611,6 @@ def arguments(): parser.add_argument("--config-channels", help="path to channels.cfg (which is by default" " read from pmaports.git, origin/master branch)") - parser.add_argument("-d", "--port-distccd", dest="port_distccd") parser.add_argument("-mp", "--mirror-pmOS", dest="mirrors_postmarketos", help="postmarketOS mirror, disable with: -mp=''," " specify multiple with: -mp='one' -mp='two'," @@ -652,10 +651,6 @@ def arguments(): # Compiler parser.add_argument("--no-ccache", action="store_false", dest="ccache", help="do not cache the compiled output") - parser.add_argument("--distcc-nofallback", action="store_false", - help="when using the cross compiler via distcc fails," - "do not fall back to compiling slowly with QEMU", - dest="distcc_fallback") parser.add_argument("--no-cross", action="store_false", dest="cross", help="disable cross compiler, build only with QEMU and" " gcc (slow!)") @@ -699,14 +694,10 @@ def arguments(): # Action: log log = sub.add_parser("log", help="follow the pmbootstrap logfile") - log_distccd = sub.add_parser( - "log_distccd", - help="follow the distccd logfile") - for action in [log, log_distccd]: - action.add_argument("-n", "--lines", default="60", - help="count of initial output lines") - action.add_argument("-c", "--clear", help="clear the log", - action="store_true", dest="clear_log") + log.add_argument("-n", "--lines", default="60", + help="count of initial output lines") + log.add_argument("-c", "--clear", help="clear the log", + action="store_true", dest="clear_log") # Action: zap zap = sub.add_parser("zap", help="safely delete chroot folders") diff --git a/test/test_build_package.py b/test/test_build_package.py index 0d18b793..39327d09 100644 --- a/test/test_build_package.py +++ b/test/test_build_package.py @@ -233,7 +233,6 @@ def test_init_buildenv(args, monkeypatch): monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends", return_true) monkeypatch.setattr(pmb.chroot.apk, "install", return_none) - monkeypatch.setattr(pmb.chroot.distccd, "start", return_none) # Shortcut and fake apkbuild func = pmb.build._package.init_buildenv @@ -243,7 +242,6 @@ def test_init_buildenv(args, monkeypatch): # Build is necessary (various code paths) assert func(args, apkbuild, "armhf", strict=True) is True assert func(args, apkbuild, "armhf", cross="native") is True - assert func(args, apkbuild, "armhf", cross="distcc") is True # Build is not necessary (only builds dependencies) monkeypatch.setattr(pmb.build._package, "is_necessary_warn_depends", @@ -292,17 +290,6 @@ def test_run_abuild(args, monkeypatch): cmd = ["abuild", "-D", "postmarketOS", "-d"] assert func(args, apkbuild, "armhf", cross="native") == (output, cmd, env) - # cross=distcc - (output, cmd, env) = func(args, apkbuild, "armhf", cross="distcc") - assert output == "armhf/test-1-r2.apk" - assert env["CARCH"] == "armhf" - assert env["GOCACHE"] == "/home/pmos/.cache/go-build" - assert env["CCACHE_PREFIX"] == "distcc" - assert env["CCACHE_PATH"] == "/usr/lib/arch-bin-masquerade/armhf:/usr/bin" - assert env["CCACHE_COMPILERCHECK"].startswith("string:") - assert env["DISTCC_HOSTS"] == "@127.0.0.1:/home/pmos/.distcc-sshd/distccd" - assert env["DISTCC_BACKOFF_PERIOD"] == "0" - def test_finish(args, monkeypatch): # Real output path diff --git a/test/test_cross_compile_distcc.py b/test/test_cross_compile_distcc.py deleted file mode 100644 index 737cbe28..00000000 --- a/test/test_cross_compile_distcc.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Oliver Smith -# SPDX-License-Identifier: GPL-3.0-or-later -import os -import pytest -import sys - -import pmb_test # noqa -import pmb.build -import pmb.chroot.distccd -import pmb.helpers.logging - - -@pytest.fixture -def args(tmpdir, request): - import pmb.parse - sys.argv = ["pmbootstrap", "init"] - args = pmb.parse.arguments() - args.log = args.work + "/log_testsuite.txt" - pmb.helpers.logging.init(args) - request.addfinalizer(pmb.helpers.logging.logfd.close) - return args - - -def test_cross_compile_distcc(args): - # Delete old distccd log - pmb.chroot.distccd.stop(args) - distccd_log = args.work + "/chroot_native/home/pmos/distccd.log" - if os.path.exists(distccd_log): - pmb.helpers.run.root(args, ["rm", distccd_log]) - - # Force usage of distcc (no fallback, no ccache) - args.verbose = True - args.ccache = False - args.distcc_fallback = False - - # Compile, print distccd and sshd logs on error - try: - pmb.build.package(args, "hello-world", arch="armhf", force=True) - except RuntimeError: - print("distccd log:") - pmb.helpers.run.user(args, ["cat", distccd_log], output="stdout", - check=False) - print("sshd log:") - sshd_log = args.work + "/chroot_native/home/pmos/.distcc-sshd/log.txt" - pmb.helpers.run.root(args, ["cat", sshd_log], output="stdout", - check=False) - raise