Add "crossdirect": faster cross compiling (!1758)

Launch native cross compilers inside foreign chroot. Enable by default,
but allow disabling with --no-crossdirect for now. This option and the
distcc-sshd related code will be removed in the future.

pmb/build/_package.py

@@ -193,7 +193,7 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
     just initialized the build environment for nothing) and then setup the
     whole build environment (abuild, gcc, dependencies, cross-compiler).
 
-    :param cross: None, "native" or "distcc"
+    :param cross: None, "native", "distcc", or "crossdirect"
     :param skip_init_buildenv: can be set to False to avoid initializing the
                                build environment. Use this when building
                                something during initialization of the build
@@ -232,9 +232,15 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None,
             cross_pkgs += ["gcc6-" + arch]
         else:
             cross_pkgs += ["gcc-" + arch, "g++-" + arch]
+        if "clang" in depends or "clang-dev" in depends:
+            cross_pkgs += ["clang"]
+        if cross == "crossdirect":
+            cross_pkgs += ["crossdirect"]
         pmb.chroot.apk.install(args, cross_pkgs)
     if cross == "distcc":
         pmb.chroot.distccd.start(args, arch)
+    if cross == "crossdirect":
+        pmb.chroot.mount_native_into_foreign(args, suffix)
 
     return True
 
@@ -350,7 +356,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
     depending on the cross-compiler method and target architecture), copy
     the aport to the chroot and execute abuild.
 
-    :param cross: None, "native" or "distcc"
+    :param cross: None, "native", "distcc", or "crossdirect"
     :param src: override source used to build the package with a local folder
     :returns: (output, cmd, env), output is the destination apk path relative
               to the package folder ("x86_64/hello-1-r2.apk"). cmd and env are
@@ -391,6 +397,9 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None,
             env["DISTCC_FALLBACK"] = "0"
         if args.verbose:
             env["DISTCC_VERBOSE"] = "1"
+    if cross == "crossdirect":
+        env["CCACHE_PATH"] = "/native/usr/lib/crossdirect/" + arch + ":/usr/bin"
+        env["CCACHE_COMPILERCHECK"] = "string:" + get_gcc_version(args, arch)
     if not args.ccache:
         env["CCACHE_DISABLE"] = "1"
 

pmb/build/autodetect.py

@@ -92,7 +92,7 @@ def suffix(args, apkbuild, arch):
 
 def crosscompile(args, apkbuild, arch, suffix):
     """
-        :returns: None, "native" or "distcc"
+        :returns: None, "native", "crossdirect" or "distcc"
     """
     if not args.cross:
         return None
@@ -100,4 +100,6 @@ def crosscompile(args, apkbuild, arch, suffix):
         return None
     if suffix == "native":
         return "native"
-    return "distcc"
+    if args.no_crossdirect:
+        return "distcc"
+    return "crossdirect"

pmb/chroot/__init__.py

@@ -17,7 +17,7 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 from pmb.chroot.init import init
-from pmb.chroot.mount import mount
+from pmb.chroot.mount import mount, mount_native_into_foreign
 from pmb.chroot.root import root
 from pmb.chroot.user import user
 from pmb.chroot.user import exists as user_exists

pmb/chroot/mount.py

@@ -16,6 +16,7 @@ GNU General Public License for more details.
 You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
+import glob
 import logging
 import os
 import pmb.config
@@ -101,3 +102,15 @@ def mount(args, suffix="native"):
     for source, target in mountpoints.items():
         target_full = args.work + "/chroot_" + suffix + target
         pmb.helpers.mount.bind(args, source, target_full)
+
+
+def mount_native_into_foreign(args, suffix):
+    source = args.work + "/chroot_native"
+    target = args.work + "/chroot_" + suffix + "/native"
+    pmb.helpers.mount.bind(args, source, target)
+
+    musl = os.path.basename(glob.glob(source + "/lib/ld-musl-*.so.1")[0])
+    musl_link = args.work + "/chroot_" + suffix + "/lib/" + musl
+    if not os.path.lexists(musl_link):
+        pmb.helpers.run.root(args, ["ln", "-s", "/native/lib/" + musl,
+                                    musl_link])

pmb/config/__init__.py

@@ -40,7 +40,7 @@ apk_keys_path = pmb_src + "/pmb/data/keys"
 apk_tools_static_min_version = "2.10.3-r1"
 
 # postmarketOS aports compatibility (checked against "version" in pmaports.cfg)
-pmaports_min_version = "2"
+pmaports_min_version = "3"
 
 # Version of the work folder (as asked during 'pmbootstrap init'). Increase
 # this number, whenever migration is required and provide the migration code,

pmb/parse/arguments.py

@@ -324,6 +324,12 @@ def arguments():
     # Compiler
     parser.add_argument("--ccache-disable", action="store_false",
                         dest="ccache", help="do not cache the compiled output")
+    parser.add_argument("--no-crossdirect", action="store_true",
+                        help="Don't use the new, faster 'crossdirect' method,"
+                             " use the old 'distcc-sshd' method instead. Use"
+                             " if crossdirect broke something. This option"
+                             " and the legacy 'distcc-sshd' code will be"
+                             " removed soon if no problems turn up.")
     parser.add_argument("--distcc-nofallback", action="store_false",
                         help="when using the cross compiler via distcc fails,"
                              "do not fall back to compiling slowly with QEMU",