aportgen: add feature to fork upstream packages (!1811)

This MR add the --fork-alpine argument to the pmboostrap aportgen
command, which downloads the APKBUILD and related files and copies them
into the pmaports/temp folder.

pmb/aportgen/__init__.py

@@ -52,17 +52,26 @@ def properties(pkgname):
 
 
 def generate(args, pkgname):
-    # Confirm overwrite
+    if args.fork_alpine:
+        prefix, folder, options = (pkgname, "temp", {"confirm_overwrite": True})
+    else:
         prefix, folder, options = properties(pkgname)
     path_target = args.aports + "/" + folder + "/" + pkgname
+
+    # Confirm overwrite
     if options["confirm_overwrite"] and os.path.exists(path_target):
         logging.warning("WARNING: Target folder already exists: " + path_target)
         if not pmb.helpers.cli.confirm(args, "Continue and overwrite?"):
             raise RuntimeError("Aborted.")
 
-    # Run pmb.aportgen.PREFIX.generate()
     if os.path.exists(args.work + "/aportgen"):
         pmb.helpers.run.user(args, ["rm", "-r", args.work + "/aportgen"])
+    if args.fork_alpine:
+        upstream = pmb.aportgen.core.get_upstream_aport(args, pkgname)
+        pmb.helpers.run.user(args, ["cp", "-r", upstream, args.work + "/aportgen"])
+        pmb.aportgen.core.rewrite(args, pkgname, replace_simple={"# Contributor:*": None, "# Maintainer:*": None})
+    else:
+        # Run pmb.aportgen.PREFIX.generate()
         getattr(pmb.aportgen, prefix.replace("-", "_")).generate(args, pkgname)
 
     # Move to the aports folder

pmb/aportgen/core.py

@@ -62,7 +62,7 @@ def format_function(name, body, remove_indent=4):
     return name + "() {\n" + ret + "}\n"
 
 
-def rewrite(args, pkgname, path_original, fields={}, replace_pkgname=None,
+def rewrite(args, pkgname, path_original="", fields={}, replace_pkgname=None,
             replace_functions={}, replace_simple={}, below_header="",
             remove_indent=4):
     """
@@ -70,6 +70,7 @@ def rewrite(args, pkgname, path_original, fields={}, replace_pkgname=None,
     lines (so they won't be bugged with issues regarding our generated aports),
     and add reference to the original aport.
 
+    :param path_original: The original path of the automatically generated aport
     :param fields: key-value pairs of fields, that shall be changed in the
         APKBUILD. For example: {"pkgdesc": "my new package", "subpkgs": ""}
     :param replace_pkgname: When set, $pkgname gets replaced with that string in
@@ -85,12 +86,18 @@ def rewrite(args, pkgname, path_original, fields={}, replace_pkgname=None,
 
     """
     # Header
+    if path_original:
         lines_new = [
             "# Automatically generated aport, do not edit!\n",
             "# Generator: pmbootstrap aportgen " + pkgname + "\n",
             "# Based on: " + path_original + "\n",
             "\n",
         ]
+    else:
+        lines_new = [
+            "# Forked from Alpine INSERT-REASON-HERE (CHANGEME!)\n",
+            "\n",
+        ]
 
     if below_header:
         for line in below_header.split("\n"):

pmb/parse/arguments.py

@@ -487,6 +487,8 @@ def arguments():
                           " APKBUILD match, instead of updating them")
     aportgen = sub.add_parser("aportgen", help="generate a postmarketOS"
                               " specific package build recipe (aport/APKBUILD)")
+    aportgen.add_argument("--fork-alpine", help="fork the alpine upstream package",
+                          action="store_true", dest="fork_alpine")
     build = sub.add_parser("build", help="create a package for a"
                            " specific architecture")
     build.add_argument("--arch", choices=arch_choices, default=None,

test/test_aportgen.py

@@ -17,7 +17,6 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import os
-import shutil
 import sys
 import pytest
 import filecmp
@@ -38,17 +37,17 @@ def args(tmpdir, request):
     sys.argv = ["pmbootstrap.py", "chroot"]
     args = pmb.parse.arguments()
     args.log = args.work + "/log_testsuite.txt"
+    args.fork_alpine = False
     pmb.helpers.logging.init(args)
     request.addfinalizer(args.logfd.close)
     return args
 
 
 def test_aportgen_compare_output(args, tmpdir, monkeypatch):
-    # Copy pmaports testdata to tmpdir
+    # Fake aports folder in tmpdir
-    tmpdir = str(tmpdir)
+    args.aports = str(tmpdir)
+    os.mkdir(tmpdir + "/cross")
     testdata = pmb_src + "/test/testdata/aportgen"
-    shutil.copytree(testdata + "/pmaports/cross", tmpdir + "/cross")
-    args.aports = tmpdir
 
     # Override get_upstream_aport() to point to testdata
     def func(args, upstream_path):
@@ -65,6 +64,27 @@ def test_aportgen_compare_output(args, tmpdir, monkeypatch):
         assert filecmp.cmp(path_new, path_old, False)
 
 
+def test_aportgen_fork_alpine_compare_output(args, tmpdir, monkeypatch):
+    # Fake aports folder in tmpdir
+    args.aports = str(tmpdir)
+    os.mkdir(tmpdir + "/temp")
+    testdata = pmb_src + "/test/testdata/aportgen"
+    args.fork_alpine = True
+
+    # Override get_upstream_aport() to point to testdata
+    def func(args, upstream_path):
+        return testdata + "/aports/main/" + upstream_path
+    monkeypatch.setattr(pmb.aportgen.core, "get_upstream_aport", func)
+
+    # Run aportgen and compare output
+    pkgname = "binutils"
+    pmb.aportgen.generate(args, pkgname)
+    path_new = args.aports + "/temp/" + pkgname + "/APKBUILD"
+    path_old = testdata + "/pmaports/temp/" + pkgname + "/APKBUILD"
+    assert os.path.exists(path_new)
+    assert filecmp.cmp(path_new, path_old, False)
+
+
 def test_aportgen(args, tmpdir):
     # Fake aports folder in tmpdir
     args.aports = str(tmpdir)

test/test_aportgen_device_wizard.py

@@ -35,6 +35,7 @@ def args(tmpdir, request):
     sys.argv = ["pmbootstrap.py", "build", "-i", "device-testsuite-testdevice"]
     args = pmb.parse.arguments()
     args.log = args.work + "/log_testsuite.txt"
+    args.fork_alpine = False
     pmb.helpers.logging.init(args)
     request.addfinalizer(args.logfd.close)
 

test/testdata/aportgen/pmaports/temp/binutils/APKBUILD

@@ -0,0 +1,118 @@
+# Forked from Alpine INSERT-REASON-HERE (CHANGEME!)
+
+pkgname=binutils
+pkgver=2.31.1
+pkgrel=1
+pkgdesc="Tools necessary to build programs"
+url="https://www.gnu.org/software/binutils/"
+depends=""
+makedepends_build="bison flex texinfo"
+makedepends_host="zlib-dev"
+makedepends="$makedepends_build $makedepends_host"
+arch="all"
+license="GPL-2.0 GPL-3.0-or-later LGPL-2.0 BSD"
+subpackages="$pkgname-dev $pkgname-doc $pkgname-gold"
+source="https://ftp.gnu.org/gnu/$pkgname/$pkgname-$pkgver.tar.bz2
+	x86-Add-a-GNU_PROPERTY_X86_ISA_1_USED-note-if-needed.patch
+	x86-Properly-merge-GNU_PROPERTY_X86_ISA_1_USED.patch
+	x86-Properly-add-X86_ISA_1_NEEDED-property.patch
+	binutils-ld-fix-static-linking.patch
+	gold-mips.patch
+	"
+builddir="$srcdir/$pkgname-$pkgver"
+
+if [ "$CHOST" != "$CTARGET" ]; then
+	pkgname="$pkgname-$CTARGET_ARCH"
+	subpackages=""
+	sonameprefix="$pkgname:"
+fi
+
+# secfixes:
+#   2.28-r1:
+#   - CVE-2017-7614
+
+build() {
+	local _sysroot=/
+	local _cross_configure="--enable-install-libiberty --enable-shared"
+	local _arch_configure=""
+
+	if [ "$CHOST" != "$CTARGET" ]; then
+		_sysroot="$CBUILDROOT"
+		_cross_configure="--disable-install-libiberty"
+	fi
+
+	if [ "$CTARGET_ARCH" = "x86_64" ]; then
+		_arch_configure="--enable-targets=x86_64-pep"
+	fi
+
+	case "$CTARGET_ARCH" in
+	mips*)  _hash_style_configure="--enable-default-hash-style=sysv" ;;
+	*)      _hash_style_configure="--enable-default-hash-style=gnu"	;;
+	esac
+
+	cd "$builddir"
+	./configure \
+		--build=$CBUILD \
+		--host=$CHOST \
+		--target=$CTARGET \
+		--with-build-sysroot="$CBUILDROOT" \
+		--with-sysroot=$_sysroot \
+		--prefix=/usr \
+		--mandir=/usr/share/man \
+		--infodir=/usr/share/info \
+		--disable-multilib \
+		--enable-ld=default \
+		--enable-gold=yes \
+		--enable-64-bit-bfd \
+		--enable-plugins \
+		--enable-relro \
+		--enable-deterministic-archives \
+		$_cross_configure \
+		$_arch_configure \
+		$_hash_style_configure \
+		--with-pic \
+		--disable-werror \
+		--disable-nls \
+		--with-system-zlib
+	make
+}
+
+package() {
+	cd "$builddir"
+	make install DESTDIR="$pkgdir"
+	if [ -d "$pkgdir"/usr/lib64 ]; then
+		mv "$pkgdir"/usr/lib64/* "$pkgdir"/usr/lib/
+		rmdir "$pkgdir"/usr/lib64
+	fi
+	if [ "$CHOST" != "$CTARGET" ]; then
+		# creating cross tools: remove any files that would conflict
+		# with the native tools, or other cross tools
+		rm -r "$pkgdir"/usr/share
+		rm -f "$pkgdir"/usr/lib/libiberty.a
+	fi
+}
+
+libs() {
+	pkgdesc="Runtime libraries from binutils - libbfd and libopcodes"
+
+	mkdir -p "$subpkgdir"/usr/lib
+	mv "$pkgdir"/usr/lib/lib*.so "$subpkgdir"/usr/lib/
+}
+
+gold() {
+	pkgdesc="GNU binutils - gold linker"
+
+	if [ -e "$pkgdir"/usr/bin/ld.gold ]; then
+		mkdir -p "$subpkgdir"/usr/bin
+		mv "$pkgdir"/usr/bin/ld.gold "$subpkgdir"/usr/bin
+	fi
+	mkdir -p "$subpkgdir"/usr/$CTARGET/bin
+	mv "$pkgdir"/usr/$CTARGET/bin/ld.gold "$subpkgdir"/usr/$CTARGET/bin/ld.gold
+}
+
+sha512sums="b42954e6f49a0adcd2676bdd77dfb59bfc25cec8184b007521d1e2b1d5d0593b58639e3d9448d5a40fe024c3cea386a37743627d6bb16d502f52a4a32b9573bd  binutils-2.31.1.tar.bz2
+d95fd77e1c2c4670a2a11979d6811b358ba0f067b917b33d241eca20cfe66553f6a6fccc5ec26d8d29045e487cb74389fbf86426f80cf81df95608835a566cfc  x86-Add-a-GNU_PROPERTY_X86_ISA_1_USED-note-if-needed.patch
+ce5d3c935057d624c1ce75722e7b5e4583812d46797edce8c381a94b2643f44f7bd0165e7e9b8e358955f4d979074ee487598efbf24389a4013681f99ff7c595  x86-Properly-merge-GNU_PROPERTY_X86_ISA_1_USED.patch
+1878bed194529afb2430af21ba9712d6819c5fdfdc5c8db652175bf86d0a0c710ac2bd3ec728bf874887301bd187d91bf60a374c47850c7bd2eafbf6653d74d8  x86-Properly-add-X86_ISA_1_NEEDED-property.patch
+ecee33b0e435aa704af1c334e560f201638ff79e199aa11ed78a72f7c9b46f85fbb227af5748e735fd681d1965fcc42ac81b0c8824e540430ce0c706c81e8b49  binutils-ld-fix-static-linking.patch
+f55cf2e0bf82f97583a1abe10710e4013ecf7d64f1da2ef8659a44a06d0dd8beaf58dab98a183488ea137f03e32d62efc878d95f018f836f8cec870bc448556f  gold-mips.patch"