Add `pmbootstrap bootimg_analyze` / prompt during new device wizard (#905)

pmb/aportgen/device.py

@@ -17,9 +17,11 @@ You should have received a copy of the GNU General Public License
 along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
 """
 import logging
+import os
 import pmb.helpers.run
 import pmb.aportgen.core
 import pmb.parse.apkindex
+import pmb.parse.bootimg
 
 
 def ask_for_architecture(args):
@@ -79,8 +81,47 @@ def ask_for_flash_method(args):
                       " pmb/config/__init__.py.")
 
 
+def ask_for_bootimg(args):
+    logging.info("You can analyze a known working boot.img file to automatically fill"
+                 " out the flasher information for your deviceinfo file. Either specify"
+                 " the path to an image or press return to skip this step (you can do"
+                 " it later with 'pmbootstrap bootimg_analyze').")
+
+    while True:
+        path = os.path.expanduser(pmb.helpers.cli.ask(args, "Path", None, "", False))
+        if not len(path):
+            return None
+        try:
+            return pmb.parse.bootimg(args, path)
+        except Exception as e:
+            logging.fatal("ERROR: " + str(e) + ". Please try again.")
+
+
+def generate_deviceinfo_fastboot_content(args, bootimg=None):
+    if bootimg is None:
+        bootimg = {"cmdline": "",
+                   "qcdt": "false",
+                   "base": "",
+                   "kernel_offset": "",
+                   "ramdisk_offset": "",
+                   "second_offset": "",
+                   "tags_offset": "",
+                   "pagesize": "2048"}
+    return """\
+        deviceinfo_kernel_cmdline=\"""" + bootimg["cmdline"] + """\"
+        deviceinfo_generate_bootimg="true"
+        deviceinfo_bootimg_qcdt=\"""" + bootimg["qcdt"] + """\"
+        deviceinfo_flash_offset_base=\"""" + bootimg["base"] + """\"
+        deviceinfo_flash_offset_kernel=\"""" + bootimg["kernel_offset"] + """\"
+        deviceinfo_flash_offset_ramdisk=\"""" + bootimg["ramdisk_offset"] + """\"
+        deviceinfo_flash_offset_second=\"""" + bootimg["second_offset"] + """\"
+        deviceinfo_flash_offset_tags=\"""" + bootimg["tags_offset"] + """\"
+        deviceinfo_flash_pagesize=\"""" + bootimg["pagesize"] + """\"
+        """
+
+
 def generate_deviceinfo(args, pkgname, name, manufacturer, arch, has_keyboard,
-                        has_external_storage, flash_method):
+                        has_external_storage, flash_method, bootimg=None):
     content = """\
         # Reference: <https://postmarketos.org/deviceinfo>
         # Please use double quotes only. You can source this file in shell scripts.
@@ -106,18 +147,6 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, arch, has_keyboard,
         deviceinfo_flash_methods=\"""" + flash_method + """\"
         """
 
-    content_fastboot = """\
-        deviceinfo_kernel_cmdline=""
-        deviceinfo_generate_bootimg="true"
-        deviceinfo_bootimg_qcdt="false"
-        deviceinfo_flash_offset_base=""
-        deviceinfo_flash_offset_kernel=""
-        deviceinfo_flash_offset_ramdisk=""
-        deviceinfo_flash_offset_second=""
-        deviceinfo_flash_offset_tags=""
-        deviceinfo_flash_pagesize="2048"
-        """
-
     content_heimdall_bootimg = """\
         deviceinfo_flash_heimdall_partition_kernel=""
         deviceinfo_flash_heimdall_partition_system=""
@@ -134,9 +163,9 @@ def generate_deviceinfo(args, pkgname, name, manufacturer, arch, has_keyboard,
         """
 
     if flash_method == "fastboot":
-        content += content_fastboot
+        content += generate_deviceinfo_fastboot_content(args, bootimg)
     elif flash_method == "heimdall-bootimg":
-        content += content_fastboot
+        content += generate_deviceinfo_fastboot_content(args, bootimg)
         content += content_heimdall_bootimg
     elif flash_method == "heimdall-isorec":
         content += content_heimdall_isorec
@@ -190,7 +219,10 @@ def generate(args, pkgname):
     has_keyboard = ask_for_keyboard(args)
     has_external_storage = ask_for_external_storage(args)
     flash_method = ask_for_flash_method(args)
+    bootimg = None
+    if flash_method in ["fastboot", "heimdall-bootimg"]:
+        bootimg = ask_for_bootimg(args)
 
     generate_deviceinfo(args, pkgname, name, manufacturer, arch, has_keyboard,
-                        has_external_storage, flash_method)
+                        has_external_storage, flash_method, bootimg)
     generate_apkbuild(args, pkgname, name, manufacturer, arch, flash_method)

pmb/helpers/frontend.py

@@ -223,3 +223,11 @@ def zap(args):
     pmb.chroot.zap(args, packages=args.packages, http=args.http,
                    mismatch_bins=args.mismatch_bins, old_bins=args.old_bins,
                    distfiles=args.distfiles)
+
+
+def bootimg_analyze(args):
+    bootimg = pmb.parse.bootimg(args, args.path)
+    tmp_output = "Put these variables in the deviceinfo file of your device:\n"
+    for line in pmb.aportgen.device.generate_deviceinfo_fastboot_content(args, bootimg).split("\n"):
+        tmp_output += "\n" + line.lstrip()
+    logging.info(tmp_output)

pmb/parse/__init__.py

@@ -21,4 +21,5 @@ from pmb.parse.apkbuild import apkbuild
 from pmb.parse.binfmt_info import binfmt_info
 from pmb.parse.deviceinfo import deviceinfo
 from pmb.parse.kconfig import check
+from pmb.parse.bootimg import bootimg
 import pmb.parse.arch

pmb/parse/arguments.py

@@ -316,6 +316,11 @@ def arguments():
     config.add_argument("name", nargs="?", help="variable name")
     config.add_argument("value", nargs="?", help="set variable to value")
 
+    # Action: bootimg_analyze
+    bootimg_analyze = sub.add_parser("bootimg_analyze", help="Extract all the"
+                                     " information from an existing boot.img")
+    bootimg_analyze.add_argument("path", help="path to the boot.img")
+
     # Use defaults from the user's config file
     args = parser.parse_args()
     cfg = pmb.config.load(args)
@@ -346,7 +351,7 @@ def arguments():
                             "find_aport": {}})
 
     # Add and verify the deviceinfo (only after initialization)
-    if args.action not in ("init", "config"):
+    if args.action not in ("init", "config", "bootimg_analyze"):
         setattr(args, "deviceinfo", pmb.parse.deviceinfo(args))
         arch = args.deviceinfo["arch"]
         if (arch != args.arch_native and

pmb/parse/bootimg.py

@@ -0,0 +1,73 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import logging
+import pmb
+
+
+def bootimg(args, path):
+    if not os.path.exists(path):
+        raise RuntimeError("Could not find file '" + path + "'")
+
+    logging.info("NOTE: You will be prompted for your sudo password, so we can set"
+                 " up a chroot to extract and analyze your boot.img file")
+    pmb.chroot.apk.install(args, ["file", "unpackbootimg"])
+
+    temp_path = pmb.chroot.other.tempfolder(args, "/tmp/bootimg_parser")
+    bootimg_path = args.work + "/chroot_native" + temp_path + "/boot.img"
+
+    # Copy the boot.img into the chroot temporary folder
+    pmb.helpers.run.root(args, ["cp", path, bootimg_path])
+
+    file_output = pmb.chroot.user(args, ["file", "-b", "boot.img"], working_dir=temp_path,
+                                  return_stdout=True).rstrip()
+    if "android bootimg" not in file_output.lower():
+        if "linux kernel" in file_output.lower():
+            raise RuntimeError("File is a Kernel image, you might need the 'heimdall-isorec'"
+                               " flash method. See also: "
+                               "<https://wiki.postmarketos.org/wiki/Deviceinfo_flash_methods>")
+        else:
+            raise RuntimeError("File is not an Android bootimg. (" + file_output + ")")
+
+    # Extract all the files
+    pmb.chroot.user(args, ["unpackbootimg", "-i", "boot.img"], working_dir=temp_path)
+
+    output = {}
+    # Get base, offsets, pagesize, cmdline and qcdt info
+    with open(bootimg_path + "-base", 'r') as f:
+        output["base"] = ("0x%08x" % int(f.read().replace('\n', ''), 16))
+    with open(bootimg_path + "-kernel_offset", 'r') as f:
+        output["kernel_offset"] = ("0x%08x" % int(f.read().replace('\n', ''), 16))
+    with open(bootimg_path + "-ramdisk_offset", 'r') as f:
+        output["ramdisk_offset"] = ("0x%08x" % int(f.read().replace('\n', ''), 16))
+    with open(bootimg_path + "-second_offset", 'r') as f:
+        output["second_offset"] = ("0x%08x" % int(f.read().replace('\n', ''), 16))
+    with open(bootimg_path + "-tags_offset", 'r') as f:
+        output["tags_offset"] = ("0x%08x" % int(f.read().replace('\n', ''), 16))
+    with open(bootimg_path + "-pagesize", 'r') as f:
+        output["pagesize"] = f.read().replace('\n', '')
+    with open(bootimg_path + "-cmdline", 'r') as f:
+        output["cmdline"] = f.read().replace('\n', '')
+    output["qcdt"] = ("true" if os.path.isfile(bootimg_path + "-dt") and
+                      os.path.getsize(bootimg_path + "-dt") > 0 else "false")
+
+    # Cleanup
+    pmb.chroot.root(args, ["rm", "-r", temp_path])
+
+    return output

test/test_aportgen_device_wizard.py

@@ -139,6 +139,7 @@ def test_aportgen_device_wizard(args, monkeypatch):
     # fastboot (mkbootimg)
     answers["overwrite"] = "y"
     answers["Flash method"] = "fastboot"
+    answers["Path"] = ""
     deviceinfo, apkbuild, apkbuild_linux = generate(args, monkeypatch, answers)
     assert apkbuild["depends"] == ["linux-testsuite-testdevice", "mkbootimg"]
     assert deviceinfo["flash_methods"] == answers["Flash method"]

test/test_bootimg.py

@@ -0,0 +1,85 @@
+"""
+Copyright 2017 Oliver Smith
+
+This file is part of pmbootstrap.
+
+pmbootstrap is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+pmbootstrap is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with pmbootstrap.  If not, see <http://www.gnu.org/licenses/>.
+"""
+import os
+import sys
+import pytest
+
+# Import from parent directory
+pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
+sys.path.append(pmb_src)
+import pmb.chroot.apk_static
+import pmb.parse.apkindex
+import pmb.helpers.logging
+import pmb.parse.bootimg
+
+
+@pytest.fixture
+def args(request):
+    import pmb.parse
+    sys.argv = ["pmbootstrap.py", "chroot"]
+    args = pmb.parse.arguments()
+    args.log = args.work + "/log_testsuite.txt"
+    pmb.helpers.logging.init(args)
+    request.addfinalizer(args.logfd.close)
+    return args
+
+
+def test_bootimg_invalid_path(args):
+    with pytest.raises(RuntimeError) as e:
+        pmb.parse.bootimg(args, "/invalid-path")
+    assert "Could not find file" in str(e.value)
+
+
+def test_bootimg_kernel(args):
+    path = pmb_src + "/test/testdata/bootimg/kernel-boot.img"
+    with pytest.raises(RuntimeError) as e:
+        pmb.parse.bootimg(args, path)
+    assert "heimdall-isorec" in str(e.value)
+
+
+def test_bootimg_invalid_file(args):
+    with pytest.raises(RuntimeError) as e:
+        pmb.parse.bootimg(args, __file__)
+    assert "File is not an Android bootimg" in str(e.value)
+
+
+def test_bootimg_normal(args):
+    path = pmb_src + "/test/testdata/bootimg/normal-boot.img"
+    output = {"base": "0x80000000",
+              "kernel_offset": "0x00008000",
+              "ramdisk_offset": "0x04000000",
+              "second_offset": "0x00f00000",
+              "tags_offset": "0x0e000000",
+              "pagesize": "2048",
+              "cmdline": "bootopt=64S3,32S1,32S1",
+              "qcdt": "false"}
+    assert pmb.parse.bootimg(args, path) == output
+
+
+def test_bootimg_qcdt(args):
+    path = pmb_src + "/test/testdata/bootimg/qcdt-boot.img"
+    output = {"base": "0x80000000",
+              "kernel_offset": "0x00008000",
+              "ramdisk_offset": "0x04000000",
+              "second_offset": "0x00f00000",
+              "tags_offset": "0x0e000000",
+              "pagesize": "2048",
+              "cmdline": "bootopt=64S3,32S1,32S1",
+              "qcdt": "true"}
+    assert pmb.parse.bootimg(args, path) == output

test/test_questions.py

@@ -22,8 +22,8 @@ import pytest
 import sys
 
 # Import from parent directory
-sys.path.append(os.path.realpath(
-    os.path.join(os.path.dirname(__file__) + "/..")))
+pmb_src = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
+sys.path.append(pmb_src)
 import pmb.aportgen.device
 import pmb.config
 import pmb.config.init
@@ -92,6 +92,23 @@ def test_questions(args, monkeypatch, tmpdir):
     answers = ["invalid_arch", "aarch64"]
     assert pmb.aportgen.device.ask_for_architecture(args) == "aarch64"
 
+    # Bootimg
+    func = pmb.aportgen.device.ask_for_bootimg
+    answers = ["invalid_path", ""]
+    assert func(args) is None
+
+    bootimg_path = pmb_src + "/test/testdata/bootimg/normal-boot.img"
+    answers = [bootimg_path]
+    output = {"base": "0x80000000",
+              "kernel_offset": "0x00008000",
+              "ramdisk_offset": "0x04000000",
+              "second_offset": "0x00f00000",
+              "tags_offset": "0x0e000000",
+              "pagesize": "2048",
+              "cmdline": "bootopt=64S3,32S1,32S1",
+              "qcdt": "false"}
+    assert func(args) == output
+
     # Device
     func = pmb.config.init.ask_for_device
     answers = ["lg-mako"]