diff --git a/pmb/aportgen/__init__.py b/pmb/aportgen/__init__.py index 897bf84b..5eb8542d 100644 --- a/pmb/aportgen/__init__.py +++ b/pmb/aportgen/__init__.py @@ -15,8 +15,11 @@ import pmb.helpers.cli def get_cross_package_arches(pkgname): """ Get the arches for which we want to build cross packages. + :param pkgname: package name, e.g. "gcc-aarch64", "gcc-x86_64" + :returns: string of architecture(s) (space separated) + """ if pkgname.endswith("-x86_64"): return "aarch64" @@ -32,7 +35,9 @@ def properties(pkgname): Example: "musl-armhf" => ("musl", "cross", {"confirm_overwrite": False}) :param pkgname: package name + :returns: (prefix, folder, options) + """ for folder, options in pmb.config.aportgen.items(): for prefix in options["prefixes"]: diff --git a/pmb/aportgen/core.py b/pmb/aportgen/core.py index 7e6789b4..6a6b8074 100644 --- a/pmb/aportgen/core.py +++ b/pmb/aportgen/core.py @@ -21,6 +21,7 @@ def format_function(name, body, remove_indent=4): """ Format the body of a shell function passed to rewrite() below, so it fits the format of the original APKBUILD. + :param remove_indent: Maximum number of spaces to remove from the beginning of each line of the function body. """ diff --git a/pmb/build/_package.py b/pmb/build/_package.py index d059e684..06d288b2 100644 --- a/pmb/build/_package.py +++ b/pmb/build/_package.py @@ -26,9 +26,9 @@ class BootstrapStage(enum.IntEnum): def skip_already_built(pkgname, arch): - """ - Check if the package was already built in this session, and add it - to the cache in case it was not built yet. + """Check if the package was already built in this session. + + Add it to the cache in case it was not built yet. :returns: True when it can be skipped or False """ @@ -45,9 +45,9 @@ def skip_already_built(pkgname, arch): def get_apkbuild(args, pkgname, arch): - """ - Parse the APKBUILD path for pkgname. When there is none, try to find it in - the binary package APKINDEX files or raise an exception. + """Parse the APKBUILD path for pkgname. + + When there is none, try to find it in the binary package APKINDEX files or raise an exception. :param pkgname: package name to be built, as specified in the APKBUILD :returns: None or parsed APKBUILD @@ -66,8 +66,8 @@ def get_apkbuild(args, pkgname, arch): def check_build_for_arch(args, pkgname, arch): - """ - Check if pmaport can be built or exists as binary for a specific arch. + """Check if pmaport can be built or exists as binary for a specific arch. + :returns: * True when it can be built * False when it can't be built, but exists in a binary repo (e.g. temp/mesa can't be built for x86_64, but Alpine has it) @@ -100,9 +100,10 @@ def check_build_for_arch(args, pkgname, arch): def get_depends(args, apkbuild): - """ - Alpine's abuild always builds/installs the "depends" and "makedepends" - of a package before building it. We used to only care about "makedepends" + """Alpine's abuild always builds/installs the "depends" and "makedepends" of a package + before building it. + + We used to only care about "makedepends" and it's still possible to ignore the depends with --ignore-depends. :returns: list of dependency pkgnames (eg. ["sdl2", "sdl2_net"]) @@ -126,8 +127,7 @@ def get_depends(args, apkbuild): def build_depends(args, apkbuild, arch, strict): - """ - Get and build dependencies with verbose logging messages. + """Get and build dependencies with verbose logging messages. :returns: (depends, depends_built) """ @@ -173,9 +173,7 @@ def build_depends(args, apkbuild, arch, strict): def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built): - """ - Check if a build is necessary, and warn if it is not, but there were - dependencies built. + """Check if a build is necessary, and warn if it is not, but there were dependencies built. :returns: True or False """ @@ -197,8 +195,9 @@ def is_necessary_warn_depends(args, apkbuild, arch, force, depends_built): def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None, suffix="native", skip_init_buildenv=False, src=None): - """ - Build all dependencies, check if we need to build at all (otherwise we've + """Build all dependencies. + + Check if we need to build at all (otherwise we've just initialized the build environment for nothing) and then setup the whole build environment (abuild, gcc, dependencies, cross-compiler). @@ -245,11 +244,11 @@ def init_buildenv(args, apkbuild, arch, strict=False, force=False, cross=None, def get_pkgver(original_pkgver, original_source=False, now=None): - """ - Get the original pkgver when using the original source. Otherwise, get the - pkgver with an appended suffix of current date and time. For example: - _p20180218550502 - When appending the suffix, an existing suffix (e.g. _git20171231) gets + """Get the original pkgver when using the original source. + + Otherwise, get the pkgver with an appended suffix of current date and time. + For example: ``_p20180218550502`` + When appending the suffix, an existing suffix (e.g. ``_git20171231``) gets replaced. :param original_pkgver: unmodified pkgver from the package's APKBUILD. @@ -268,8 +267,7 @@ def get_pkgver(original_pkgver, original_source=False, now=None): def override_source(args, apkbuild, pkgver, src, suffix="native"): - """ - Mount local source inside chroot and append new functions (prepare() etc.) + """Mount local source inside chroot and append new functions (prepare() etc.) to the APKBUILD to make it use the local source. """ if not src: @@ -346,8 +344,7 @@ def mount_pmaports(args, destination, suffix="native"): def link_to_git_dir(args, suffix): - """ - Make /home/pmos/build/.git point to the .git dir from pmaports.git, with a + """ Make ``/home/pmos/build/.git`` point to the .git dir from pmaports.git, with a symlink so abuild does not fail (#1841). abuild expects the current working directory to be a subdirectory of a @@ -457,9 +454,7 @@ def run_abuild(args, apkbuild, arch, strict=False, force=False, cross=None, def finish(args, apkbuild, arch, output, strict=False, suffix="native"): - """ - Various finishing tasks that need to be done after a build. - """ + """Various finishing tasks that need to be done after a build.""" # Verify output file channel = pmb.config.pmaports.read_config(args)["channel"] path = f"{args.work}/packages/{channel}/{output}" diff --git a/pmb/build/checksum.py b/pmb/build/checksum.py index 921ed676..c2e15ba8 100644 --- a/pmb/build/checksum.py +++ b/pmb/build/checksum.py @@ -9,7 +9,7 @@ import pmb.helpers.pmaports def update(args, pkgname): - """ Fetch all sources and update the checksums in the APKBUILD. """ + """Fetch all sources and update the checksums in the APKBUILD.""" pmb.build.init_abuild_minimal(args) pmb.build.copy_to_buildpath(args, pkgname) logging.info("(native) generate checksums for " + pkgname) @@ -23,7 +23,7 @@ def update(args, pkgname): def verify(args, pkgname): - """ Fetch all sources and verify their checksums. """ + """Fetch all sources and verify their checksums.""" pmb.build.init_abuild_minimal(args) pmb.build.copy_to_buildpath(args, pkgname) logging.info("(native) verify checksums for " + pkgname) diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py index 692200fa..c2dea2ff 100644 --- a/pmb/build/envkernel.py +++ b/pmb/build/envkernel.py @@ -13,8 +13,7 @@ import pmb.parse def match_kbuild_out(word): - """ - Look for paths in the following formats: + """Look for paths in the following formats: "//arch//boot" "//include/config/kernel.release" @@ -48,16 +47,15 @@ def match_kbuild_out(word): def find_kbuild_output_dir(function_body): - """ - Guess what the kernel build output directory is. Parses each line of the - function word by word, looking for paths which contain the kbuild output - directory. + """Guess what the kernel build output directory is. + + Parses each line of the function word by word, looking for paths which + contain the kbuild output directory. :param function_body: contents of a function from the kernel APKBUILD :returns: kbuild output dir None, when output dir is not found """ - guesses = [] for line in function_body: for item in line.split(): @@ -87,9 +85,7 @@ def find_kbuild_output_dir(function_body): def modify_apkbuild(args, pkgname, aport): - """ - Modify kernel APKBUILD to package build output from envkernel.sh - """ + """Modify kernel APKBUILD to package build output from envkernel.sh.""" apkbuild_path = aport + "/APKBUILD" apkbuild = pmb.parse.apkbuild(apkbuild_path) if os.path.exists(args.work + "/aportgen"): @@ -174,10 +170,7 @@ def run_abuild(args, pkgname, arch, apkbuild_path, kbuild_out): def package_kernel(args): - """ - Frontend for 'pmbootstrap build --envkernel': creates a package from - envkernel output. - """ + """Frontend for 'pmbootstrap build --envkernel': creates a package from envkernel output.""" pkgname = args.packages[0] if len(args.packages) > 1 or not pkgname.startswith("linux-"): raise RuntimeError("--envkernel needs exactly one linux-* package as " diff --git a/pmb/build/init.py b/pmb/build/init.py index fcc2a663..505f9fc3 100644 --- a/pmb/build/init.py +++ b/pmb/build/init.py @@ -14,8 +14,7 @@ import pmb.parse.arch def init_abuild_minimal(args, suffix="native"): - """ Initialize a minimal chroot with abuild where one can do - 'abuild checksum'. """ + """Initialize a minimal chroot with abuild where one can do 'abuild checksum'.""" marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_abuild_init_done" if os.path.exists(marker): return @@ -35,7 +34,7 @@ def init_abuild_minimal(args, suffix="native"): def init(args, suffix="native"): - """ Initialize a chroot for building packages with abuild. """ + """Initialize a chroot for building packages with abuild.""" marker = f"{args.work}/chroot_{suffix}/tmp/pmb_chroot_build_init_done" if os.path.exists(marker): return diff --git a/pmb/build/kconfig.py b/pmb/build/kconfig.py index 1640f30c..0000f80b 100644 --- a/pmb/build/kconfig.py +++ b/pmb/build/kconfig.py @@ -15,13 +15,15 @@ import pmb.parse def get_arch(apkbuild): - """ - Take the architecture from the APKBUILD or complain if it's ambiguous. This - function only gets called if --arch is not set. + """Take the architecture from the APKBUILD or complain if it's ambiguous. + + This function only gets called if --arch is not set. :param apkbuild: looks like: {"pkgname": "linux-...", "arch": ["x86_64", "armhf", "aarch64"]} - or: {"pkgname": "linux-...", "arch": ["armhf"]} + + or: {"pkgname": "linux-...", "arch": ["armhf"]} + """ pkgname = apkbuild["pkgname"] @@ -40,8 +42,8 @@ def get_arch(apkbuild): def get_outputdir(args, pkgname, apkbuild): - """ - Get the folder for the kernel compilation output. + """Get the folder for the kernel compilation output. + For most APKBUILDs, this is $builddir. But some older ones still use $srcdir/build (see the discussion in #1551). """ diff --git a/pmb/build/other.py b/pmb/build/other.py index 75a1a2d6..12c1ab7c 100644 --- a/pmb/build/other.py +++ b/pmb/build/other.py @@ -43,9 +43,9 @@ def copy_to_buildpath(args, package, suffix="native"): def is_necessary(args, arch, apkbuild, indexes=None): - """ - Check if the package has already been built. Compared to abuild's check, - this check also works for different architectures. + """Check if the package has already been built. + + Compared to abuild's check, this check also works for different architectures. :param arch: package target architecture :param apkbuild: from pmb.parse.apkbuild() @@ -89,8 +89,7 @@ def is_necessary(args, arch, apkbuild, indexes=None): def index_repo(args, arch=None): - """ - Recreate the APKINDEX.tar.gz for a specific repo, and clear the parsing + """Recreate the APKINDEX.tar.gz for a specific repo, and clear the parsing cache for that file for the current pmbootstrap session (to prevent rebuilding packages twice, in case the rebuild takes less than a second). @@ -126,8 +125,7 @@ def index_repo(args, arch=None): def configure_abuild(args, suffix, verify=False): - """ - Set the correct JOBS count in abuild.conf + """Set the correct JOBS count in ``abuild.conf``. :param verify: internally used to test if changing the config has worked. """ @@ -152,8 +150,7 @@ def configure_abuild(args, suffix, verify=False): def configure_ccache(args, suffix="native", verify=False): - """ - Set the maximum ccache size + """Set the maximum ccache size. :param verify: internally used to test if changing the config has worked. """ diff --git a/pmb/chroot/apk.py b/pmb/chroot/apk.py index de5cd687..e6fef576 100644 --- a/pmb/chroot/apk.py +++ b/pmb/chroot/apk.py @@ -256,13 +256,15 @@ def installed(args, suffix="native"): :returns: a dictionary with the following structure: { "postmarketos-mkinitfs": - { - "pkgname": "postmarketos-mkinitfs" - "version": "0.0.4-r10", - "depends": ["busybox-extras", "lddtree", ...], - "provides": ["mkinitfs=0.0.1"] - }, ... + { + "pkgname": "postmarketos-mkinitfs" + "version": "0.0.4-r10", + "depends": ["busybox-extras", "lddtree", ...], + "provides": ["mkinitfs=0.0.1"] + }, ... + } + """ path = f"{args.work}/chroot_{suffix}/lib/apk/db/installed" return pmb.parse.apkindex.parse(path, False) diff --git a/pmb/ci/__init__.py b/pmb/ci/__init__.py index 39d3c61a..01f291c8 100644 --- a/pmb/ci/__init__.py +++ b/pmb/ci/__init__.py @@ -13,12 +13,12 @@ def get_ci_scripts(topdir): """ Find 'pmbootstrap ci'-compatible scripts inside a git repository, and parse their metadata (description, options). The reference is at: https://postmarketos.org/pmb-ci - :param topdir: top directory of the git repository, get it with: - pmb.helpers.git.get_topdir() - :returns: a dict of CI scripts found in the git repository, e.g. - {"ruff": {"description": "lint all python scripts", - "options": []}, - ...} """ + + :param topdir: top directory of the git repository, get it with: pmb.helpers.git.get_topdir() + + :returns: a dict of CI scripts found in the git repository, e.g. + {"ruff": {"description": "lint all python scripts", "options": []}, ...} + """ ret = {} for script in glob.glob(f"{topdir}/.ci/*.sh"): is_pmb_ci_script = False @@ -59,9 +59,13 @@ def sort_scripts_by_speed(scripts): """ Order the scripts, so fast scripts run before slow scripts. Whether a script is fast or not is determined by the '# Options: slow' comment in the file. - :param scripts: return of get_ci_scripts() - :returns: same format as get_ci_scripts(), but as ordered dict with - fast scripts before slow scripts """ + + :param scripts: return of get_ci_scripts() + + :returns: same format as get_ci_scripts(), but as ordered dict with + fast scripts before slow scripts + + """ ret = collections.OrderedDict() # Fast scripts first @@ -81,8 +85,12 @@ def sort_scripts_by_speed(scripts): def ask_which_scripts_to_run(scripts_available): """ Display an interactive prompt about which of the scripts the user wishes to run, or all of them. - :param scripts_available: same format as get_ci_scripts() - :returns: either full scripts_available (all selected), or a subset """ + + :param scripts_available: same format as get_ci_scripts() + + :returns: either full scripts_available (all selected), or a subset + + """ count = len(scripts_available.items()) choices = ["all"] @@ -107,8 +115,11 @@ def ask_which_scripts_to_run(scripts_available): def copy_git_repo_to_chroot(args, topdir): """ Create a tarball of the git repo (including unstaged changes and new files) and extract it in chroot_native. - :param topdir: top directory of the git repository, get it with: - pmb.helpers.git.get_topdir() """ + + :param topdir: top directory of the git repository, get it with: + pmb.helpers.git.get_topdir() + + """ pmb.chroot.init(args) tarball_path = f"{args.work}/chroot_native/tmp/git.tar.gz" files = pmb.helpers.git.get_files(args, topdir) @@ -132,9 +143,13 @@ def run_scripts(args, topdir, scripts): """ Run one of the given scripts after another, either natively or in a chroot. Display a progress message and stop on error (without printing a python stack trace). - :param topdir: top directory of the git repository, get it with: - pmb.helpers.git.get_topdir() - :param scripts: return of get_ci_scripts() """ + + :param topdir: top directory of the git repository, get it with: + pmb.helpers.git.get_topdir() + + :param scripts: return of get_ci_scripts() + + """ steps = len(scripts) step = 0 repo_copied = False diff --git a/pmb/config/__init__.py b/pmb/config/__init__.py index 83e391f6..e781dffc 100644 --- a/pmb/config/__init__.py +++ b/pmb/config/__init__.py @@ -559,7 +559,7 @@ kconfig_options_containers = { } }, ">=3.13": { - "all": { # needed for iptables-nft (used by docker,tailscale) + "all": { # needed for iptables-nft (used by docker,tailscale) "NFT_COMPAT": True, } }, @@ -831,17 +831,17 @@ deviceinfo_attributes = [ "flash_heimdall_partition_kernel", "flash_heimdall_partition_initfs", "flash_heimdall_partition_rootfs", - "flash_heimdall_partition_system", # deprecated + "flash_heimdall_partition_system", # deprecated "flash_heimdall_partition_vbmeta", "flash_heimdall_partition_dtbo", "flash_fastboot_partition_kernel", "flash_fastboot_partition_rootfs", - "flash_fastboot_partition_system", # deprecated + "flash_fastboot_partition_system", # deprecated "flash_fastboot_partition_vbmeta", "flash_fastboot_partition_dtbo", "flash_rk_partition_kernel", "flash_rk_partition_rootfs", - "flash_rk_partition_system", # deprecated + "flash_rk_partition_system", # deprecated "flash_mtkclient_partition_kernel", "flash_mtkclient_partition_rootfs", "flash_mtkclient_partition_vbmeta", @@ -851,7 +851,7 @@ deviceinfo_attributes = [ "generate_bootimg", "header_version", "bootimg_qcdt", - "bootimg_mtk_mkimage", # deprecated + "bootimg_mtk_mkimage", # deprecated "bootimg_mtk_label_kernel", "bootimg_mtk_label_ramdisk", "bootimg_dtb_second", diff --git a/pmb/config/init.py b/pmb/config/init.py index 195d40db..ba9e741f 100644 --- a/pmb/config/init.py +++ b/pmb/config/init.py @@ -35,8 +35,7 @@ def require_programs(): def ask_for_username(args): - """ - Ask for a reasonable username for the non-root user. + """Ask for a reasonable username for the non-root user. :returns: the username """ @@ -52,13 +51,12 @@ def ask_for_username(args): def ask_for_work_path(args): - """ - Ask for the work path, until we can create it (when it does not exist) and - write into it. + """Ask for the work path, until we can create it (when it does not exist) and write into it. + :returns: (path, exists) - * path: is the full path, with expanded ~ sign - * exists: is False when the folder did not exist before we tested - whether we can create it + * path: is the full path, with expanded ~ sign + * exists: is False when the folder did not exist before we tested whether we can create it + """ logging.info("Location of the 'work' path. Multiple chroots" " (native, device arch, device rootfs) will be created" @@ -100,10 +98,12 @@ def ask_for_work_path(args): def ask_for_channel(args): - """ Ask for the postmarketOS release channel. The channel dictates, which - pmaports branch pmbootstrap will check out, and which repository URLs - will be used when initializing chroots. - :returns: channel name (e.g. "edge", "v21.03") """ + """Ask for the postmarketOS release channel. + The channel dictates, which pmaports branch pmbootstrap will check out, + and which repository URLs will be used when initializing chroots. + + :returns: channel name (e.g. "edge", "v21.03") + """ channels_cfg = pmb.helpers.git.parse_channels_cfg(args) count = len(channels_cfg["channels"]) @@ -257,9 +257,7 @@ def ask_for_timezone(args): def ask_for_provider_select(args, apkbuild, providers_cfg): - """ - Ask for selectable providers that are specified using "_pmb_select" - in a APKBUILD. + """Ask for selectable providers that are specified using "_pmb_select" in a APKBUILD. :param apkbuild: the APKBUILD with the _pmb_select :param providers_cfg: the configuration section with previously selected @@ -314,8 +312,7 @@ def ask_for_provider_select(args, apkbuild, providers_cfg): def ask_for_provider_select_pkg(args, pkgname, providers_cfg): - """ - Look up the APKBUILD for the specified pkgname and ask for selectable + """Look up the APKBUILD for the specified pkgname and ask for selectable providers that are specified using "_pmb_select". :param pkgname: name of the package to search APKBUILD for @@ -331,12 +328,14 @@ def ask_for_provider_select_pkg(args, pkgname, providers_cfg): def ask_for_device_kernel(args, device): - """ - Ask for the kernel that should be used with the device. + """Ask for the kernel that should be used with the device. :param device: code name, e.g. "lg-mako" + :returns: None if the kernel is hardcoded in depends without subpackages + :returns: kernel type ("downstream", "stable", "mainline", ...) + """ # Get kernels kernels = pmb.parse._apkbuild.kernels(args, device) diff --git a/pmb/config/merge_with_args.py b/pmb/config/merge_with_args.py index 262a28f0..50361fb0 100644 --- a/pmb/config/merge_with_args.py +++ b/pmb/config/merge_with_args.py @@ -4,8 +4,7 @@ import pmb.config def merge_with_args(args): - """ - We have the internal config (pmb/config/__init__.py) and the user config + """We have the internal config (pmb/config/__init__.py) and the user config (usually ~/.config/pmbootstrap.cfg, can be changed with the '-c' parameter). diff --git a/pmb/config/pmaports.py b/pmb/config/pmaports.py index 4660ab71..efa36796 100644 --- a/pmb/config/pmaports.py +++ b/pmb/config/pmaports.py @@ -104,7 +104,7 @@ def read_config_repos(args): def read_config(args): - """ Read and verify pmaports.cfg. """ + """Read and verify pmaports.cfg.""" # Try cache first cache_key = "pmb.config.pmaports.read_config" if pmb.helpers.other.cache[cache_key]: @@ -140,12 +140,16 @@ def read_config(args): def read_config_channel(args): - """ Get the properties of the currently active channel in pmaports.git, - as specified in channels.cfg (https://postmarketos.org/channels.cfg). - :returns: {"description: ..., - "branch_pmaports": ..., - "branch_aports": ..., - "mirrordir_alpine": ...} """ + """Get the properties of the currently active channel in pmaports.git. + + As specified in channels.cfg (https://postmarketos.org/channels.cfg). + + :returns: {"description: ..., + "branch_pmaports": ..., + "branch_aports": ..., + "mirrordir_alpine": ...} + + """ channel = read_config(args)["channel"] channels_cfg = pmb.helpers.git.parse_channels_cfg(args) @@ -179,9 +183,12 @@ def init(args): def switch_to_channel_branch(args, channel_new): - """ Checkout the channel's branch in pmaports.git. - :channel_new: channel name (e.g. "edge", "v21.03") - :returns: True if another branch was checked out, False otherwise """ + """Checkout the channel's branch in pmaports.git. + + :channel_new: channel name (e.g. "edge", "v21.03") + + :returns: True if another branch was checked out, False otherwise + """ # Check current pmaports branch channel channel_current = read_config(args)["channel"] if channel_current == channel_new: diff --git a/pmb/config/sudo.py b/pmb/config/sudo.py index baa5963e..0112e962 100644 --- a/pmb/config/sudo.py +++ b/pmb/config/sudo.py @@ -8,12 +8,11 @@ from typing import Optional @lru_cache() def which_sudo() -> Optional[str]: - """Returns a command required to run commands as root, if any. + """Return a command required to run commands as root, if any. Find whether sudo or doas is installed for commands that require root. Allows user to override preferred sudo with PMB_SUDO env variable. """ - if os.getuid() == 0: return None diff --git a/pmb/config/workdir.py b/pmb/config/workdir.py index 916a9366..863e44c8 100644 --- a/pmb/config/workdir.py +++ b/pmb/config/workdir.py @@ -13,7 +13,7 @@ import pmb.config.pmaports def chroot_save_init(args, suffix): - """ Save the chroot initialization data in $WORK/workdir.cfg. """ + """Save the chroot initialization data in $WORK/workdir.cfg.""" # Read existing cfg cfg = configparser.ConfigParser() path = args.work + "/workdir.cfg" @@ -88,10 +88,12 @@ def chroot_check_channel(args, suffix): def clean(args): - """ Remove obsolete data data from workdir.cfg. - :returns: None if workdir does not exist, - True if config was rewritten, - False if config did not change """ + """Remove obsolete data data from workdir.cfg. + + :returns: None if workdir does not exist, + True if config was rewritten, + False if config did not change + """ # Skip if workdir.cfg doesn't exist path = args.work + "/workdir.cfg" if not os.path.exists(path): diff --git a/pmb/helpers/apk.py b/pmb/helpers/apk.py index 774ca0b9..8f5329eb 100644 --- a/pmb/helpers/apk.py +++ b/pmb/helpers/apk.py @@ -11,8 +11,7 @@ import pmb.parse.version def _run(args, command, chroot=False, suffix="native", output="log"): - """ - Run a command. + """Run a command. :param command: command in list form :param chroot: whether to run the command inside the chroot or on the host @@ -29,8 +28,7 @@ def _run(args, command, chroot=False, suffix="native", output="log"): def _prepare_fifo(args, chroot=False, suffix="native"): - """ - Prepare the progress fifo for reading / writing. + """Prepare the progress fifo for reading / writing. :param chroot: whether to run the command inside the chroot or on the host :param suffix: chroot suffix. Only applies if the "chroot" parameter is @@ -53,9 +51,7 @@ def _prepare_fifo(args, chroot=False, suffix="native"): def _create_command_with_progress(command, fifo): - """ - Build a full apk command from a subcommand, set up to redirect progress - into a fifo. + """Build a full apk command from a subcommand, set up to redirect progress into a fifo. :param command: apk subcommand in list form :param fifo: path of the fifo @@ -69,8 +65,7 @@ def _create_command_with_progress(command, fifo): def _compute_progress(line): - """ - Compute the progress as a number between 0 and 1. + """Compute the progress as a number between 0 and 1. :param line: line as read from the progress fifo :returns: progress as a number between 0 and 1 @@ -86,8 +81,7 @@ def _compute_progress(line): def apk_with_progress(args, command, chroot=False, suffix="native"): - """ - Run an apk subcommand while printing a progress bar to STDOUT. + """Run an apk subcommand while printing a progress bar to STDOUT. :param command: apk subcommand in list form :param chroot: whether to run commands inside the chroot or on the host @@ -112,10 +106,10 @@ def apk_with_progress(args, command, chroot=False, suffix="native"): def check_outdated(args, version_installed, action_msg): - """ - Check if the provided alpine version is outdated, depending on the alpine - mirrordir (edge, v3.12, ...) related to currently checked out pmaports - branch. + """Check if the provided alpine version is outdated. + + This depends on the alpine mirrordir (edge, v3.12, ...) related to currently checked out + pmaports branch. :param version_installed: currently installed apk version, e.g. "2.12.1-r0" :param action_msg: string explaining what the user should do to resolve diff --git a/pmb/helpers/aportupgrade.py b/pmb/helpers/aportupgrade.py index 02d606c4..d9597990 100644 --- a/pmb/helpers/aportupgrade.py +++ b/pmb/helpers/aportupgrade.py @@ -96,9 +96,8 @@ def get_package_version_info_gitlab(gitlab_host: str, repo_name: str, def upgrade_git_package(args, pkgname: str, package) -> None: - """ - Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if - args.dry is set). + """Update _commit/pkgver/pkgrel in a git-APKBUILD (or pretend to do it if args.dry is set). + :param pkgname: the package name :param package: a dict containing package information """ @@ -254,8 +253,7 @@ def upgrade_stable_package(args, pkgname: str, package) -> None: def upgrade(args, pkgname, git=True, stable=True) -> None: - """ - Find new versions of a single package and upgrade it. + """Find new versions of a single package and upgrade it. :param pkgname: the name of the package :param git: True if git packages should be upgraded @@ -275,9 +273,7 @@ def upgrade(args, pkgname, git=True, stable=True) -> None: def upgrade_all(args) -> None: - """ - Upgrade all packages, based on args.all, args.all_git and args.all_stable. - """ + """Upgrade all packages, based on args.all, args.all_git and args.all_stable.""" for pkgname in pmb.helpers.pmaports.get_list(args): # Always ignore postmarketOS-specific packages that have no upstream # source diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index f0168037..4e0ae6fe 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -5,9 +5,9 @@ import os import pmb.config import pmb.helpers.git -""" This file constructs the args variable, which is passed to almost all - functions in the pmbootstrap code base. Here's a listing of the kind of - information it stores. +"""This file constructs the args variable, which is passed to almost all + functions in the pmbootstrap code base. Here's a listing of the kind of + information it stores. 1. Argparse Variables directly from command line argument parsing (see @@ -44,16 +44,16 @@ import pmb.helpers.git def fix_mirrors_postmarketos(args): - """ Fix args.mirrors_postmarketos when it is supposed to be empty or the - default value. + """Fix args.mirrors_postmarketos when it is supposed to be empty or the default value. - In pmb/parse/arguments.py, we set the -mp/--mirror-pmOS argument to - action="append" and start off with an empty list. That way, users can - specify multiple custom mirrors by specifying -mp multiple times on the - command line. Here we fix the default and no mirrors case. + In pmb/parse/arguments.py, we set the -mp/--mirror-pmOS argument to + action="append" and start off with an empty list. That way, users can + specify multiple custom mirrors by specifying -mp multiple times on the + command line. Here we fix the default and no mirrors case. - NOTE: we don't use nargs="+", because it does not play nicely with - subparsers: """ + NOTE: we don't use nargs="+", because it does not play nicely with + subparsers: + """ # -mp not specified: use default mirrors if not args.mirrors_postmarketos: cfg = pmb.config.load(args) @@ -66,19 +66,21 @@ def fix_mirrors_postmarketos(args): def check_pmaports_path(args): - """ Make sure that args.aports exists when it was overridden by --aports. - Without this check, 'pmbootstrap init' would start cloning the - pmaports into the default folder when args.aports does not exist. """ + """Make sure that args.aports exists when it was overridden by --aports. + + Without this check, 'pmbootstrap init' would start cloning the + pmaports into the default folder when args.aports does not exist. + """ if args.from_argparse.aports and not os.path.exists(args.aports): raise ValueError("pmaports path (specified with --aports) does" " not exist: " + args.aports) def replace_placeholders(args): - """ Replace $WORK and ~ (for path variables) in variables from any config - (user's config file, default config settings or config parameters - specified on commandline) """ + """Replace $WORK and ~ (for path variables) in variables from any config. + (user's config file, default config settings or config parameters specified on commandline) + """ # Replace $WORK for key, value in pmb.config.defaults.items(): if key not in args: @@ -94,7 +96,7 @@ def replace_placeholders(args): def add_deviceinfo(args): - """ Add and verify the deviceinfo (only after initialization) """ + """Add and verify the deviceinfo (only after initialization)""" setattr(args, "deviceinfo", pmb.parse.deviceinfo(args)) arch = args.deviceinfo["arch"] if (arch != pmb.config.arch_native and @@ -126,7 +128,7 @@ def init(args): def update_work(args, work): - """ Update the work path in args.work and wherever $WORK was used. """ + """Update the work path in args.work and wherever $WORK was used.""" # Start with the unmodified args from argparse args_new = copy.deepcopy(args.from_argparse) diff --git a/pmb/helpers/cli.py b/pmb/helpers/cli.py index a9fefe42..93b9a631 100644 --- a/pmb/helpers/cli.py +++ b/pmb/helpers/cli.py @@ -11,11 +11,10 @@ import pmb.config class ReadlineTabCompleter: - """ Stores intermediate state for completer function """ + """Store intermediate state for completer function.""" + def __init__(self, options): - """ - :param options: list of possible completions - """ + """:param options: list of possible completions.""" self.options = sorted(options) self.matches = [] @@ -40,11 +39,10 @@ class ReadlineTabCompleter: def ask(question="Continue?", choices=["y", "n"], default="n", lowercase_answer=True, validation_regex=None, complete=None): - """ - Ask a question on the terminal. + """Ask a question on the terminal. + :param question: display prompt - :param choices: short list of possible answers, - displayed after prompt if set + :param choices: short list of possible answers, displayed after prompt if set :param default: default value to return if user doesn't input anything :param lowercase_answer: if True, convert return value to lower case :param validation_regex: if set, keep asking until regex matches @@ -99,11 +97,9 @@ def ask(question="Continue?", choices=["y", "n"], default="n", def confirm(args, question="Continue?", default=False, no_assumptions=False): - """ - Convenience wrapper around ask for simple yes-no questions with validation. + """Convenience wrapper around ask for simple yes-no questions with validation. - :param no_assumptions: ask for confirmation, even if "pmbootstrap -y' - is set + :param no_assumptions: ask for confirmation, even if "pmbootstrap -y' is set :returns: True for "y", False for "n" """ default_str = "y" if default else "n" @@ -115,9 +111,9 @@ def confirm(args, question="Continue?", default=False, no_assumptions=False): def progress_print(args, progress): - """ - Print a snapshot of a progress bar to STDOUT. Call progress_flush to end - printing progress and clear the line. No output is printed in + """Print a snapshot of a progress bar to STDOUT. + + Call progress_flush to end printing progress and clear the line. No output is printed in non-interactive mode. :param progress: completion percentage as a number between 0 and 1 @@ -138,9 +134,9 @@ def progress_print(args, progress): def progress_flush(args): - """ - Finish printing a progress bar. This will erase the line. Does nothing in - non-interactive mode. + """Finish printing a progress bar. + + This will erase the line. Does nothing in non-interactive mode. """ if pmb.config.is_interactive and not args.details_to_stdout: sys.stdout.flush() diff --git a/pmb/helpers/devices.py b/pmb/helpers/devices.py index b0583047..c6966c36 100644 --- a/pmb/helpers/devices.py +++ b/pmb/helpers/devices.py @@ -6,8 +6,8 @@ import pmb.parse def find_path(args, codename, file=''): - """ - Find path to device APKBUILD under `device/*/device-`. + """Find path to device APKBUILD under `device/*/device-`. + :param codename: device codename :param file: file to look for (e.g. APKBUILD or deviceinfo), may be empty :returns: path to APKBUILD @@ -24,8 +24,8 @@ def find_path(args, codename, file=''): def list_codenames(args, vendor=None, unmaintained=True): - """ - Get all devices, for which aports are available + """Get all devices, for which aports are available. + :param vendor: vendor name to choose devices from, or None for all vendors :param unmaintained: include unmaintained devices :returns: ["first-device", "second-device", ...] @@ -41,8 +41,8 @@ def list_codenames(args, vendor=None, unmaintained=True): def list_vendors(args): - """ - Get all device vendors, for which aports are available + """Get all device vendors, for which aports are available. + :returns: {"vendor1", "vendor2", ...} """ ret = set() @@ -53,9 +53,7 @@ def list_vendors(args): def list_apkbuilds(args): - """ - :returns: { "first-device": {"pkgname": ..., "pkgver": ...}, ... } - """ + """:returns: { "first-device": {"pkgname": ..., "pkgver": ...}, ... }""" ret = {} for device in list_codenames(args): apkbuild_path = f"{args.aports}/device/*/device-{device}/APKBUILD" @@ -64,9 +62,7 @@ def list_apkbuilds(args): def list_deviceinfos(args): - """ - :returns: { "first-device": {"name": ..., "screen_width": ...}, ... } - """ + """:returns: { "first-device": {"name": ..., "screen_width": ...}, ... }""" ret = {} for device in list_codenames(args): ret[device] = pmb.parse.deviceinfo(args, device) diff --git a/pmb/helpers/file.py b/pmb/helpers/file.py index c6a0ec4a..89ec50f9 100644 --- a/pmb/helpers/file.py +++ b/pmb/helpers/file.py @@ -19,11 +19,13 @@ def replace(path, old, new): def replace_apkbuild(args, pkgname, key, new, in_quotes=False): - """ Replace one key=value line in an APKBUILD and verify it afterwards. - :param pkgname: package name, e.g. "hello-world" - :param key: key that should be replaced, e.g. "pkgver" - :param new: new value - :param in_quotes: expect the value to be in quotation marks ("") """ + """Replace one key=value line in an APKBUILD and verify it afterwards. + + :param pkgname: package name, e.g. "hello-world" + :param key: key that should be replaced, e.g. "pkgver" + :param new: new value + :param in_quotes: expect the value to be in quotation marks ("") + """ # Read old value path = pmb.helpers.pmaports.find(args, pkgname) + "/APKBUILD" apkbuild = pmb.parse.apkbuild(path) @@ -51,8 +53,8 @@ def replace_apkbuild(args, pkgname, key, new, in_quotes=False): def is_up_to_date(path_sources, path_target=None, lastmod_target=None): - """ - Check if a file is up-to-date by comparing the last modified timestamps + """Check if a file is up-to-date by comparing the last modified timestamps. + (just like make does it). :param path_sources: list of full paths to the source files @@ -78,9 +80,7 @@ def is_up_to_date(path_sources, path_target=None, lastmod_target=None): def is_older_than(path, seconds): - """ - Check if a single file is older than a given amount of seconds. - """ + """Check if a single file is older than a given amount of seconds.""" if not os.path.exists(path): return True lastmod = os.path.getmtime(path) @@ -88,9 +88,7 @@ def is_older_than(path, seconds): def symlink(args, file, link): - """ - Checks if the symlink is already present, otherwise create it. - """ + """Check if the symlink is already present, otherwise create it.""" if os.path.exists(link): if (os.path.islink(link) and os.path.realpath(os.readlink(link)) == os.path.realpath(file)): diff --git a/pmb/helpers/frontend.py b/pmb/helpers/frontend.py index 87b6ba9f..7a5234ca 100644 --- a/pmb/helpers/frontend.py +++ b/pmb/helpers/frontend.py @@ -38,8 +38,8 @@ from argparse import Namespace def _parse_flavor(args, autoinstall=True): - """ - Verify the flavor argument if specified, or return a default value. + """Verify the flavor argument if specified, or return a default value. + :param autoinstall: make sure that at least one kernel flavor is installed """ # Install a kernel and get its "flavor", where flavor is a pmOS-specific diff --git a/pmb/helpers/git.py b/pmb/helpers/git.py index dab1b1a9..a896ba3a 100644 --- a/pmb/helpers/git.py +++ b/pmb/helpers/git.py @@ -12,22 +12,25 @@ import pmb.helpers.run def get_path(args, name_repo): - """ Get the path to the repository, which is either the default one in the - work dir, or a user-specified one in args. + """Get the path to the repository. - :returns: full path to repository """ + The path is either the default one in the work dir, or a user-specified one in args. + + :returns: full path to repository + """ if name_repo == "pmaports": return args.aports return args.work + "/cache_git/" + name_repo def clone(args, name_repo): - """ Clone a git repository to $WORK/cache_git/$name_repo (or to the - overridden path set in args, as with pmbootstrap --aports). + """Clone a git repository to $WORK/cache_git/$name_repo. - :param name_repo: short alias used for the repository name, from - pmb.config.git_repos (e.g. "aports_upstream", - "pmaports") """ + (or to the overridden path set in args, as with ``pmbootstrap --aports``). + + :param name_repo: short alias used for the repository name, from pmb.config.git_repos + (e.g. "aports_upstream", "pmaports") + """ # Check for repo name in the config if name_repo not in pmb.config.git_repos: raise ValueError("No git repository configured for " + name_repo) @@ -52,14 +55,14 @@ def clone(args, name_repo): def rev_parse(args, path, revision="HEAD", extra_args: list = []): - """ Run "git rev-parse" in a specific repository dir. + """Run "git rev-parse" in a specific repository dir. - :param path: to the git repository - :param extra_args: additional arguments for "git rev-parse". Pass - "--abbrev-ref" to get the branch instead of the - commit, if possible. - :returns: commit string like "90cd0ad84d390897efdcf881c0315747a4f3a966" - or (with --abbrev-ref): the branch name, e.g. "master" """ + :param path: to the git repository + :param extra_args: additional arguments for ``git rev-parse``. Pass + ``--abbrev-ref`` to get the branch instead of the commit, if possible. + :returns: commit string like "90cd0ad84d390897efdcf881c0315747a4f3a966" + or (with ``--abbrev-ref``): the branch name, e.g. "master" + """ command = ["git", "rev-parse"] + extra_args + [revision] rev = pmb.helpers.run.user(args, command, path, output_return=True) return rev.rstrip() @@ -77,15 +80,16 @@ def can_fast_forward(args, path, branch_upstream, branch="HEAD"): def clean_worktree(args, path): - """ Check if there are not any modified files in the git dir. """ + """Check if there are not any modified files in the git dir.""" command = ["git", "status", "--porcelain"] return pmb.helpers.run.user(args, command, path, output_return=True) == "" def get_upstream_remote(args, name_repo): - """ Find the remote, which matches the git URL from the config. Usually - "origin", but the user may have set up their git repository - differently. """ + """Find the remote, which matches the git URL from the config. + + Usually "origin", but the user may have set up their git repository differently. + """ url = pmb.config.git_repos[name_repo] path = get_path(args, name_repo) command = ["git", "remote", "-v"] @@ -98,14 +102,17 @@ def get_upstream_remote(args, name_repo): def parse_channels_cfg(args): - """ Parse channels.cfg from pmaports.git, origin/master branch. - Reference: https://postmarketos.org/channels.cfg - :returns: dict like: {"meta": {"recommended": "edge"}, - "channels": {"edge": {"description": ..., - "branch_pmaports": ..., - "branch_aports": ..., - "mirrordir_alpine": ...}, - ...}} """ + """Parse channels.cfg from pmaports.git, origin/master branch. + + Reference: https://postmarketos.org/channels.cfg + + :returns: dict like: {"meta": {"recommended": "edge"}, + "channels": {"edge": {"description": ..., + "branch_pmaports": ..., + "branch_aports": ..., + "mirrordir_alpine": ...}, + ...}} + """ # Cache during one pmbootstrap run cache_key = "pmb.helpers.git.parse_channels_cfg" if pmb.helpers.other.cache[cache_key]: @@ -151,8 +158,10 @@ def parse_channels_cfg(args): def get_branches_official(args, name_repo): - """ Get all branches that point to official release channels. - :returns: list of supported branches, e.g. ["master", "3.11"] """ + """Get all branches that point to official release channels. + + :returns: list of supported branches, e.g. ["master", "3.11"] + """ # This functions gets called with pmaports and aports_upstream, because # both are displayed in "pmbootstrap status". But it only makes sense # to display pmaports there, related code will be refactored soon (#1903). @@ -167,12 +176,14 @@ def get_branches_official(args, name_repo): def pull(args, name_repo): - """ Check if on official branch and essentially try 'git pull --ff-only'. - Instead of really doing 'git pull --ff-only', do it in multiple steps - (fetch, merge --ff-only), so we can display useful messages depending - on which part fails. + """Check if on official branch and essentially try ``git pull --ff-only``. - :returns: integer, >= 0 on success, < 0 on error """ + Instead of really doing ``git pull --ff-only``, do it in multiple steps + (``fetch, merge --ff-only``), so we can display useful messages depending + on which part fails. + + :returns: integer, >= 0 on success, < 0 on error + """ branches_official = get_branches_official(args, name_repo) # Skip if repo wasn't cloned @@ -229,18 +240,24 @@ def pull(args, name_repo): def get_topdir(args, path): - """ :returns: a string with the top dir of the git repository, or an - empty string if it's not a git repository. """ + """Get top-dir of git repo. + + :returns: a string with the top dir of the git repository, + or an empty string if it's not a git repository. + """ return pmb.helpers.run.user(args, ["git", "rev-parse", "--show-toplevel"], path, output_return=True, check=False).rstrip() def get_files(args, path): - """ Get all files inside a git repository, that are either already in the - git tree or are not in gitignore. Do not list deleted files. To be used - for creating a tarball of the git repository. - :param path: top dir of the git repository - :returns: all files in a git repository as list, relative to path """ + """Get all files inside a git repository, that are either already in the git tree or are not in gitignore. + + Do not list deleted files. To be used for creating a tarball of the git repository. + + :param path: top dir of the git repository + + :returns: all files in a git repository as list, relative to path + """ ret = [] files = pmb.helpers.run.user(args, ["git", "ls-files"], path, output_return=True).split("\n") diff --git a/pmb/helpers/http.py b/pmb/helpers/http.py index fe0f7c58..414698e1 100644 --- a/pmb/helpers/http.py +++ b/pmb/helpers/http.py @@ -12,20 +12,21 @@ import pmb.helpers.run def download(args, url, prefix, cache=True, loglevel=logging.INFO, allow_404=False): - """ Download a file to disk. + """Download a file to disk. - :param url: the http(s) address of to the file to download - :param prefix: for the cache, to make it easier to find (cache files - get a hash of the URL after the prefix) - :param cache: if True, and url is cached, do not download it again - :param loglevel: change to logging.DEBUG to only display the download - message in 'pmbootstrap log', not in stdout. We use - this when downloading many APKINDEX files at once, no - point in showing a dozen messages. - :param allow_404: do not raise an exception when the server responds - with a 404 Not Found error. Only display a warning on - stdout (no matter if loglevel is changed). - :returns: path to the downloaded file in the cache or None on 404 """ + :param url: the http(s) address of to the file to download + :param prefix: for the cache, to make it easier to find (cache files + get a hash of the URL after the prefix) + :param cache: if True, and url is cached, do not download it again + :param loglevel: change to logging.DEBUG to only display the download + message in 'pmbootstrap log', not in stdout. + We use this when downloading many APKINDEX files at once, no + point in showing a dozen messages. + :param allow_404: do not raise an exception when the server responds with a 404 Not Found error. + Only display a warning on stdout (no matter if loglevel is changed). + + :returns: path to the downloaded file in the cache or None on 404 + """ # Create cache folder if not os.path.exists(args.work + "/cache_http"): pmb.helpers.run.user(args, ["mkdir", "-p", args.work + "/cache_http"]) @@ -62,13 +63,14 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO, def retrieve(url, headers=None, allow_404=False): - """ Fetch the content of a URL and returns it as string. + """Fetch the content of a URL and returns it as string. - :param url: the http(s) address of to the resource to fetch - :param headers: dict of HTTP headers to use - :param allow_404: do not raise an exception when the server responds - with a 404 Not Found error. Only display a warning - :returns: str with the content of the response + :param url: the http(s) address of to the resource to fetch + :param headers: dict of HTTP headers to use + :param allow_404: do not raise an exception when the server responds with a + 404 Not Found error. Only display a warning + + :returns: str with the content of the response """ # Download the file logging.verbose("Retrieving " + url) @@ -89,6 +91,8 @@ def retrieve(url, headers=None, allow_404=False): def retrieve_json(*args, **kwargs): - """ Fetch the contents of a URL, parse it as JSON and return it. See - retrieve() for the list of all parameters. """ + """Fetch the contents of a URL, parse it as JSON and return it. + + See retrieve() for the list of all parameters. + """ return json.loads(retrieve(*args, **kwargs)) diff --git a/pmb/helpers/lint.py b/pmb/helpers/lint.py index 3ba60bec..00455a56 100644 --- a/pmb/helpers/lint.py +++ b/pmb/helpers/lint.py @@ -11,8 +11,7 @@ import pmb.helpers.pmaports def check(args, pkgnames): - """ - Run apkbuild-lint on the supplied packages + """Run apkbuild-lint on the supplied packages. :param pkgnames: Names of the packages to lint """ diff --git a/pmb/helpers/logging.py b/pmb/helpers/logging.py index 1e8fdf4e..a74e73eb 100644 --- a/pmb/helpers/logging.py +++ b/pmb/helpers/logging.py @@ -9,9 +9,7 @@ logfd = None class log_handler(logging.StreamHandler): - """ - Write to stdout and to the already opened log file. - """ + """Write to stdout and to the already opened log file.""" _args = None def emit(self, record): @@ -72,9 +70,9 @@ class log_handler(logging.StreamHandler): def add_verbose_log_level(): - """ - Add a new log level "verbose", which is below "debug". Also monkeypatch - logging, so it can be used with logging.verbose(). + """Add a new log level "verbose", which is below "debug". + + Also monkeypatch logging, so it can be used with logging.verbose(). This function is based on work by Voitek Zylinski and sleepycal: https://stackoverflow.com/a/20602183 @@ -91,10 +89,7 @@ def add_verbose_log_level(): def init(args): - """ - Set log format and add the log file descriptor to logfd, add the - verbose log level. - """ + """Set log format and add the log file descriptor to logfd, add the verbose log level.""" global logfd # Set log file descriptor (logfd) if args.details_to_stdout: diff --git a/pmb/helpers/mount.py b/pmb/helpers/mount.py index a3203b53..50de29b5 100644 --- a/pmb/helpers/mount.py +++ b/pmb/helpers/mount.py @@ -5,8 +5,8 @@ import pmb.helpers.run def ismount(folder): - """ - Ismount() implementation that works for mount --bind. + """Ismount() implementation that works for mount --bind. + Workaround for: https://bugs.python.org/issue29707 """ folder = os.path.realpath(os.path.realpath(folder)) @@ -21,8 +21,8 @@ def ismount(folder): def bind(args, source, destination, create_folders=True, umount=False): - """ - Mount --bind a folder and create necessary directory structure. + """Mount --bind a folder and create necessary directory structure. + :param umount: when destination is already a mount point, umount it first. """ # Check/umount destination @@ -51,10 +51,7 @@ def bind(args, source, destination, create_folders=True, umount=False): def bind_file(args, source, destination, create_folders=False): - """ - Mount a file with the --bind option, and create the destination file, - if necessary. - """ + """Mount a file with the --bind option, and create the destination file, if necessary.""" # Skip existing mountpoint if ismount(destination): return @@ -74,10 +71,12 @@ def bind_file(args, source, destination, create_folders=False): def umount_all_list(prefix, source="/proc/mounts"): - """ - Parses `/proc/mounts` for all folders beginning with a prefix. + """Parse `/proc/mounts` for all folders beginning with a prefix. + :source: can be changed for testcases + :returns: a list of folders that need to be umounted + """ ret = [] prefix = os.path.realpath(prefix) @@ -99,9 +98,7 @@ def umount_all_list(prefix, source="/proc/mounts"): def umount_all(args, folder): - """ - Umount all folders that are mounted inside a given folder. - """ + """Umount all folders that are mounted inside a given folder.""" for mountpoint in umount_all_list(folder): pmb.helpers.run.root(args, ["umount", mountpoint]) if ismount(mountpoint): diff --git a/pmb/helpers/other.py b/pmb/helpers/other.py index c0bec5f9..9f6ed4f7 100644 --- a/pmb/helpers/other.py +++ b/pmb/helpers/other.py @@ -12,10 +12,10 @@ import pmb.helpers.run def folder_size(args, path): - """ - Run `du` to calculate the size of a folder (this is less code and - faster than doing the same task in pure Python). This result is only - approximatelly right, but good enough for pmbootstrap's use case (#760). + """Run `du` to calculate the size of a folder. + + (this is less code and faster than doing the same task in pure Python) + This result is only approximatelly right, but good enough for pmbootstrap's use case (#760). :returns: folder size in kilobytes """ @@ -30,10 +30,10 @@ def folder_size(args, path): def check_grsec(): - """ - Check if the current kernel is based on the grsec patchset, and if - the chroot_deny_chmod option is enabled. Raise an exception in that - case, with a link to the issue. Otherwise, do nothing. + """Check if the current kernel is based on the grsec patchset. + + Also check if the chroot_deny_chmod option is enabled. + Raise an exception in that case, with a link to the issue. Otherwise, do nothing. """ path = "/proc/sys/kernel/grsecurity/chroot_deny_chmod" if not os.path.exists(path): @@ -44,9 +44,10 @@ def check_grsec(): def check_binfmt_misc(args): - """ - Check if the 'binfmt_misc' module is loaded by checking, if - /proc/sys/fs/binfmt_misc/ exists. If it exists, then do nothing. + """Check if the 'binfmt_misc' module is loaded. + + This is done by checking, if /proc/sys/fs/binfmt_misc/ exists. + If it exists, then do nothing. Otherwise, load the module and mount binfmt_misc. If that fails as well, raise an exception pointing the user to the wiki. """ @@ -241,11 +242,10 @@ def migrate_work_folder(args): def check_old_devices(args): - """ - Check if there are any device ports in device/*/APKBUILD, - rather than device/*/*/APKBUILD (e.g. device/testing/...). - """ + """Check if there are any device ports in device/\\*/APKBUILD. + Devices should be in device/\\*/\\*/APKBUILD (e.g. device/testing/...). + """ g = glob.glob(args.aports + "/device/*/APKBUILD") if not g: return @@ -257,8 +257,9 @@ def check_old_devices(args): def validate_hostname(hostname): - """ - Check whether the string is a valid hostname, according to + """Check whether the string is a valid hostname. + + Check is performed according to """ # Check length @@ -299,8 +300,7 @@ cache = None def init_cache(): global cache - """ Add a caching dict (caches parsing of files etc. for the current - session) """ + """Add a caching dict (caches parsing of files etc. for the current session).""" repo_update = {"404": [], "offline_msg_shown": False} cache = {"apkindex": {}, "apkbuild": {}, diff --git a/pmb/helpers/package.py b/pmb/helpers/package.py index f3541086..daa42e60 100644 --- a/pmb/helpers/package.py +++ b/pmb/helpers/package.py @@ -1,9 +1,12 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later -""" -Functions that work with both pmaports and binary package repos. See also: -- pmb/helpers/pmaports.py (work with pmaports) -- pmb/helpers/repo.py (work with binary package repos) +"""Functions that work with both pmaports and binary package repos. + +See also: + + - pmb/helpers/pmaports.py (work with pmaports) + + - pmb/helpers/repo.py (work with binary package repos) """ import copy import logging @@ -21,24 +24,24 @@ def remove_operators(package): def get(args, pkgname, arch, replace_subpkgnames=False, must_exist=True): - """ Find a package in pmaports, and as fallback in the APKINDEXes of the - binary packages. - :param pkgname: package name (e.g. "hello-world") - :param arch: preferred architecture of the binary package. When it - can't be found for this arch, we'll still look for another - arch to see whether the package exists at all. So make - sure to check the returned arch against what you wanted - with check_arch(). Example: "armhf" - :param replace_subpkgnames: replace all subpkgnames with their main - pkgnames in the depends (see #1733) - :param must_exist: raise an exception, if not found - :returns: * data from the parsed APKBUILD or APKINDEX in the following - format: {"arch": ["noarch"], - "depends": ["busybox-extras", "lddtree", ...], - "pkgname": "postmarketos-mkinitfs", - "provides": ["mkinitfs=0..1"], - "version": "0.0.4-r10"} - * None if the package was not found """ + """Find a package in pmaports, and as fallback in the APKINDEXes of the binary packages. + + :param pkgname: package name (e.g. "hello-world") + :param arch: preferred architecture of the binary package. + When it can't be found for this arch, we'll still look for another arch to see whether the + package exists at all. So make sure to check the returned arch against what you wanted + with check_arch(). Example: "armhf" + :param replace_subpkgnames: replace all subpkgnames with their main pkgnames in the depends + (see #1733) + :param must_exist: raise an exception, if not found + + :returns: * data from the parsed APKBUILD or APKINDEX in the following format: + {"arch": ["noarch"], "depends": ["busybox-extras", "lddtree", ...], + "pkgname": "postmarketos-mkinitfs", "provides": ["mkinitfs=0..1"], + "version": "0.0.4-r10"} + + * None if the package was not found + """ # Cached result cache_key = "pmb.helpers.package.get" if ( @@ -127,12 +130,14 @@ def get(args, pkgname, arch, replace_subpkgnames=False, must_exist=True): def depends_recurse(args, pkgname, arch): - """ Recursively resolve all of the package's dependencies. - :param pkgname: name of the package (e.g. "device-samsung-i9100") - :param arch: preferred architecture for binary packages - :returns: a list of pkgname_start and all its dependencies, e.g: - ["busybox-static-armhf", "device-samsung-i9100", - "linux-samsung-i9100", ...] """ + """Recursively resolve all of the package's dependencies. + + :param pkgname: name of the package (e.g. "device-samsung-i9100") + :param arch: preferred architecture for binary packages + :returns: a list of pkgname_start and all its dependencies, e.g: + ["busybox-static-armhf", "device-samsung-i9100", + "linux-samsung-i9100", ...] + """ # Cached result cache_key = "pmb.helpers.package.depends_recurse" if (arch in pmb.helpers.other.cache[cache_key] and @@ -164,15 +169,14 @@ def depends_recurse(args, pkgname, arch): def check_arch(args, pkgname, arch, binary=True): - """ Can a package be built for a certain architecture, or is there a binary - package for it? + """Check if a package be built for a certain architecture, or is there a binary package for it. - :param pkgname: name of the package - :param arch: architecture to check against - :param binary: set to False to only look at the pmaports, not at binary - packages - :returns: True when the package can be built, or there is a binary - package, False otherwise + :param pkgname: name of the package + :param arch: architecture to check against + :param binary: set to False to only look at the pmaports, not at binary + packages + + :returns: True when the package can be built, or there is a binary package, False otherwise """ if binary: arches = get(args, pkgname, arch)["arch"] diff --git a/pmb/helpers/pkgrel_bump.py b/pmb/helpers/pkgrel_bump.py index 982cc2fc..17bc4fbe 100644 --- a/pmb/helpers/pkgrel_bump.py +++ b/pmb/helpers/pkgrel_bump.py @@ -9,8 +9,7 @@ import pmb.parse def package(args, pkgname, reason="", dry=False): - """ - Increase the pkgrel in the APKBUILD of a specific package. + """Increase the pkgrel in the APKBUILD of a specific package. :param pkgname: name of the package :param reason: string to display as reason why it was increased @@ -44,9 +43,7 @@ def package(args, pkgname, reason="", dry=False): def auto_apkindex_package(args, arch, aport, apk, dry=False): - """ - Bump the pkgrel of a specific package if it is outdated in the given - APKINDEX. + """Bump the pkgrel of a specific package if it is outdated in the given APKINDEX. :param arch: the architecture, e.g. "armhf" :param aport: parsed APKBUILD of the binary package's origin: @@ -103,9 +100,7 @@ def auto_apkindex_package(args, arch, aport, apk, dry=False): def auto(args, dry=False): - """ - :returns: list of aport names, where the pkgrel needed to be changed - """ + """:returns: list of aport names, where the pkgrel needed to be changed""" ret = [] for arch in pmb.config.build_device_architectures: paths = pmb.helpers.repo.apkindex_files(args, arch, alpine=False) diff --git a/pmb/helpers/pmaports.py b/pmb/helpers/pmaports.py index 6d5535bf..e3f4cfcb 100644 --- a/pmb/helpers/pmaports.py +++ b/pmb/helpers/pmaports.py @@ -1,7 +1,8 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later -""" -Functions that work with pmaports. See also: +"""Functions that work with pmaports. + +See also: - pmb/helpers/repo.py (work with binary package repos) - pmb/helpers/package.py (work with both) """ @@ -42,10 +43,9 @@ def get_list(args): def guess_main_dev(args, subpkgname): - """ - Check if a package without "-dev" at the end exists in pmaports or not, and - log the appropriate message. Don't call this function directly, use - guess_main() instead. + """Check if a package without "-dev" at the end exists in pmaports or not, and log the appropriate message. + + Don't call this function directly, use guess_main() instead. :param subpkgname: subpackage name, must end in "-dev" :returns: full path to the pmaport or None @@ -64,8 +64,8 @@ def guess_main_dev(args, subpkgname): def guess_main(args, subpkgname): - """ - Find the main package by assuming it is a prefix of the subpkgname. + """Find the main package by assuming it is a prefix of the subpkgname. + We do that, because in some APKBUILDs the subpkgname="" variable gets filled with a shell loop and the APKBUILD parser in pmbootstrap can't parse this right. (Intentionally, we don't want to implement a full shell @@ -101,9 +101,8 @@ def guess_main(args, subpkgname): def _find_package_in_apkbuild(package, path): - """ - Look through subpackages and all provides to see if the APKBUILD at the - specified path contains (or provides) the specified package. + """Look through subpackages and all provides to see if the APKBUILD at the specified path + contains (or provides) the specified package. :param package: The package to search for :param path: The path to the apkbuild @@ -136,8 +135,8 @@ def _find_package_in_apkbuild(package, path): def find(args, package, must_exist=True): - """ - Find the aport path that provides a certain subpackage. + """Find the aport path that provides a certain subpackage. + If you want the parsed APKBUILD instead, use pmb.helpers.pmaports.get(). :param must_exist: Raise an exception, when not found @@ -191,22 +190,23 @@ def find(args, package, must_exist=True): def get(args, pkgname, must_exist=True, subpackages=True): - """ Find and parse an APKBUILD file. - Run 'pmbootstrap apkbuild_parse hello-world' for a full output example. - Relevant variables are defined in pmb.config.apkbuild_attributes. + """Find and parse an APKBUILD file. - :param pkgname: the package name to find - :param must_exist: raise an exception when it can't be found - :param subpackages: also search for subpackages with the specified - names (slow! might need to parse all APKBUILDs to - find it) - :returns: relevant variables from the APKBUILD as dictionary, e.g.: + Run 'pmbootstrap apkbuild_parse hello-world' for a full output example. + Relevant variables are defined in pmb.config.apkbuild_attributes. + + :param pkgname: the package name to find + :param must_exist: raise an exception when it can't be found + :param subpackages: also search for subpackages with the specified + names (slow! might need to parse all APKBUILDs to find it) + + :returns: relevant variables from the APKBUILD as dictionary, e.g.: { "pkgname": "hello-world", - "arch": ["all"], - "pkgrel": "4", - "pkgrel": "1", - "options": [], - ... } + "arch": ["all"], + "pkgrel": "4", + "pkgrel": "1", + "options": [], + ... } """ pkgname = pmb.helpers.package.remove_operators(pkgname) if subpackages: @@ -225,8 +225,8 @@ def get(args, pkgname, must_exist=True, subpackages=True): def find_providers(args, provide): - """ - Search for providers of the specified (virtual) package in pmaports. + """Search for providers of the specified (virtual) package in pmaports. + Note: Currently only providers from a single APKBUILD are returned. :param provide: the (virtual) package to search providers for @@ -249,12 +249,13 @@ def find_providers(args, provide): def get_repo(args, pkgname, must_exist=True): - """ Get the repository folder of an aport. + """Get the repository folder of an aport. - :pkgname: package name - :must_exist: raise an exception when it can't be found - :returns: a string like "main", "device", "cross", ... - or None when the aport could not be found """ + :pkgname: package name + :must_exist: raise an exception when it can't be found + :returns: a string like "main", "device", "cross", ... + or None when the aport could not be found + """ aport = find(args, pkgname, must_exist) if not aport: return None @@ -262,13 +263,15 @@ def get_repo(args, pkgname, must_exist=True): def check_arches(arches, arch): - """ Check if building for a certain arch is allowed. + """Check if building for a certain arch is allowed. - :param arches: list of all supported arches, as it can be found in the - arch="" line of APKBUILDS (including all, noarch, - !arch, ...). For example: ["x86_64", "x86", "!armhf"] - :param arch: the architecture to check for - :returns: True when building is allowed, False otherwise + :param arches: list of all supported arches, as it can be found in the + arch="" line of APKBUILDS (including all, noarch, !arch, ...). + For example: ["x86_64", "x86", "!armhf"] + + :param arch: the architecture to check for + + :returns: True when building is allowed, False otherwise """ if "!" + arch in arches: return False @@ -279,12 +282,13 @@ def check_arches(arches, arch): def get_channel_new(channel): - """ Translate legacy channel names to the new ones. Legacy names are still - supported for compatibility with old branches (pmb#2015). - :param channel: name as read from pmaports.cfg or channels.cfg, like - "edge", "v21.03" etc., or potentially a legacy name - like "stable". - :returns: name in the new format, e.g. "edge" or "v21.03" + """Translate legacy channel names to the new ones. + + Legacy names are still supported for compatibility with old branches (pmb#2015). + :param channel: name as read from pmaports.cfg or channels.cfg, like "edge", "v21.03" etc., + or potentially a legacy name like "stable". + + :returns: name in the new format, e.g. "edge" or "v21.03" """ legacy_cfg = pmb.config.pmaports_channels_legacy if channel in legacy_cfg: diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 76d09be8..4773d498 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -1,7 +1,9 @@ # Copyright 2023 Oliver Smith # SPDX-License-Identifier: GPL-3.0-or-later """ -Functions that work with binary package repos. See also: +Functions that work with binary package repos. + +See also: - pmb/helpers/pmaports.py (work with pmaports) - pmb/helpers/package.py (work with both) """ @@ -14,9 +16,9 @@ import pmb.helpers.run def hash(url, length=8): - """ - Generate the hash that APK adds to the APKINDEX and apk packages - in its apk cache folder. It is the "12345678" part in this example: + r"""Generate the hash that APK adds to the APKINDEX and apk packages in its apk cache folder. + + It is the "12345678" part in this example: "APKINDEX.12345678.tar.gz". :param length: The length of the hash in the output file. @@ -24,7 +26,7 @@ def hash(url, length=8): See also: official implementation in apk-tools: - blob.c: apk_blob_push_hexdump(), "const char *xd" + blob.c: apk_blob_push_hexdump(), "const char \\*xd" apk_defines.h: APK_CACHE_CSUM_BYTES database.c: apk_repo_format_cache_index() """ @@ -41,8 +43,8 @@ def hash(url, length=8): def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True): - """ - Get a list of repository URLs, as they are in /etc/apk/repositories. + """Get a list of repository URLs, as they are in /etc/apk/repositories. + :param user_repository: add /mnt/pmbootstrap/packages :param postmarketos_mirror: add postmarketos mirror URLs :param alpine: add alpine mirror URLs @@ -86,9 +88,8 @@ def urls(args, user_repository=True, postmarketos_mirror=True, alpine=True): def apkindex_files(args, arch=None, user_repository=True, pmos=True, alpine=True): - """ - Get a list of outside paths to all resolved APKINDEX.tar.gz files for a - specific arch. + """Get a list of outside paths to all resolved APKINDEX.tar.gz files for a specific arch. + :param arch: defaults to native :param user_repository: add path to index of locally built packages :param pmos: add paths to indexes of postmarketos mirrors @@ -113,8 +114,7 @@ def apkindex_files(args, arch=None, user_repository=True, pmos=True, def update(args, arch=None, force=False, existing_only=False): - """ - Download the APKINDEX files for all URLs depending on the architectures. + """Download the APKINDEX files for all URLs depending on the architectures. :param arch: * one Alpine architecture name ("x86_64", "armhf", ...) * None for all architectures @@ -196,9 +196,7 @@ def update(args, arch=None, force=False, existing_only=False): def alpine_apkindex_path(args, repo="main", arch=None): - """ - Get the path to a specific Alpine APKINDEX file on disk and download it if - necessary. + """Get the path to a specific Alpine APKINDEX file on disk and download it if necessary. :param repo: Alpine repository name (e.g. "main") :param arch: Alpine architecture (e.g. "armhf"), defaults to native arch. diff --git a/pmb/helpers/repo_missing.py b/pmb/helpers/repo_missing.py index 3e3511ca..ae1b657f 100644 --- a/pmb/helpers/repo_missing.py +++ b/pmb/helpers/repo_missing.py @@ -8,11 +8,12 @@ import pmb.helpers.pmaports def filter_missing_packages(args, arch, pkgnames): - """ Create a subset of pkgnames with missing or outdated binary packages. + """Create a subset of pkgnames with missing or outdated binary packages. - :param arch: architecture (e.g. "armhf") - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) """ + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) + """ ret = [] for pkgname in pkgnames: binary = pmb.parse.apkindex.package(args, pkgname, arch, False) @@ -24,11 +25,12 @@ def filter_missing_packages(args, arch, pkgnames): def filter_aport_packages(args, arch, pkgnames): - """ Create a subset of pkgnames where each one has an aport. + """Create a subset of pkgnames where each one has an aport. - :param arch: architecture (e.g. "armhf") - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) """ + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) + """ ret = [] for pkgname in pkgnames: if pmb.helpers.pmaports.find(args, pkgname, False): @@ -37,12 +39,12 @@ def filter_aport_packages(args, arch, pkgnames): def filter_arch_packages(args, arch, pkgnames): - """ Create a subset of pkgnames with packages removed that can not be - built for a certain arch. + """Create a subset of pkgnames with packages removed that can not be built for a certain arch. - :param arch: architecture (e.g. "armhf") - :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) - :returns: subset of pkgnames (e.g. ["hello-world"]) """ + :param arch: architecture (e.g. "armhf") + :param pkgnames: list of package names (e.g. ["hello-world", "test12"]) + :returns: subset of pkgnames (e.g. ["hello-world"]) + """ ret = [] for pkgname in pkgnames: if pmb.helpers.package.check_arch(args, pkgname, arch, False): @@ -51,13 +53,14 @@ def filter_arch_packages(args, arch, pkgnames): def get_relevant_packages(args, arch, pkgname=None, built=False): - """ Get all packages that can be built for the architecture in question. + """Get all packages that can be built for the architecture in question. - :param arch: architecture (e.g. "armhf") - :param pkgname: only look at a specific package (and its dependencies) - :param built: include packages that have already been built - :returns: an alphabetically sorted list of pkgnames, e.g.: - ["devicepkg-dev", "hello-world", "unl0kr"] """ + :param arch: architecture (e.g. "armhf") + :param pkgname: only look at a specific package (and its dependencies) + :param built: include packages that have already been built + :returns: an alphabetically sorted list of pkgnames, e.g.: + ["devicepkg-dev", "hello-world", "osk-sdl"] + """ if pkgname: if not pmb.helpers.package.check_arch(args, pkgname, arch, False): raise RuntimeError(pkgname + " can't be built for " + arch + ".") @@ -84,19 +87,21 @@ def get_relevant_packages(args, arch, pkgname=None, built=False): def generate_output_format(args, arch, pkgnames): - """ Generate the detailed output format. - :param arch: architecture - :param pkgnames: list of package names that should be in the output, - e.g.: ["hello-world", "pkg-depending-on-hello-world"] + """Generate the detailed output format. + + :param arch: architecture + :param pkgnames: list of package names that should be in the output, + e.g.: ["hello-world", "pkg-depending-on-hello-world"] :returns: a list like the following: - [{"pkgname": "hello-world", - "repo": "main", - "version": "1-r4", - "depends": []}, - {"pkgname": "pkg-depending-on-hello-world", - "version": "0.5-r0", - "repo": "main", - "depends": ["hello-world"]}] """ + [{"pkgname": "hello-world", + "repo": "main", + "version": "1-r4", + "depends": []}, + {"pkgname": "pkg-depending-on-hello-world", + "version": "0.5-r0", + "repo": "main", + "depends": ["hello-world"]}] + """ ret = [] for pkgname in pkgnames: entry = pmb.helpers.package.get(args, pkgname, arch, True) @@ -108,18 +113,14 @@ def generate_output_format(args, arch, pkgnames): def generate(args, arch, overview, pkgname=None, built=False): - """ Get packages that need to be built, with all their dependencies. + """Get packages that need to be built, with all their dependencies. - :param arch: architecture (e.g. "armhf") - :param pkgname: only look at a specific package - :param built: include packages that have already been built - :returns: a list like the following: - [{"pkgname": "hello-world", - "repo": "main", - "version": "1-r4"}, - {"pkgname": "package-depending-on-hello-world", - "version": "0.5-r0", - "repo": "main"}] + :param arch: architecture (e.g. "armhf") + :param pkgname: only look at a specific package + :param built: include packages that have already been built + :returns: a list like the following: + [{"pkgname": "hello-world", "repo": "main", "version": "1-r4"}, + {"pkgname": "package-depending-on-hello-world", "version": "0.5-r0", "repo": "main"}] """ # Log message packages_str = pkgname if pkgname else "all packages" diff --git a/pmb/helpers/run.py b/pmb/helpers/run.py index 16644056..1bf22e5f 100644 --- a/pmb/helpers/run.py +++ b/pmb/helpers/run.py @@ -35,8 +35,7 @@ def user(args: Namespace, cmd: List[str], working_dir: Optional[str]=None, outpu def root(args, cmd, working_dir=None, output="log", output_return=False, check=None, env={}): - """ - Run a command on the host system as root, with sudo or doas. + """Run a command on the host system as root, with sudo or doas. :param env: dict of environment variables to be passed to the command, e.g. {"JOBS": "5"} diff --git a/pmb/helpers/run_core.py b/pmb/helpers/run_core.py index 5aa17632..7a9a68a9 100644 --- a/pmb/helpers/run_core.py +++ b/pmb/helpers/run_core.py @@ -11,15 +11,13 @@ import threading import time import pmb.helpers.run -""" For a detailed description of all output modes, read the description of - core() at the bottom. All other functions in this file get (indirectly) - called by core(). """ +"""For a detailed description of all output modes, read the description of + core() at the bottom. All other functions in this file get (indirectly) + called by core(). """ def flat_cmd(cmd, working_dir=None, env={}): - """ - Convert a shell command passed as list into a flat shell string with - proper escaping. + """Convert a shell command passed as list into a flat shell string with proper escaping. :param cmd: command as list, e.g. ["echo", "string with spaces"] :param working_dir: when set, prepend "cd ...;" to execute the command @@ -46,8 +44,8 @@ def flat_cmd(cmd, working_dir=None, env={}): def sanity_checks(output="log", output_return=False, check=None): - """ - Raise an exception if the parameters passed to core() don't make sense + """Raise an exception if the parameters passed to core() don't make sense. + (all parameters are described in core() below). """ vals = ["log", "stdout", "interactive", "tui", "background", "pipe"] @@ -66,7 +64,7 @@ def sanity_checks(output="log", output_return=False, check=None): def background(cmd, working_dir=None): - """ Run a subprocess in background and redirect its output to the log. """ + """Run a subprocess in background and redirect its output to the log.""" ret = subprocess.Popen(cmd, stdout=pmb.helpers.logging.logfd, stderr=pmb.helpers.logging.logfd, cwd=working_dir) logging.debug(f"New background process: pid={ret.pid}, output=background") @@ -74,7 +72,7 @@ def background(cmd, working_dir=None): def pipe(cmd, working_dir=None): - """ Run a subprocess in background and redirect its output to a pipe. """ + """Run a subprocess in background and redirect its output to a pipe.""" ret = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=pmb.helpers.logging.logfd, cwd=working_dir) @@ -84,10 +82,9 @@ def pipe(cmd, working_dir=None): def pipe_read(process, output_to_stdout=False, output_return=False, output_return_buffer=False): - """ - Read all available output from a subprocess and copy it to the log and - optionally stdout and a buffer variable. This is only meant to be called by - foreground_pipe() below. + """Read all output from a subprocess, copy it to the log and optionally stdout and a buffer variable. + + This is only meant to be called by foreground_pipe() below. :param process: subprocess.Popen instance :param output_to_stdout: copy all output to pmbootstrap's stdout @@ -115,8 +112,7 @@ def pipe_read(process, output_to_stdout=False, output_return=False, def kill_process_tree(args, pid, ppids, sudo): - """ - Recursively kill a pid and its child processes + """Recursively kill a pid and its child processes. :param pid: process id that will be killed :param ppids: list of process id and parent process id tuples (pid, ppid) @@ -135,8 +131,7 @@ def kill_process_tree(args, pid, ppids, sudo): def kill_command(args, pid, sudo): - """ - Kill a command process and recursively kill its child processes + """Kill a command process and recursively kill its child processes. :param pid: process id that will be killed :param sudo: use sudo to kill the process @@ -157,9 +152,9 @@ def kill_command(args, pid, sudo): def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False, output_return=False, output_timeout=True, sudo=False, stdin=None): - """ - Run a subprocess in foreground with redirected output and optionally kill - it after being silent for too long. + """Run a subprocess in foreground with redirected output. + + Optionally kill it after being silent for too long. :param cmd: command as list, e.g. ["echo", "string with spaces"] :param working_dir: path in host system where the command should run @@ -220,13 +215,11 @@ def foreground_pipe(args, cmd, working_dir=None, output_to_stdout=False, def foreground_tui(cmd, working_dir=None): - """ - Run a subprocess in foreground without redirecting any of its output. + """Run a subprocess in foreground without redirecting any of its output. This is the only way text-based user interfaces (ncurses programs like vim, nano or the kernel's menuconfig) work properly. """ - logging.debug("*** output passed to pmbootstrap stdout, not to this log" " ***") process = subprocess.Popen(cmd, cwd=working_dir) @@ -234,8 +227,7 @@ def foreground_tui(cmd, working_dir=None): def check_return_code(args, code, log_message): - """ - Check the return code of a command. + """Check the return code of a command. :param code: exit code to check :param log_message: simplified and more readable form of the command, e.g. @@ -243,7 +235,6 @@ def check_return_code(args, code, log_message): entering the chroot and more escaping :raises RuntimeError: when the code indicates that the command failed """ - if code: logging.debug("^" * 70) logging.info("NOTE: The failed command's output is above the ^^^ line" @@ -253,10 +244,7 @@ def check_return_code(args, code, log_message): def sudo_timer_iterate(): - """ - Run sudo -v and schedule a new timer to repeat the same. - """ - + """Run sudo -v and schedule a new timer to repeat the same.""" if pmb.config.which_sudo() == "sudo": subprocess.Popen(["sudo", "-v"]).wait() else: @@ -268,11 +256,7 @@ def sudo_timer_iterate(): def sudo_timer_start(): - """ - Start a timer to call sudo -v periodically, so that the password is only - needed once. - """ - + """Start a timer to call sudo -v periodically, so that the password is only needed once.""" if "sudo_timer_active" in pmb.helpers.other.cache: return pmb.helpers.other.cache["sudo_timer_active"] = True @@ -281,11 +265,10 @@ def sudo_timer_start(): def add_proxy_env_vars(env): - """ - Add proxy environment variables present on the host to the environment of - the command we are running. - :param env: dict of environment variables, it will be extended with all of - the proxy env vars that are set on the host + """Add proxy environment variables from host to the environment of the command we are running. + + :param env: dict of environment variables, it will be extended with all of the proxy env vars + that are set on the host """ proxy_env_vars = [ "FTP_PROXY", @@ -304,12 +287,11 @@ def add_proxy_env_vars(env): def core(args, log_message, cmd, working_dir=None, output="log", output_return=False, check=None, sudo=False, disable_timeout=False): - """ - Run a command and create a log entry. + """Run a command and create a log entry. This is a low level function not meant to be used directly. Use one of the following instead: pmb.helpers.run.user(), pmb.helpers.run.root(), - pmb.chroot.user(), pmb.chroot.root() + pmb.chroot.user(), pmb.chroot.root() :param log_message: simplified and more readable form of the command, e.g. "(native) % echo test" instead of the full command with @@ -337,24 +319,23 @@ def core(args, log_message, cmd, working_dir=None, output="log", their properties. "wait" indicates that we wait for the process to complete. - output value | timeout | out to log | out to stdout | wait | pass stdin - ------------------------------------------------------------------------ - "log" | x | x | | x | - "stdout" | x | x | x | x | - "interactive" | | x | x | x | x - "tui" | | | x | x | x - "background" | | x | | | - "pipe" | | | | | + ============= ======= ========== ============= ==== ========== + output value timeout out to log out to stdout wait pass stdin + ============= ======= ========== ============= ==== ========== + "log" x x x + "stdout" x x x x + "interactive" x x x x + "tui" x x x + "background" x + "pipe" + ============= ======= ========== ============= ==== ========== :param output_return: in addition to writing the program's output to the - destinations above in real time, write to a buffer - and return it as string when the command has - completed. This is not possible when output is - "background", "pipe" or "tui". - :param check: an exception will be raised when the command's return code - is not 0. Set this to False to disable the check. This - parameter can not be used when the output is "background" or - "pipe". + destinations above in real time, write to a buffer and return it as string when the + command has completed. This is not possible when output is "background", "pipe" or "tui". + :param check: an exception will be raised when the command's return code is not 0. + Set this to False to disable the check. This parameter can not be used when the output is + "background" or "pipe". :param sudo: use sudo to kill the process when it hits the timeout. :returns: * program's return code (default) * subprocess.Popen instance (output is "background" or "pipe") diff --git a/pmb/helpers/ui.py b/pmb/helpers/ui.py index 7c618f17..c7d5cf8b 100644 --- a/pmb/helpers/ui.py +++ b/pmb/helpers/ui.py @@ -7,8 +7,7 @@ import pmb.parse def list(args, arch): - """ - Get all UIs, for which aports are available with their description. + """Get all UIs, for which aports are available with their description. :param arch: device architecture, for which the UIs must be available :returns: [("none", "No graphical..."), ("weston", "Wayland reference...")] diff --git a/pmb/parse/apkindex.py b/pmb/parse/apkindex.py index 3828c268..c62af095 100644 --- a/pmb/parse/apkindex.py +++ b/pmb/parse/apkindex.py @@ -11,30 +11,27 @@ import pmb.parse.version def parse_next_block(path, lines, start): - """ - Parse the next block in an APKINDEX. + """Parse the next block in an APKINDEX. :param path: to the APKINDEX.tar.gz :param start: current index in lines, gets increased in this function. Wrapped into a list, so it can be modified "by reference". Example: [5] :param lines: all lines from the "APKINDEX" file inside the archive - :returns: a dictionary with the following structure: - { "arch": "noarch", - "depends": ["busybox-extras", "lddtree", ... ], - "origin": "postmarketos-mkinitfs", - "pkgname": "postmarketos-mkinitfs", - "provides": ["mkinitfs=0.0.1"], - "timestamp": "1500000000", - "version": "0.0.4-r10" } - NOTE: "depends" is not set for packages without any dependencies, - e.g. musl. - NOTE: "timestamp" and "origin" are not set for virtual packages - (#1273). We use that information to skip these virtual - packages in parse(). + :returns: Dictionary with the following structure: + ``{ "arch": "noarch", "depends": ["busybox-extras", "lddtree", ... ], + "origin": "postmarketos-mkinitfs", + "pkgname": "postmarketos-mkinitfs", + "provides": ["mkinitfs=0.0.1"], + "timestamp": "1500000000", + "version": "0.0.4-r10" }`` + + NOTE: "depends" is not set for packages without any dependencies, e.g. ``musl``. + + NOTE: "timestamp" and "origin" are not set for virtual packages (#1273). + We use that information to skip these virtual packages in parse(). :returns: None, when there are no more blocks """ - # Parse until we hit an empty line or end of file ret = {} mapping = { @@ -100,8 +97,7 @@ def parse_next_block(path, lines, start): def parse_add_block(ret, block, alias=None, multiple_providers=True): - """ - Add one block to the return dictionary of parse(). + """Add one block to the return dictionary of parse(). :param ret: dictionary of all packages in the APKINDEX that is getting built right now. This function will extend it. @@ -113,7 +109,6 @@ def parse_add_block(ret, block, alias=None, multiple_providers=True): APKINDEX files from a repository (#1122), but not when parsing apk's installed packages DB. """ - # Defaults pkgname = block["pkgname"] alias = alias or pkgname @@ -142,8 +137,7 @@ def parse_add_block(ret, block, alias=None, multiple_providers=True): def parse(path, multiple_providers=True): - """ - Parse an APKINDEX.tar.gz file, and return its content as dictionary. + r"""Parse an APKINDEX.tar.gz file, and return its content as dictionary. :param path: path to an APKINDEX.tar.gz file or apk package database (almost the same format, but not compressed). @@ -152,22 +146,23 @@ def parse(path, multiple_providers=True): APKINDEX files from a repository (#1122), but not when parsing apk's installed packages DB. :returns: (without multiple_providers) - generic format: - { pkgname: block, ... } + + Generic format: + ``{ pkgname: block, ... }`` - example: - { "postmarketos-mkinitfs": block, - "so:libGL.so.1": block, ...} + Example: + ``{ "postmarketos-mkinitfs": block, "so:libGL.so.1": block, ...}`` :returns: (with multiple_providers) - generic format: - { provide: { pkgname: block, ... }, ... } - example: - { "postmarketos-mkinitfs": {"postmarketos-mkinitfs": block}, - "so:libGL.so.1": {"mesa-egl": block, "libhybris": block}, ...} + Generic format: + ``{ provide: { pkgname: block, ... }, ... }`` + + Example: + ``{ "postmarketos-mkinitfs": {"postmarketos-mkinitfs": block},"so:libGL.so.1": {"mesa-egl": block, "libhybris": block}, ...}`` + + *NOTE:* ``block`` is the return value from ``parse_next_block()`` above. - NOTE: "block" is the return value from parse_next_block() above. """ # Require the file to exist if not os.path.isfile(path): @@ -230,7 +225,7 @@ def parse_blocks(path): :returns: all blocks in the APKINDEX, without restructuring them by pkgname or removing duplicates with lower versions (use parse() if you need these features). Structure: - [block, block, ...] + ``[block, block, ...]`` NOTE: "block" is the return value from parse_next_block() above. """ @@ -276,10 +271,9 @@ def providers(args, package, arch=None, must_exist=True, indexes=None): :param indexes: list of APKINDEX.tar.gz paths, defaults to all index files (depending on arch) :returns: list of parsed packages. Example for package="so:libGL.so.1": - {"mesa-egl": block, "libhybris": block} - block is the return value from parse_next_block() above. + ``{"mesa-egl": block, "libhybris": block}`` + block is the return value from parse_next_block() above. """ - if not indexes: arch = arch or pmb.config.arch_native indexes = pmb.helpers.repo.apkindex_files(args, arch) @@ -319,8 +313,7 @@ def providers(args, package, arch=None, must_exist=True, indexes=None): def provider_highest_priority(providers, pkgname): - """ - Get the provider(s) with the highest provider_priority and log a message. + """Get the provider(s) with the highest provider_priority and log a message. :param providers: returned dict from providers(), must not be empty :param pkgname: the package name we are interested in (for the log message) @@ -346,8 +339,7 @@ def provider_highest_priority(providers, pkgname): def provider_shortest(providers, pkgname): - """ - Get the provider with the shortest pkgname and log a message. In most cases + """Get the provider with the shortest pkgname and log a message. In most cases this should be sufficient, e.g. 'mesa-purism-gc7000-egl, mesa-egl' or 'gtk+2.0-maemo, gtk+2.0'. @@ -374,10 +366,10 @@ def package(args, package, arch=None, must_exist=True, indexes=None): (depending on arch) :returns: a dictionary with the following structure: { "arch": "noarch", - "depends": ["busybox-extras", "lddtree", ... ], - "pkgname": "postmarketos-mkinitfs", - "provides": ["mkinitfs=0.0.1"], - "version": "0.0.4-r10" } + "depends": ["busybox-extras", "lddtree", ... ], + "pkgname": "postmarketos-mkinitfs", + "provides": ["mkinitfs=0.0.1"], + "version": "0.0.4-r10" } or None when the package was not found. """ # Provider with the same package diff --git a/pmb/parse/arguments.py b/pmb/parse/arguments.py index 642ce798..ab9f5744 100644 --- a/pmb/parse/arguments.py +++ b/pmb/parse/arguments.py @@ -15,23 +15,27 @@ import pmb.parse.arch import pmb.helpers.args import pmb.helpers.pmaports -""" This file is about parsing command line arguments passed to pmbootstrap, as - well as generating the help pages (pmbootstrap -h). All this is done with - Python's argparse. The parsed arguments get extended and finally stored in - the "args" variable, which is prominently passed to most functions all - over the pmbootstrap code base. +"""This file is about parsing command line arguments passed to pmbootstrap, as + well as generating the help pages (pmbootstrap -h). All this is done with + Python's argparse. The parsed arguments get extended and finally stored in + the "args" variable, which is prominently passed to most functions all + over the pmbootstrap code base. - See pmb/helpers/args.py for more information about the args variable. """ + See pmb/helpers/args.py for more information about the args variable. +""" def toggle_other_boolean_flags(*other_destinations, value=True): - """ Helper function to group several argparse flags to one. Sets multiple - other_destination to value. + """Group several argparse flags to one. - :param other_destinations: 'the other argument names' str - :param value 'the value to set the other_destinations to' bool - :returns custom Action""" + Sets multiple other_destination to value. + :param other_destinations: 'the other argument names' str + + :param value 'the value to set the other_destinations to' bool + + :returns custom Action + """ class SetOtherDestinationsAction(argparse.Action): def __init__(self, option_strings, dest, **kwargs): super().__init__(option_strings, dest, nargs=0, const=value, @@ -45,10 +49,12 @@ def toggle_other_boolean_flags(*other_destinations, value=True): def type_ondev_cp(val): - """ Parse and validate arguments to 'pmbootstrap install --ondev --cp'. + """Parse and validate arguments to 'pmbootstrap install --ondev --cp'. - :param val: 'HOST_SRC:CHROOT_DEST' string - :returns: (HOST_SRC, CHROOT_DEST) """ + :param val: 'HOST_SRC:CHROOT_DEST' string + + :returns: (HOST_SRC, CHROOT_DEST) + """ ret = val.split(":") if len(ret) != 2: diff --git a/pmb/parse/bootimg.py b/pmb/parse/bootimg.py index 9212f8bd..36c2db3a 100644 --- a/pmb/parse/bootimg.py +++ b/pmb/parse/bootimg.py @@ -18,7 +18,7 @@ def get_mtk_label(path): an extracted boot.img. :param path: to either the kernel or ramdisk extracted from boot.img :returns: * None: file does not exist or does not have MediaTek header - * Label string (e.g. "ROOTFS", "KERNEL") """ + * Label string (e.g. "ROOTFS", "KERNEL") """ if not os.path.exists(path): return None @@ -48,7 +48,8 @@ def get_qcdt_type(path): """ Get the dt.img type by reading the first four bytes of the file. :param path: to the qcdt image extracted from boot.img :returns: * None: dt.img is of unknown type - * Type string (e.g. "qcom", "sprd", "exynos") """ + * Type string (e.g. "qcom", "sprd", "exynos") + """ if not os.path.exists(path): return None diff --git a/pmb/parse/kconfig.py b/pmb/parse/kconfig.py index d2e9048e..1bb0d7cc 100644 --- a/pmb/parse/kconfig.py +++ b/pmb/parse/kconfig.py @@ -121,6 +121,7 @@ def check_config_options_set(config, config_path, config_arch, options, component, pkgver, details=False): """ Check, whether all the kernel config passes all rules of one component. + Print a warning if any is missing. :param config: full kernel config as string @@ -128,10 +129,10 @@ def check_config_options_set(config, config_path, config_arch, options, :param config_arch: architecture name (alpine format, e.g. aarch64, x86_64) :param options: kconfig_options* var passed from pmb/config/__init__.py: kconfig_options_example = { - ">=0.0.0": { # all versions - "all": { # all arches - "ANDROID_PARANOID_NETWORK": False, - }, + ">=0.0.0": { # all versions + "all": { # all arches + "ANDROID_PARANOID_NETWORK": False, + }, } :param component: name of the component to test (postmarketOS, waydroid, …) :param pkgver: kernel version diff --git a/pmb/parse/version.py b/pmb/parse/version.py index ae2e84c6..1f8d5968 100644 --- a/pmb/parse/version.py +++ b/pmb/parse/version.py @@ -101,9 +101,9 @@ def parse_suffix(rest): :returns: (rest, value, invalid_suffix) - rest: is the input "rest" string without the suffix - value: is a signed integer (negative for pre-, - positive for post-suffixes). + positive for post-suffixes). - invalid_suffix: is true, when rest does not start - with anything from the suffixes variable. + with anything from the suffixes variable. C equivalent: get_token(), case TOKEN_SUFFIX """ @@ -284,6 +284,7 @@ def check_string(a_version, rule): :param a_version: "3.4.1" :param rule: ">=1.0.0" :returns: True if a_version matches rule, false otherwise. + """ # Operators and the expected returns of compare(a,b) operator_results = {">=": [1, 0], diff --git a/pmb/qemu/run.py b/pmb/qemu/run.py index 6f8b999f..3c45cab4 100644 --- a/pmb/qemu/run.py +++ b/pmb/qemu/run.py @@ -35,7 +35,9 @@ def system_image(args): def create_second_storage(args): """ Generate a second storage image if it does not exist. + :returns: path to the image or None + """ path = f"{args.work}/chroot_native/home/pmos/rootfs/{args.device}-2nd.img" pmb.helpers.run.root(args, ["touch", path])