diff --git a/.build.yml b/.build.yml new file mode 100644 index 00000000..5b4e4de0 --- /dev/null +++ b/.build.yml @@ -0,0 +1,19 @@ +image: alpine/edge +packages: + - sudo +tasks: + - shellcheck: | + cd pmbootstrap + sudo .ci/shellcheck.sh + - flake8: | + cd pmbootstrap + sudo .ci/flake8.sh + - vermin: | + cd pmbootstrap + sudo .ci/vermin.sh + - pytest: | + cd pmbootstrap + sudo .ci/pytest.sh +artifacts: + - ".local/var/pmbootstrap/log.txt" + - ".local/var/pmbootstrap/log_testsuite.txt" diff --git a/.ci/flake8.sh b/.ci/flake8.sh new file mode 100755 index 00000000..560b70ae --- /dev/null +++ b/.ci/flake8.sh @@ -0,0 +1,25 @@ +#!/bin/sh -e +# Description: lint all python scripts +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + apk -q add py3-flake8 + exec su "${TESTUSER:-build}" -c "sh -e $0" +fi + +# E402: module import not on top of file, not possible for testcases +# E722: do not use bare except +# W504: line break occurred after a binary operator +# W604: invalid escape sequence +ign="E402,E722,W504,W605" + +set -x + +# __init__.py with additional ignore: +# F401: imported, but not used +# shellcheck disable=SC2046 +flake8 --ignore "F401,$ign" $(find . -not -path '*/venv/*' -name '__init__.py') + +# Check all other files +flake8 --ignore="$ign" --exclude=__init__.py diff --git a/.ci/prepare.sh b/.ci/prepare.sh deleted file mode 100755 index 92b62ee9..00000000 --- a/.ci/prepare.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -e -# Install pmbootstrap depends, set up pmos user with sudo -if [ "$(id -u)" != 0 ]; then - echo "ERROR: this script is meant to be executed in the gitlab-ci" - echo "environment only." - exit 1 -fi - -topdir="$(realpath "$(dirname "$0")/..")" -cd "$topdir" - -ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap - -apk add -q \ - git \ - openssl \ - py3-pytest \ - py3-pytest-cov \ - sudo - -adduser -D pmos -chown -R pmos:pmos . -echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers - -su pmos -c "git config --global user.email postmarketos-ci@localhost" -su pmos -c "git config --global user.name postmarketOS_CI" - -echo "Initializing pmbootstrap" -if ! su pmos -c "yes '' | pmbootstrap \ - --details-to-stdout \ - init \ - >/tmp/pmb_init 2>&1"; then - cat /tmp/pmb_init - exit 1 -fi -echo "" diff --git a/.ci/pytest.sh b/.ci/pytest.sh index 54e48fbb..1cad49cc 100755 --- a/.ci/pytest.sh +++ b/.ci/pytest.sh @@ -1,6 +1,39 @@ #!/bin/sh -e -topdir="$(realpath "$(dirname "$0")/..")" -cd "$topdir" +# Description: run pmbootstrap python testsuite +# Options: native slow +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + apk -q add \ + git \ + openssl \ + py3-pytest \ + py3-pytest-cov \ + sudo + exec su "${TESTUSER:-build}" -c "sh -e $0" +fi + +# Require pytest to be installed on the host system +if [ -z "$(command -v pytest)" ]; then + echo "ERROR: pytest command not found, make sure it is in your PATH." + exit 1 +fi + +# Use pytest-cov if it is installed to display code coverage +cov_arg="" +if python -c "import pytest_cov" >/dev/null 2>&1; then + cov_arg="--cov=pmb" +fi + +echo "Initializing pmbootstrap..." +if ! yes '' | ./pmbootstrap.py \ + --details-to-stdout \ + init \ + >/tmp/pmb_init 2>&1; then + cat /tmp/pmb_init + exit 1 +fi # Make sure that the work folder format is up to date, and that there are no # mounts from aborted test cases (#1595) @@ -21,4 +54,13 @@ if ! [ -e "$deviceinfo" ]; then exit 1 fi -pytest -vv -x --cov=pmb test -m "not skip_ci" "$@" +echo "Running pytest..." +echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally." +pytest \ + --color=yes \ + -vv \ + -x \ + $cov_arg \ + test \ + -m "not skip_ci" \ + "$@" diff --git a/.ci/shellcheck.sh b/.ci/shellcheck.sh new file mode 100755 index 00000000..dd21a570 --- /dev/null +++ b/.ci/shellcheck.sh @@ -0,0 +1,15 @@ +#!/bin/sh -e +# Description: lint all shell scripts +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + apk -q add shellcheck + exec su "${TESTUSER:-build}" -c "sh -e $0" +fi + +find . -name '*.sh' | +while read -r file; do + echo "shellcheck: $file" + shellcheck "$file" +done diff --git a/.ci/vermin.sh b/.ci/vermin.sh index 12025eeb..33c94dd9 100755 --- a/.ci/vermin.sh +++ b/.ci/vermin.sh @@ -1,17 +1,22 @@ #!/bin/sh -e -_vermin() { - if ! vermin -q "$@" >/dev/null 2>&1; then - vermin -vv "$@" - fi -} +# Description: verify that we don't use too new python features +# https://postmarktos.org/pmb-ci + +if [ "$(id -u)" = 0 ]; then + set -x + apk -q add vermin + exec su "${TESTUSER:-build}" -c "sh -e $0" +fi # shellcheck disable=SC2046 -_vermin \ +vermin \ -t=3.6- \ --backport argparse \ --backport configparser \ --backport enum \ --backport typing \ + --lint \ + --no-parse-comments \ $(find . -name '*.py' \ -a -not -path "./.venv/*" \ -a -not -path "./venv/*") diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 02e578cd..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,66 +0,0 @@ ---- -# Author: Clayton Craft - -image: alpine:latest - -# defaults for "only" -# We need to run the CI jobs in a "merge request specific context", if CI is -# running in a merge request. Otherwise the environment variable that holds the -# merge request ID is not available. This means, we must set the "only" -# variable accordingly - and if we only do it for one job, all other jobs will -# not get executed. So have the defaults here, and use them in all jobs that -# should run on both the master branch, and in merge requests. -# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs -.only-default: &only-default - only: - - master - - merge_requests - - tags - -static-code-analysis: - <<: *only-default - before_script: - - .ci/prepare.sh - script: - - su pmos -c "test/static_code_analysis.sh" - -vermin: - image: alpine:latest - <<: *only-default - before_script: - - "apk -q add py3-pip" - - "pip3 -q --disable-pip-version-check install vermin" - script: - - ".ci/vermin.sh" - -# MR settings -# (Checks for "Allow commits from members who can merge to the target branch") -mr-settings: - only: - - merge_requests - before_script: - - apk -q add python3 - script: - - wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py" - - python3 ./check_mr_settings.py - -pytest: - <<: *only-default - before_script: - - .ci/prepare.sh - script: - - su pmos -c .ci/pytest.sh - after_script: - # Move logs so it can be saved as artifacts - - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt" - - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt" - - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg" - - "dmesg > $CI_PROJECT_DIR/dmesg.txt" - artifacts: - when: always - paths: - - "log.txt" - - "log_testsuite.txt" - - "dmesg.txt" - - "pmbootstrap.cfg" - expire_in: 1 week diff --git a/test/static_code_analysis.sh b/test/static_code_analysis.sh deleted file mode 100755 index 438d83a7..00000000 --- a/test/static_code_analysis.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/sh -# Copyright 2022 Oliver Smith -# SPDX-License-Identifier: AGPL-3.0-or-later -set -e -DIR="$(cd "$(dirname "$0")" && pwd -P)" -cd "$DIR/.." - -# Make sure that the work folder format is up to date, and that there are no -# mounts from aborted test cases (#1595) -./pmbootstrap.py work_migrate -./pmbootstrap.py -q shutdown - -# Install needed packages -echo "Initializing Alpine chroot (details: 'pmbootstrap log')" -./pmbootstrap.py -q chroot -- apk -q add \ - shellcheck \ - python3 \ - py3-flake8 || return 1 - -rootfs_native="$(./pmbootstrap.py config work)/chroot_native" -command="$rootfs_native/lib/ld-musl-$(uname -m).so.1" -command="$command --library-path=$rootfs_native/lib:$rootfs_native/usr/lib" -shellcheck_command="$command $rootfs_native/usr/bin/shellcheck" -flake8_command="$command $rootfs_native/usr/bin/python3 $rootfs_native/usr/bin/flake8" - -# Shell: shellcheck -find . -name '*.sh' | -while read -r file; do - echo "Test with shellcheck: $file" - cd "$DIR/../$(dirname "$file")" - $shellcheck_command -e SC1008 -x "$(basename "$file")" -done - -# Python: flake8 -# F401: imported, but not used, does not make sense in __init__ files -# E402: module import not on top of file, not possible for testcases -# E722: do not use bare except -cd "$DIR"/.. -echo "Test with flake8: *.py" -# Note: omitting a virtualenv if it is here (e.g. gitlab CI) -py_files="$(find . -not -path '*/venv/*' -name '*.py')" -_ignores="E402,E722,W504,W605" -# shellcheck disable=SC2086 -$flake8_command --exclude=__init__.py --ignore "$_ignores" $py_files -# shellcheck disable=SC2086 -$flake8_command --filename=__init__.py --ignore "F401,$_ignores" $py_files - -# Done -echo "Success!"