From 4b8a0db5bcb73f167086c12d3533175d8a1d3b3b Mon Sep 17 00:00:00 2001
From: Oliver Smith <ollieparanoid@postmarketos.org>
Date: Tue, 18 Oct 2022 22:18:52 +0200
Subject: [PATCH] CI: replace gitlab CI with sourcehut CI

Replace .gitlab-ci.yml with a minimal .build.yml that runs a script in
the .ci dir for each task. Each of these scripts runs as root first,
installs dependencies, and then drops rights to the testuser and runs
the actual test. I went with this design, so we can add a
'pmbootstrap ci' command in following patches that run these in a
pmbootstrap chroot (see pmaports#2169).

Looking at flake8, we currently ignore W504 and W604. Would be nice to
fix these instead, but let's do that in another patch.

I've added a minver package to Alpine, so we don't need to install it
form pip anymore. Use minver's new --lint argument to simplify the
script, and as recommended by minver itself, use --no-parse-comments as
speed optimization since we don't have minver specific comments.

Using doas instead of sudo in build.yml would be nice, but this doesn't
work out of the box. Can be changed in the future.

Add a note to the pytest script, that 'pmbootstrap log' can be used to
follow the log while tests are running.
---
 .build.yml                   | 19 +++++++++++
 .ci/flake8.sh                | 25 ++++++++++++++
 .ci/prepare.sh               | 36 --------------------
 .ci/pytest.sh                | 48 ++++++++++++++++++++++++--
 .ci/shellcheck.sh            | 15 ++++++++
 .ci/vermin.sh                | 17 ++++++----
 .gitlab-ci.yml               | 66 ------------------------------------
 test/static_code_analysis.sh | 49 --------------------------
 8 files changed, 115 insertions(+), 160 deletions(-)
 create mode 100644 .build.yml
 create mode 100755 .ci/flake8.sh
 delete mode 100755 .ci/prepare.sh
 create mode 100755 .ci/shellcheck.sh
 delete mode 100644 .gitlab-ci.yml
 delete mode 100755 test/static_code_analysis.sh

diff --git a/.build.yml b/.build.yml
new file mode 100644
index 00000000..5b4e4de0
--- /dev/null
+++ b/.build.yml
@@ -0,0 +1,19 @@
+image: alpine/edge
+packages:
+  - sudo
+tasks:
+  - shellcheck: |
+      cd pmbootstrap
+      sudo .ci/shellcheck.sh
+  - flake8: |
+      cd pmbootstrap
+      sudo .ci/flake8.sh
+  - vermin: |
+      cd pmbootstrap
+      sudo .ci/vermin.sh
+  - pytest: |
+      cd pmbootstrap
+      sudo .ci/pytest.sh
+artifacts:
+  - ".local/var/pmbootstrap/log.txt"
+  - ".local/var/pmbootstrap/log_testsuite.txt"
diff --git a/.ci/flake8.sh b/.ci/flake8.sh
new file mode 100755
index 00000000..560b70ae
--- /dev/null
+++ b/.ci/flake8.sh
@@ -0,0 +1,25 @@
+#!/bin/sh -e
+# Description: lint all python scripts
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add py3-flake8
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+# E402: module import not on top of file, not possible for testcases
+# E722: do not use bare except
+# W504: line break occurred after a binary operator
+# W604: invalid escape sequence
+ign="E402,E722,W504,W605"
+
+set -x
+
+# __init__.py with additional ignore:
+# F401: imported, but not used
+# shellcheck disable=SC2046
+flake8 --ignore "F401,$ign" $(find . -not -path '*/venv/*' -name '__init__.py')
+
+# Check all other files
+flake8 --ignore="$ign" --exclude=__init__.py
diff --git a/.ci/prepare.sh b/.ci/prepare.sh
deleted file mode 100755
index 92b62ee9..00000000
--- a/.ci/prepare.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/sh -e
-# Install pmbootstrap depends, set up pmos user with sudo
-if [ "$(id -u)" != 0 ]; then
-	echo "ERROR: this script is meant to be executed in the gitlab-ci"
-	echo "environment only."
-	exit 1
-fi
-
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
-
-ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
-
-apk add -q \
-	git \
-	openssl \
-	py3-pytest \
-	py3-pytest-cov \
-	sudo
-
-adduser -D pmos
-chown -R pmos:pmos .
-echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
-
-su pmos -c "git config --global user.email postmarketos-ci@localhost"
-su pmos -c "git config --global user.name postmarketOS_CI"
-
-echo "Initializing pmbootstrap"
-if ! su pmos -c "yes '' | pmbootstrap \
-		--details-to-stdout \
-		init \
-		>/tmp/pmb_init 2>&1"; then
-	cat /tmp/pmb_init
-	exit 1
-fi
-echo ""
diff --git a/.ci/pytest.sh b/.ci/pytest.sh
index 54e48fbb..1cad49cc 100755
--- a/.ci/pytest.sh
+++ b/.ci/pytest.sh
@@ -1,6 +1,39 @@
 #!/bin/sh -e
-topdir="$(realpath "$(dirname "$0")/..")"
-cd "$topdir"
+# Description: run pmbootstrap python testsuite
+# Options: native slow
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add \
+		git \
+		openssl \
+		py3-pytest \
+		py3-pytest-cov \
+		sudo
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+# Require pytest to be installed on the host system
+if [ -z "$(command -v pytest)" ]; then
+	echo "ERROR: pytest command not found, make sure it is in your PATH."
+	exit 1
+fi
+
+# Use pytest-cov if it is installed to display code coverage
+cov_arg=""
+if python -c "import pytest_cov" >/dev/null 2>&1; then
+	cov_arg="--cov=pmb"
+fi
+
+echo "Initializing pmbootstrap..."
+if ! yes '' | ./pmbootstrap.py \
+		--details-to-stdout \
+		init \
+		>/tmp/pmb_init 2>&1; then
+	cat /tmp/pmb_init
+	exit 1
+fi
 
 # Make sure that the work folder format is up to date, and that there are no
 # mounts from aborted test cases (#1595)
@@ -21,4 +54,13 @@ if ! [ -e "$deviceinfo" ]; then
 	exit 1
 fi
 
-pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"
+echo "Running pytest..."
+echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
+pytest \
+	--color=yes \
+	-vv \
+	-x \
+	$cov_arg \
+	test \
+		-m "not skip_ci" \
+		"$@"
diff --git a/.ci/shellcheck.sh b/.ci/shellcheck.sh
new file mode 100755
index 00000000..dd21a570
--- /dev/null
+++ b/.ci/shellcheck.sh
@@ -0,0 +1,15 @@
+#!/bin/sh -e
+# Description: lint all shell scripts
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add shellcheck
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
+
+find . -name '*.sh' |
+while read -r file; do
+	echo "shellcheck: $file"
+	shellcheck "$file"
+done
diff --git a/.ci/vermin.sh b/.ci/vermin.sh
index 12025eeb..33c94dd9 100755
--- a/.ci/vermin.sh
+++ b/.ci/vermin.sh
@@ -1,17 +1,22 @@
 #!/bin/sh -e
-_vermin() {
-	if ! vermin -q "$@" >/dev/null 2>&1; then
-		vermin -vv "$@"
-	fi
-}
+# Description: verify that we don't use too new python features
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	apk -q add vermin
+	exec su "${TESTUSER:-build}" -c "sh -e $0"
+fi
 
 # shellcheck disable=SC2046
-_vermin \
+vermin \
 	-t=3.6- \
 	--backport argparse \
 	--backport configparser \
 	--backport enum \
 	--backport typing \
+	--lint \
+	--no-parse-comments \
 	$(find . -name '*.py' \
 		-a -not -path "./.venv/*" \
 		-a -not -path "./venv/*")
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
deleted file mode 100644
index 02e578cd..00000000
--- a/.gitlab-ci.yml
+++ /dev/null
@@ -1,66 +0,0 @@
----
-# Author: Clayton Craft <clayton@craftyguy.net>
-
-image: alpine:latest
-
-# defaults for "only"
-# We need to run the CI jobs in a "merge request specific context", if CI is
-# running in a merge request. Otherwise the environment variable that holds the
-# merge request ID is not available. This means, we must set the "only"
-# variable accordingly - and if we only do it for one job, all other jobs will
-# not get executed. So have the defaults here, and use them in all jobs that
-# should run on both the master branch, and in merge requests.
-# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
-.only-default: &only-default
-  only:
-    - master
-    - merge_requests
-    - tags
-
-static-code-analysis:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c "test/static_code_analysis.sh"
-
-vermin:
-  image: alpine:latest
-  <<: *only-default
-  before_script:
-  - "apk -q add py3-pip"
-  - "pip3 -q --disable-pip-version-check install vermin"
-  script:
-  - ".ci/vermin.sh"
-
-# MR settings
-# (Checks for "Allow commits from members who can merge to the target branch")
-mr-settings:
-  only:
-    - merge_requests
-  before_script:
-    - apk -q add python3
-  script:
-    - wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
-    - python3 ./check_mr_settings.py
-
-pytest:
-  <<: *only-default
-  before_script:
-    - .ci/prepare.sh
-  script:
-    - su pmos -c .ci/pytest.sh
-  after_script:
-     # Move logs so it can be saved as artifacts
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
-    - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
-    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
-    - "dmesg > $CI_PROJECT_DIR/dmesg.txt"
-  artifacts:
-    when: always
-    paths:
-      - "log.txt"
-      - "log_testsuite.txt"
-      - "dmesg.txt"
-      - "pmbootstrap.cfg"
-    expire_in: 1 week
diff --git a/test/static_code_analysis.sh b/test/static_code_analysis.sh
deleted file mode 100755
index 438d83a7..00000000
--- a/test/static_code_analysis.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/sh
-# Copyright 2022 Oliver Smith
-# SPDX-License-Identifier: AGPL-3.0-or-later
-set -e
-DIR="$(cd "$(dirname "$0")" && pwd -P)"
-cd "$DIR/.."
-
-# Make sure that the work folder format is up to date, and that there are no
-# mounts from aborted test cases (#1595)
-./pmbootstrap.py work_migrate
-./pmbootstrap.py -q shutdown
-
-# Install needed packages
-echo "Initializing Alpine chroot (details: 'pmbootstrap log')"
-./pmbootstrap.py -q chroot -- apk -q add \
-	shellcheck \
-	python3 \
-	py3-flake8 || return 1
-
-rootfs_native="$(./pmbootstrap.py config work)/chroot_native"
-command="$rootfs_native/lib/ld-musl-$(uname -m).so.1"
-command="$command --library-path=$rootfs_native/lib:$rootfs_native/usr/lib"
-shellcheck_command="$command $rootfs_native/usr/bin/shellcheck"
-flake8_command="$command $rootfs_native/usr/bin/python3 $rootfs_native/usr/bin/flake8"
-
-# Shell: shellcheck
-find . -name '*.sh' |
-while read -r file; do
-	echo "Test with shellcheck: $file"
-	cd "$DIR/../$(dirname "$file")"
-	$shellcheck_command -e SC1008 -x "$(basename "$file")"
-done
-
-# Python: flake8
-# F401: imported, but not used, does not make sense in __init__ files
-# E402: module import not on top of file, not possible for testcases
-# E722: do not use bare except
-cd "$DIR"/..
-echo "Test with flake8: *.py"
-# Note: omitting a virtualenv if it is here (e.g. gitlab CI)
-py_files="$(find . -not -path '*/venv/*' -name '*.py')"
-_ignores="E402,E722,W504,W605"
-# shellcheck disable=SC2086
-$flake8_command --exclude=__init__.py --ignore "$_ignores" $py_files
-# shellcheck disable=SC2086
-$flake8_command --filename=__init__.py --ignore "F401,$_ignores" $py_files
-
-# Done
-echo "Success!"