CI: replace gitlab CI with sourcehut CI
Replace .gitlab-ci.yml with a minimal .build.yml that runs a script in the .ci dir for each task. Each of these scripts runs as root first, installs dependencies, and then drops rights to the testuser and runs the actual test. I went with this design, so we can add a 'pmbootstrap ci' command in following patches that run these in a pmbootstrap chroot (see pmaports#2169). Looking at flake8, we currently ignore W504 and W604. Would be nice to fix these instead, but let's do that in another patch. I've added a minver package to Alpine, so we don't need to install it form pip anymore. Use minver's new --lint argument to simplify the script, and as recommended by minver itself, use --no-parse-comments as speed optimization since we don't have minver specific comments. Using doas instead of sudo in build.yml would be nice, but this doesn't work out of the box. Can be changed in the future. Add a note to the pytest script, that 'pmbootstrap log' can be used to follow the log while tests are running.
This commit is contained in:
Oliver Smith
parent
4428c7bcdc
commit
4b8a0db5bc
|
|
@ -0,0 +1,19 @@
|
|||
image: alpine/edge
|
||||
packages:
|
||||
- sudo
|
||||
tasks:
|
||||
- shellcheck: |
|
||||
cd pmbootstrap
|
||||
sudo .ci/shellcheck.sh
|
||||
- flake8: |
|
||||
cd pmbootstrap
|
||||
sudo .ci/flake8.sh
|
||||
- vermin: |
|
||||
cd pmbootstrap
|
||||
sudo .ci/vermin.sh
|
||||
- pytest: |
|
||||
cd pmbootstrap
|
||||
sudo .ci/pytest.sh
|
||||
artifacts:
|
||||
- ".local/var/pmbootstrap/log.txt"
|
||||
- ".local/var/pmbootstrap/log_testsuite.txt"
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/sh -e
|
||||
# Description: lint all python scripts
|
||||
# https://postmarktos.org/pmb-ci
|
||||
|
||||
if [ "$(id -u)" = 0 ]; then
|
||||
set -x
|
||||
apk -q add py3-flake8
|
||||
exec su "${TESTUSER:-build}" -c "sh -e $0"
|
||||
fi
|
||||
|
||||
# E402: module import not on top of file, not possible for testcases
|
||||
# E722: do not use bare except
|
||||
# W504: line break occurred after a binary operator
|
||||
# W604: invalid escape sequence
|
||||
ign="E402,E722,W504,W605"
|
||||
|
||||
set -x
|
||||
|
||||
# __init__.py with additional ignore:
|
||||
# F401: imported, but not used
|
||||
# shellcheck disable=SC2046
|
||||
flake8 --ignore "F401,$ign" $(find . -not -path '*/venv/*' -name '__init__.py')
|
||||
|
||||
# Check all other files
|
||||
flake8 --ignore="$ign" --exclude=__init__.py
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
#!/bin/sh -e
|
||||
# Install pmbootstrap depends, set up pmos user with sudo
|
||||
if [ "$(id -u)" != 0 ]; then
|
||||
echo "ERROR: this script is meant to be executed in the gitlab-ci"
|
||||
echo "environment only."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
topdir="$(realpath "$(dirname "$0")/..")"
|
||||
cd "$topdir"
|
||||
|
||||
ln -sf "$PWD"/pmbootstrap.py /usr/local/bin/pmbootstrap
|
||||
|
||||
apk add -q \
|
||||
git \
|
||||
openssl \
|
||||
py3-pytest \
|
||||
py3-pytest-cov \
|
||||
sudo
|
||||
|
||||
adduser -D pmos
|
||||
chown -R pmos:pmos .
|
||||
echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
|
||||
|
||||
su pmos -c "git config --global user.email postmarketos-ci@localhost"
|
||||
su pmos -c "git config --global user.name postmarketOS_CI"
|
||||
|
||||
echo "Initializing pmbootstrap"
|
||||
if ! su pmos -c "yes '' | pmbootstrap \
|
||||
--details-to-stdout \
|
||||
init \
|
||||
>/tmp/pmb_init 2>&1"; then
|
||||
cat /tmp/pmb_init
|
||||
exit 1
|
||||
fi
|
||||
echo ""
|
||||
|
|
@ -1,6 +1,39 @@
|
|||
#!/bin/sh -e
|
||||
topdir="$(realpath "$(dirname "$0")/..")"
|
||||
cd "$topdir"
|
||||
# Description: run pmbootstrap python testsuite
|
||||
# Options: native slow
|
||||
# https://postmarktos.org/pmb-ci
|
||||
|
||||
if [ "$(id -u)" = 0 ]; then
|
||||
set -x
|
||||
apk -q add \
|
||||
git \
|
||||
openssl \
|
||||
py3-pytest \
|
||||
py3-pytest-cov \
|
||||
sudo
|
||||
exec su "${TESTUSER:-build}" -c "sh -e $0"
|
||||
fi
|
||||
|
||||
# Require pytest to be installed on the host system
|
||||
if [ -z "$(command -v pytest)" ]; then
|
||||
echo "ERROR: pytest command not found, make sure it is in your PATH."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use pytest-cov if it is installed to display code coverage
|
||||
cov_arg=""
|
||||
if python -c "import pytest_cov" >/dev/null 2>&1; then
|
||||
cov_arg="--cov=pmb"
|
||||
fi
|
||||
|
||||
echo "Initializing pmbootstrap..."
|
||||
if ! yes '' | ./pmbootstrap.py \
|
||||
--details-to-stdout \
|
||||
init \
|
||||
>/tmp/pmb_init 2>&1; then
|
||||
cat /tmp/pmb_init
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Make sure that the work folder format is up to date, and that there are no
|
||||
# mounts from aborted test cases (#1595)
|
||||
|
|
@ -21,4 +54,13 @@ if ! [ -e "$deviceinfo" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
pytest -vv -x --cov=pmb test -m "not skip_ci" "$@"
|
||||
echo "Running pytest..."
|
||||
echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
|
||||
pytest \
|
||||
--color=yes \
|
||||
-vv \
|
||||
-x \
|
||||
$cov_arg \
|
||||
test \
|
||||
-m "not skip_ci" \
|
||||
"$@"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/sh -e
|
||||
# Description: lint all shell scripts
|
||||
# https://postmarktos.org/pmb-ci
|
||||
|
||||
if [ "$(id -u)" = 0 ]; then
|
||||
set -x
|
||||
apk -q add shellcheck
|
||||
exec su "${TESTUSER:-build}" -c "sh -e $0"
|
||||
fi
|
||||
|
||||
find . -name '*.sh' |
|
||||
while read -r file; do
|
||||
echo "shellcheck: $file"
|
||||
shellcheck "$file"
|
||||
done
|
||||
|
|
@ -1,17 +1,22 @@
|
|||
#!/bin/sh -e
|
||||
_vermin() {
|
||||
if ! vermin -q "$@" >/dev/null 2>&1; then
|
||||
vermin -vv "$@"
|
||||
fi
|
||||
}
|
||||
# Description: verify that we don't use too new python features
|
||||
# https://postmarktos.org/pmb-ci
|
||||
|
||||
if [ "$(id -u)" = 0 ]; then
|
||||
set -x
|
||||
apk -q add vermin
|
||||
exec su "${TESTUSER:-build}" -c "sh -e $0"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2046
|
||||
_vermin \
|
||||
vermin \
|
||||
-t=3.6- \
|
||||
--backport argparse \
|
||||
--backport configparser \
|
||||
--backport enum \
|
||||
--backport typing \
|
||||
--lint \
|
||||
--no-parse-comments \
|
||||
$(find . -name '*.py' \
|
||||
-a -not -path "./.venv/*" \
|
||||
-a -not -path "./venv/*")
|
||||
|
|
|
|||
|
|
@ -1,66 +0,0 @@
|
|||
---
|
||||
# Author: Clayton Craft <clayton@craftyguy.net>
|
||||
|
||||
image: alpine:latest
|
||||
|
||||
# defaults for "only"
|
||||
# We need to run the CI jobs in a "merge request specific context", if CI is
|
||||
# running in a merge request. Otherwise the environment variable that holds the
|
||||
# merge request ID is not available. This means, we must set the "only"
|
||||
# variable accordingly - and if we only do it for one job, all other jobs will
|
||||
# not get executed. So have the defaults here, and use them in all jobs that
|
||||
# should run on both the master branch, and in merge requests.
|
||||
# https://docs.gitlab.com/ee/ci/merge_request_pipelines/index.html#excluding-certain-jobs
|
||||
.only-default: &only-default
|
||||
only:
|
||||
- master
|
||||
- merge_requests
|
||||
- tags
|
||||
|
||||
static-code-analysis:
|
||||
<<: *only-default
|
||||
before_script:
|
||||
- .ci/prepare.sh
|
||||
script:
|
||||
- su pmos -c "test/static_code_analysis.sh"
|
||||
|
||||
vermin:
|
||||
image: alpine:latest
|
||||
<<: *only-default
|
||||
before_script:
|
||||
- "apk -q add py3-pip"
|
||||
- "pip3 -q --disable-pip-version-check install vermin"
|
||||
script:
|
||||
- ".ci/vermin.sh"
|
||||
|
||||
# MR settings
|
||||
# (Checks for "Allow commits from members who can merge to the target branch")
|
||||
mr-settings:
|
||||
only:
|
||||
- merge_requests
|
||||
before_script:
|
||||
- apk -q add python3
|
||||
script:
|
||||
- wget -q "https://gitlab.com/postmarketOS/ci-common/-/raw/master/check_mr_settings.py"
|
||||
- python3 ./check_mr_settings.py
|
||||
|
||||
pytest:
|
||||
<<: *only-default
|
||||
before_script:
|
||||
- .ci/prepare.sh
|
||||
script:
|
||||
- su pmos -c .ci/pytest.sh
|
||||
after_script:
|
||||
# Move logs so it can be saved as artifacts
|
||||
- "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
|
||||
- "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
|
||||
- "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
|
||||
- "dmesg > $CI_PROJECT_DIR/dmesg.txt"
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- "log.txt"
|
||||
- "log_testsuite.txt"
|
||||
- "dmesg.txt"
|
||||
- "pmbootstrap.cfg"
|
||||
expire_in: 1 week
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#!/bin/sh
|
||||
# Copyright 2022 Oliver Smith
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
set -e
|
||||
DIR="$(cd "$(dirname "$0")" && pwd -P)"
|
||||
cd "$DIR/.."
|
||||
|
||||
# Make sure that the work folder format is up to date, and that there are no
|
||||
# mounts from aborted test cases (#1595)
|
||||
./pmbootstrap.py work_migrate
|
||||
./pmbootstrap.py -q shutdown
|
||||
|
||||
# Install needed packages
|
||||
echo "Initializing Alpine chroot (details: 'pmbootstrap log')"
|
||||
./pmbootstrap.py -q chroot -- apk -q add \
|
||||
shellcheck \
|
||||
python3 \
|
||||
py3-flake8 || return 1
|
||||
|
||||
rootfs_native="$(./pmbootstrap.py config work)/chroot_native"
|
||||
command="$rootfs_native/lib/ld-musl-$(uname -m).so.1"
|
||||
command="$command --library-path=$rootfs_native/lib:$rootfs_native/usr/lib"
|
||||
shellcheck_command="$command $rootfs_native/usr/bin/shellcheck"
|
||||
flake8_command="$command $rootfs_native/usr/bin/python3 $rootfs_native/usr/bin/flake8"
|
||||
|
||||
# Shell: shellcheck
|
||||
find . -name '*.sh' |
|
||||
while read -r file; do
|
||||
echo "Test with shellcheck: $file"
|
||||
cd "$DIR/../$(dirname "$file")"
|
||||
$shellcheck_command -e SC1008 -x "$(basename "$file")"
|
||||
done
|
||||
|
||||
# Python: flake8
|
||||
# F401: imported, but not used, does not make sense in __init__ files
|
||||
# E402: module import not on top of file, not possible for testcases
|
||||
# E722: do not use bare except
|
||||
cd "$DIR"/..
|
||||
echo "Test with flake8: *.py"
|
||||
# Note: omitting a virtualenv if it is here (e.g. gitlab CI)
|
||||
py_files="$(find . -not -path '*/venv/*' -name '*.py')"
|
||||
_ignores="E402,E722,W504,W605"
|
||||
# shellcheck disable=SC2086
|
||||
$flake8_command --exclude=__init__.py --ignore "$_ignores" $py_files
|
||||
# shellcheck disable=SC2086
|
||||
$flake8_command --filename=__init__.py --ignore "F401,$_ignores" $py_files
|
||||
|
||||
# Done
|
||||
echo "Success!"
|
||||
Loading…
Reference in New Issue