Implement support for gitlab CI

This implements support for the gitlab CI system. Wiki, static analysis,
and build tests are implemented.

.gitlab-ci.yml

@@ -0,0 +1,79 @@
+---
+# Author: Clayton Craft <clayton@craftyguy.net>
+
+image: python:3.6-slim-stretch
+
+cache:
+  paths:
+    - venv
+
+before_script:
+  - ./.gitlab/setup-pmos-environment.sh
+  # venv created in CI_PROJECT_DIR for caching
+  - "[[ ! -d venv ]] && virtualenv venv -p $(which python3.6)"
+  - "source venv/bin/activate"
+  - "pip3 install flake8 pytest-cov python-coveralls pytest"
+  - "python3 --version"
+
+stages:
+  - checks
+  - tests
+
+wiki-test:
+  stage: checks
+  script:
+    - "./test/check_devices_in_wiki.py"
+
+static-code-analysis:
+  stage: checks
+  script:
+    - "./test/static_code_analysis.sh"
+
+pmbootstrap-tests:
+  stage: tests
+  script:
+    # Note: This script uses CI_PROJECT_DIR
+    - su pmos -c "CI_PROJECT_DIR=$CI_PROJECT_DIR .gitlab/shared-runner-tests.sh"
+  after_script:
+    # Move logs so it can be saved as artifacts
+    - "[[ -f /home/pmos/.local/var/pmbootstrap/log.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
+    - "[[ -f /home/pmos/.local/var/pmbootstrap/log_testsuite.txt ]] && mv /home/pmos/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
+    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
+    - "sudo dmesg > $CI_PROJECT_DIR/dmesg.txt"
+  artifacts:
+    when: always
+    paths:
+      - "log.txt"
+      - "log_testsuite.txt"
+      - "dmesg.txt"
+      - "pmbootstrap.cfg"
+    expire_in: 1 week
+
+pmbootstrap-qemu-tests:
+  stage: tests
+  tags:
+    # This requires a specific runner, shared runners generally don't work.
+    - qemu
+  script:
+    # Note: this is a workaround for pmbootstrap's losetup failing
+    # due to loop devices not coming up fast enough when modprobe is
+    # called by pmbootstrap.
+    - "[[ ! -c /dev/loop0 ]] && sudo modprobe loop"
+    # Init test (pipefail disabled so 'yes' doesn't fail test)
+    - "set +o pipefail; yes ''| ./pmbootstrap.py init; set -o pipefail"
+    # qemu running process tests
+    - "python -m pytest -vv ./test/test_qemu_running_processes.py"
+  after_script:
+    # Move logs so it can be saved as artifacts
+    - "[[ -f ~/.local/var/pmbootstrap/log.txt ]] && mv ~/.local/var/pmbootstrap/log.txt $CI_PROJECT_DIR/log.txt"
+    - "[[ -f ~/.local/var/pmbootstrap/log_testsuite.txt ]] && mv ~/.local/var/pmbootstrap/log_testsuite.txt $CI_PROJECT_DIR/log_testsuite.txt"
+    - "[[ -f /home/pmos/.config/pmbootstrap.cfg ]] && cp /home/pmos/.config/pmbootstrap.cfg $CI_PROJECT_DIR/pmbootstrap.cfg"
+    - "sudo dmesg > $CI_PROJECT_DIR/dmesg.txt"
+  artifacts:
+    when: always
+    paths:
+      - "log.txt"
+      - "log_testsuite.txt"
+      - "dmesg.txt"
+      - "pmbootstrap.cfg"
+    expire_in: 1 week

.gitlab/config.toml

@@ -0,0 +1,22 @@
+# 'qemu' gitlab runner configuration file
+# Author: Clayton Craft <clayton@craftyguy.net>
+concurrent = 4
+check_interval = 0
+log_level = "debug"
+
+[[runners]]
+  name = "corredor"
+  url = "https://gitlab.com/"
+  token = <REDACTED>
+  executor = "virtualbox"
+  builds_dir = "/home/pmos/builds"
+  [runners.ssh]
+    user = "pmos"
+    password = <REDACTED>
+    identity_file = "/home/pmos/.ssh/id_ecdsa"
+  [runners.virtualbox]
+    base_name = "pmbootstrap-vm"
+    base_snapshot = "ci-snapshot-python-3.6"
+    #disable_snapshots = false
+    disable_snapshots = true
+  [runners.cache]

.gitlab/setup-pmos-environment.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# This script is meant for the gitlab CI shared runners, not for
+# any specific runners. Specific runners are expected to provide
+# all of these configurations to save time, at least for now.
+# Author: Clayton Craft <clayton@craftyguy.net>
+
+[[ -d "/home/pmos" ]] && echo "pmos user already exists, assume running on pre-configured runner" && exit
+mount -t binfmt_misc none /proc/sys/fs/binfmt_misc
+apt update
+apt install -q -y git sudo shellcheck
+pip3 install virtualenv
+echo "Creating pmos user"
+useradd pmos -m -s /bin/bash -b "/home"
+chown -R pmos:pmos .
+echo 'pmos ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers

.gitlab/shared-runner-tests.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# This script is meant to be executed by a non-root user, since pmbootstrap
+# commands will fail otherwise. This is primarily used by the gitlab CI shared
+# runners.
+# This script also assumes, if run outside a gitlab CI runner, that cwd is
+# the root of the pmbootstrap project. For gitlab CI runners, $CI_PROJECT_DIR
+# is used.
+# Author: Clayton Craft <clayton@craftyguy.net>
+
+# Return failure on any failure of commands below
+set -e
+set -x
+
+# Fail quickly if run as root, since other commands here will fail
+[[ "$(id -u)" != "0" ]]
+
+# These are specific to the gitlab CI
+[[ $CI_PROJECT_DIR ]] && cd "$CI_PROJECT_DIR"
+# shellcheck disable=SC1091
+[[ -d venv ]] && source ./venv/bin/activate
+
+# Init test (pipefail disabled so 'yes' doesn't fail test)
+set +o pipefail
+yes ''| ./pmbootstrap.py init
+set -o pipefail
+
+# kconfig_check
+./pmbootstrap.py kconfig check
+
+# check_checksums
+./test/check_checksums.py --build
+
+# this seems to be needed for some tests to pass
+set +o pipefail
+yes | ./pmbootstrap.py zap -m -p
+set -o pipefail
+
+# testcases_fast (qemu is omitted by not passing --all)
+./test/testcases_fast.sh
+

test/check_checksums.py

@@ -6,13 +6,12 @@ import sys
 
 def get_changed_files():
     try:
-        raw = subprocess.check_output(['git', 'diff', '--name-only', os.environ['TRAVIS_COMMIT_RANGE']])
-    except (KeyError, subprocess.CalledProcessError) as e:
-        if 'TRAVIS_PULL_REQUEST' in os.environ and os.environ['TRAVIS_PULL_REQUEST'] == "true":
-            branch = os.environ['TRAVIS_PULL_REQUEST_BRANCH']
-            raw = subprocess.check_output(['git', 'diff', '--name-only', 'master...{}'.format(branch)])
-        else:
-            raw = subprocess.check_output(['git', 'diff', '--name-only', 'HEAD~1'])
+        branch = os.environ['CI_COMMIT_REF_NAME']
+        raw = subprocess.check_output(['git', 'diff', '--name-only',
+                                       'master...{}'.format(branch)])
+    except (KeyError, subprocess.CalledProcessError):
+            raw = subprocess.check_output(['git', 'diff', '--name-only',
+                                           'HEAD~1'])
     return raw.decode().splitlines()
 
 
@@ -85,11 +84,6 @@ if __name__ == "__main__":
             print("usage: {} [--build]".format(sys.argv[0]))
             exit(1)
 
-    if 'TRAVIS_COMMIT_RANGE' in os.environ:
-        print('Checking commit range: {}'.format(os.environ['TRAVIS_COMMIT_RANGE']))
-    if 'TRAVIS_PULL_REQUEST_BRANCH' in os.environ:
-        print('Checking PR branch: {}'.format(os.environ['TRAVIS_PULL_REQUEST_BRANCH']))
-
     packages = get_changed_packages()
 
     if len(packages) == 0:

test/static_code_analysis.sh

@@ -46,6 +46,8 @@ sh_files="
 	./aports/main/postmarketos-ui-hildon/postmarketos-ui-hildon.post-install
 	./helpers/envsetup.sh
 	./helpers/envkernel.sh
+	./.gitlab/setup-pmos-environment.sh
+	./.gitlab/shared-runner-tests.sh
 	$(find . -path './aports/main/postmarketos-ui-hildon/*.sh')
 	$(find . -name '*.trigger')
 	$(find . -path './aports/main/devicepkg-dev/*.sh')
@@ -64,7 +66,8 @@ done
 cd "$DIR"/..
 echo "Test with flake8: *.py"
 echo "NOTE: Run 'autopep8 -ria $PWD' to fix code style issues"
-py_files="$(find . -name '*.py')"
+# Note: omitting a virtualenv if it is here (e.g. gitlab CI)
+py_files="$(find . -not -path '*/venv/*' -name '*.py')"
 _ignores="E501,E402,E722"
 # shellcheck disable=SC2086
 flake8 --exclude=__init__.py --ignore "$_ignores" $py_files

test/testcases_fast.sh

@@ -34,5 +34,7 @@ for file in test/test_*.py; do
 done
 
 # Run enabled testcases with coverage enabled
+# Note: Pytest is called through python so that this is compatible with
+# running from a venv (e.g. in gitlab CI)
 # shellcheck disable=SC2086
-pytest --cov=pmb $enabled --tb=native
+python -m pytest -vv --cov=pmb $enabled --tb=native