Move all non-essential services that are not required for testing to the nix-processmgmt-services repository

This commit is contained in:
Sander van der Burg 2021-02-07 14:34:32 +01:00 committed by Sander van der Burg
parent 9cd9e240e0
commit 972fa2278e
15 changed files with 8 additions and 752 deletions

View File

@ -1013,9 +1013,9 @@ This repository contains a number of example systems, that can be found in the
* `webapps-agnostic` is the same as the previous example, but using a process
manager agnostic configuration. It can be used to target all process managers
that this toolset supports.
* `services-agnostic` is a process manager-agnostic configuration set of common
system services, such as Apache HTTP server, MySQL, PostgreSQL and
Apache Tomcat.
* `services-agnostic` is a process manager-agnostic configuration set of
additional system services used for tests, such as docker, supervisord, and
nginx
* `service-containers-agnostic` extends the previous examples with configuration
files so that these system services can be deployed as Disnix containers --
services in which other services can be hosted.
@ -1023,6 +1023,11 @@ This repository contains a number of example systems, that can be found in the
image that concurrently runs all processes described in the `webapps-agnostic`
example managed by a process management solution of choice.
The
[Nix process management services](https://github.com/svanderburg/nix-processmgmt-services)
contains a collection of commonly used services that can be managed with the Nix
process management framework.
Troubleshooting
===============
This section contains a number of known problems and their resolutions.

View File

@ -1,34 +0,0 @@
{createManagedProcess, apacheHttpd, cacheDir}:
{instanceSuffix ? "", instanceName ? "httpd${instanceSuffix}", configFile, initialize ? "", environment ? {}, postInstall ? ""}:
let
user = instanceName;
group = instanceName;
in
createManagedProcess {
name = instanceName;
inherit instanceName initialize environment postInstall;
process = "${apacheHttpd}/bin/httpd";
args = [ "-f" configFile ];
foregroundProcessExtraArgs = [ "-DFOREGROUND" ];
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
inherit group;
homeDir = "${cacheDir}/${user}";
description = "Apache HTTP daemon user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -15,40 +15,6 @@ let
};
in
{
apache = import ./apache.nix {
inherit createManagedProcess cacheDir;
inherit (pkgs) apacheHttpd;
};
simpleWebappApache = import ./simple-webapp-apache.nix {
inherit createManagedProcess logDir cacheDir runtimeDir forceDisableUserChange;
inherit (pkgs) stdenv runCommand apacheHttpd php writeTextFile;
};
mysql = import ./mysql.nix {
inherit createManagedProcess stateDir runtimeDir forceDisableUserChange;
inherit (pkgs) stdenv mysql;
};
postgresql = import ./postgresql.nix {
inherit createManagedProcess stateDir runtimeDir forceDisableUserChange;
inherit (pkgs) stdenv postgresql su;
};
tomcat = import ./tomcat.nix {
inherit createManagedProcess stateDir runtimeDir tmpDir forceDisableUserChange;
inherit (pkgs) stdenv;
jre = pkgs.jre8;
tomcat = pkgs.tomcat9;
};
simpleAppservingTomcat = import ./simple-appserving-tomcat.nix {
inherit createManagedProcess stateDir runtimeDir tmpDir forceDisableUserChange;
inherit (pkgs) stdenv;
jre = pkgs.jre8;
tomcat = pkgs.tomcat9;
};
nginx = import ./nginx.nix {
inherit createManagedProcess stateDir runtimeDir cacheDir forceDisableUserChange;
inherit (pkgs) stdenv nginx;
@ -59,21 +25,6 @@ in
inherit (pkgs) stdenv writeTextFile nginx;
};
nginxReverseProxyPathBased = import ./nginx-reverse-proxy-pathbased.nix {
inherit createManagedProcess stateDir runtimeDir cacheDir forceDisableUserChange;
inherit (pkgs) stdenv writeTextFile nginx;
};
mongodb = import ./mongodb.nix {
inherit createManagedProcess runtimeDir;
inherit (pkgs) mongodb;
};
simpleMongodb = import ./simplemongodb.nix {
inherit createManagedProcess runtimeDir stateDir forceDisableUserChange;
inherit (pkgs) stdenv mongodb writeTextFile;
};
supervisord = import ./supervisord.nix {
inherit createManagedProcess runtimeDir logDir;
inherit (pkgs.pythonPackages) supervisor;
@ -85,26 +36,6 @@ in
inherit (pkgs.pythonPackages) supervisor;
};
svnserve = import ./svnserve.nix {
inherit createManagedProcess runtimeDir forceDisableUserChange;
inherit (pkgs) stdenv subversion;
};
simpleInfluxdb = import ./simpleinfluxdb.nix {
inherit createManagedProcess stateDir;
inherit (pkgs) influxdb writeTextFile;
};
influxdb = import ./influxdb.nix {
inherit createManagedProcess stateDir;
inherit (pkgs) influxdb;
};
sshd = import ./sshd.nix {
inherit createManagedProcess stateDir runtimeDir tmpDir forceDisableUserChange;
inherit (pkgs) writeTextFile openssh;
};
docker = import ./docker.nix {
inherit createManagedProcess;
inherit (pkgs) docker kmod;

View File

@ -1,35 +0,0 @@
{createManagedProcess, influxdb, stateDir}:
{instanceSuffix ? "", instanceName ? "influxdb${instanceSuffix}", configFile, postInstall ? ""}:
let
user = instanceName;
group = instanceName;
influxdbStateDir = "${stateDir}/lib/${instanceName}";
in
createManagedProcess {
name = instanceName;
inherit instanceName user postInstall;
foregroundProcess = "${influxdb}/bin/influxd";
args = [ "-config" configFile ];
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
homeDir = influxdbStateDir;
createHomeDir = true;
inherit group;
description = "InfluxDB user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,33 +0,0 @@
{createManagedProcess, mongodb, runtimeDir}:
{instanceSuffix ? "", instanceName ? "mongodb${instanceSuffix}", configFile, initialize ? "", postInstall ? ""}:
let
user = instanceName;
group = instanceName;
in
createManagedProcess {
name = instanceName;
inherit instanceName initialize postInstall;
process = "${mongodb}/bin/mongod";
args = [ "--config" configFile ];
daemonExtraArgs = [ "--fork" "--pidfilepath" "${runtimeDir}/${instanceName}.pid" ];
user = instanceName;
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
inherit group;
description = "MongoDB user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,62 +0,0 @@
{createManagedProcess, stdenv, mysql, stateDir, runtimeDir, forceDisableUserChange}:
{port ? 3306, instanceSuffix ? "", instanceName ? "mysql${instanceSuffix}", postInstall ? ""}:
let
dataDir = "${stateDir}/db/${instanceName}";
# By default, the socket file resides in $runtimeDir/mysqld/mysqld.sock.
# We only change the path component: 'mysqld' into the instance name if no
# instanceSuffix parameter is specified. Otherwise, we append the
# instanceSuffix to 'mysqld'.
#
# This construction is used to allow the mysql client executable to work
# without a socket parameter for the default configuration.
instanceRuntimeDir =
if instanceName != "mysql" && instanceSuffix == "" then "${runtimeDir}/${instanceName}"
else "${runtimeDir}/mysqld${instanceSuffix}";
user = instanceName;
group = instanceName;
in
createManagedProcess {
name = instanceName;
inherit instanceName postInstall;
initialize = ''
mkdir -m0700 -p ${dataDir}
mkdir -m0700 -p ${instanceRuntimeDir}
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown ${user}:${group} ${dataDir}
chown ${user}:${group} ${instanceRuntimeDir}
''}
if [ ! -e "${dataDir}/mysql" ]
then
${mysql}/bin/mysql_install_db --basedir=${mysql} --datadir=${dataDir} ${if forceDisableUserChange then "" else "--user=${user}"}
fi
'';
foregroundProcess = "${mysql}/bin/mysqld";
foregroundProcessArgs = [ "--basedir" mysql "--datadir" dataDir "--port" port "--socket" "${instanceRuntimeDir}/mysqld.sock" ]
++ stdenv.lib.optionals (!forceDisableUserChange) [ "--user" user ];
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
inherit group;
description = "MySQL user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,84 +0,0 @@
{createManagedProcess, stdenv, writeTextFile, nginx, runtimeDir, stateDir, cacheDir, forceDisableUserChange}:
{port ? 80, webapps ? [], instanceSuffix ? "", instanceName ? "nginx${instanceSuffix}", enableCache ? false}:
interDependencies:
let
user = instanceName;
group = instanceName;
nginxStateDir = "${stateDir}/${instanceName}";
nginxLogDir = "${nginxStateDir}/logs";
nginxCacheDir = "${cacheDir}/${instanceName}";
dependencies = webapps ++ (builtins.attrValues interDependencies);
in
import ./nginx.nix {
inherit createManagedProcess stdenv nginx stateDir forceDisableUserChange runtimeDir cacheDir;
} {
inherit instanceName;
dependencies = map (webapp: webapp.pkg) webapps
++ map (interDependency: interDependency.pkgs."${stdenv.system}") (builtins.attrValues interDependencies);
configFile = writeTextFile {
name = "nginx.conf";
text = ''
pid ${runtimeDir}/${instanceName}.pid;
error_log ${nginxLogDir}/error.log;
${stdenv.lib.optionalString (!forceDisableUserChange) ''
user ${user} ${group};
''}
events {
worker_connections 190000;
}
http {
access_log ${nginxLogDir}/access.log;
error_log ${nginxLogDir}/error.log;
proxy_temp_path ${nginxCacheDir}/proxy;
client_body_temp_path ${nginxCacheDir}/client_body;
fastcgi_temp_path ${nginxCacheDir}/fastcgi;
uwsgi_temp_path ${nginxCacheDir}/uwsgi;
scgi_temp_path ${nginxCacheDir}/scgi;
${stdenv.lib.optionalString enableCache ''
${stdenv.lib.concatMapStrings (dependency:
''
proxy_cache_path ${nginxCacheDir}/${dependency.name} keys_zone=${dependency.name}:8m inactive=5m max_size=128m;
''
) dependencies}
''}
${stdenv.lib.concatMapStrings (dependency:
''
upstream ${dependency.name} {
ip_hash;
${if dependency ? targets
then stdenv.lib.concatMapStrings (target: "server ${target.properties.hostname}:${toString dependency.port};\n") dependency.targets
else "server localhost:${dependency.port};\n"
}
}
''
) dependencies}
server {
${stdenv.lib.concatMapStrings (dependency:
''
location ${dependency.baseURL} {
proxy_pass http://${dependency.name};
${stdenv.lib.optionalString enableCache ''
proxy_cache ${dependency.name};
proxy_cache_key $host$uri$is_args$args;
proxy_cache_valid 200 5m;
proxy_cache_lock on;
''}
}
'') dependencies}
}
}
'';
};
}

View File

@ -1,53 +0,0 @@
{createManagedProcess, stdenv, postgresql, su, stateDir, runtimeDir, forceDisableUserChange}:
{port ? 5432, instanceSuffix ? "", instanceName ? "postgresql${instanceSuffix}", postInstall ? ""}:
let
postgresqlStateDir = "${stateDir}/db/${instanceName}";
dataDir = "${postgresqlStateDir}/data";
socketDir = "${runtimeDir}/${instanceName}";
user = instanceName;
group = instanceName;
in
createManagedProcess rec {
name = instanceName;
inherit instanceName user postInstall;
path = [ postgresql su ];
initialize = ''
mkdir -m0700 -p ${socketDir}
mkdir -m0700 -p ${dataDir}
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown ${user}:${group} ${socketDir}
chown ${user}:${group} ${dataDir}
''}
if [ ! -e "${dataDir}/PG_VERSION" ]
then
${stdenv.lib.optionalString (!forceDisableUserChange) "su ${user} -c '"}${postgresql}/bin/initdb -D ${dataDir} --no-locale${stdenv.lib.optionalString (!forceDisableUserChange) "'"}
fi
'';
foregroundProcess = "${postgresql}/bin/postgres";
args = [ "-D" dataDir "-p" port "-k" socketDir ];
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
homeDir = postgresqlStateDir;
createHomeDir = true;
inherit group;
description = "PostgreSQL user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,29 +0,0 @@
{createManagedProcess, stdenv, tomcat, jre, stateDir, runtimeDir, tmpDir, forceDisableUserChange}:
{instanceSuffix ? "", instanceName ? "tomcat${instanceSuffix}", serverPort ? 8005, httpPort ? 8080, httpsPort ? 8443, ajpPort ? 8009, commonLibs ? [], postInstall ? ""}:
let
tomcatConfigFiles = stdenv.mkDerivation {
name = "tomcat-config-files";
buildCommand = ''
mkdir -p $out
cd $out
mkdir conf
cp ${tomcat}/conf/* conf
sed -i \
-e 's|<Server port="8005" shutdown="SHUTDOWN">|<Server port="${toString serverPort}" shutdown="SHUTDOWN">|' \
-e 's|<Connector port="8080" protocol="HTTP/1.1"|<Connector port="${toString httpPort}" protocol="HTTP/1.1"|' \
-e 's|redirectPort="8443"|redirectPort="${toString httpsPort}"|' \
-e 's|<Connector port="8009" protocol="AJP/1.3"|<Connector port="${toString ajpPort}" protocol="AJP/1.3"|' \
conf/server.xml
mkdir webapps
cp -av ${tomcat.webapps}/webapps/* webapps
'';
};
in
import ./tomcat.nix {
inherit createManagedProcess stdenv tomcat jre stateDir runtimeDir tmpDir forceDisableUserChange commonLibs;
} {
inherit tomcatConfigFiles instanceName postInstall;
}

View File

@ -1,112 +0,0 @@
{createManagedProcess, stdenv, runCommand, apacheHttpd, php, writeTextFile, logDir, runtimeDir, cacheDir, forceDisableUserChange}:
{instanceSuffix ? "", instanceName ? "httpd${instanceSuffix}", port ? 80, modules ? [], serverName ? "localhost", serverAdmin, documentRoot ? ./webapp, enablePHP ? false, enableCGI ? false, extraConfig ? "", postInstall ? ""}:
let
user = instanceName;
group = instanceName;
baseModules = [
"mpm_prefork"
"authn_file"
"authn_core"
"authz_host"
"authz_groupfile"
"authz_user"
"authz_core"
"access_compat"
"auth_basic"
"reqtimeout"
"filter"
"mime"
"log_config"
"env"
"headers"
"setenvif"
"version"
"unixd"
"status"
"autoindex"
"alias"
"dir"
]
++ stdenv.lib.optional enableCGI "cgi";
apacheLogDir = "${logDir}/${instanceName}";
phpIni = runCommand "php.ini"
{
preferLocalBuild = true;
}
''
cat ${php}/etc/php.ini > $out
cat ${php.phpIni} > $out
'';
in
import ./apache.nix {
inherit createManagedProcess apacheHttpd cacheDir;
} {
inherit instanceName postInstall;
environment = stdenv.lib.optionalAttrs enablePHP {
PHPRC = phpIni;
};
initialize = ''
mkdir -m0700 -p ${apacheLogDir}
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown ${user}:${group} ${apacheLogDir}
''}
if [ ! -e "${documentRoot}" ]
then
mkdir -p "${documentRoot}"
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown ${user}:${group} ${documentRoot}
''}
fi
'';
configFile = writeTextFile {
name = "httpd.conf";
text = ''
ErrorLog "${apacheLogDir}/error_log"
PidFile "${runtimeDir}/${instanceName}.pid"
${stdenv.lib.optionalString (!forceDisableUserChange) ''
User ${user}
Group ${group}
''}
ServerName ${serverName}
ServerRoot ${apacheHttpd}
Listen ${toString port}
${stdenv.lib.concatMapStrings (module: ''
LoadModule ${module}_module ${apacheHttpd}/modules/mod_${module}.so
'') baseModules}
${stdenv.lib.concatMapStrings (module: ''
LoadModule ${module.name}_module ${module.module}
'') modules}
${stdenv.lib.optionalString enablePHP ''
LoadModule php7_module ${php}/modules/libphp7.so
''}
ServerAdmin ${serverAdmin}
DocumentRoot "${documentRoot}"
${stdenv.lib.optionalString enablePHP ''
<FilesMatch \.php$>
SetHandler application/x-httpd-php
</FilesMatch>
<Directory ${documentRoot}>
DirectoryIndex index.php
</Directory>
''}
${extraConfig}
'';
};
}

View File

@ -1,31 +0,0 @@
{createManagedProcess, influxdb, writeTextFile, stateDir}:
{instanceSuffix ? "", instanceName ? "influxdb${instanceSuffix}", rpcBindIP ? "127.0.0.1", rpcPort ? 8088, httpBindIP ? "", httpPort ? 8086, extraConfig ? "", postInstall ? ""}:
let
influxdbStateDir = "${stateDir}/lib/${instanceName}";
configFile = writeTextFile {
name = "influxdb.conf";
text = ''
bind-address = "${rpcBindIP}:${toString rpcPort}"
[meta]
dir = "${influxdbStateDir}/meta"
[data]
dir = "${influxdbStateDir}/data"
wal-dir = "${influxdbStateDir}/wal"
[http]
enabled = true
bind-address = "${httpBindIP}:${toString httpPort}"
${extraConfig}
'';
};
in
import ./influxdb.nix {
inherit createManagedProcess influxdb stateDir;
} {
inherit instanceName configFile postInstall;
}

View File

@ -1,28 +0,0 @@
{createManagedProcess, stdenv, writeTextFile, mongodb, runtimeDir, stateDir, forceDisableUserChange}:
{instanceSuffix ? "", instanceName ? "mongodb${instanceSuffix}", bindIP ? "127.0.0.1", port ? 27017, postInstall ? ""}:
let
mongodbDir = "${stateDir}/db/${instanceName}";
user = instanceName;
group = instanceName;
in
import ./mongodb.nix {
inherit createManagedProcess mongodb runtimeDir;
} {
inherit instanceName postInstall;
configFile = writeTextFile {
name = "mongodb.conf";
text = ''
systemLog.destination: syslog
storage.dbPath: ${mongodbDir}
net.bindIp: ${bindIP}
net.port: ${toString port}
'';
};
initialize = ''
mkdir -p ${mongodbDir}
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown ${user}:${group} ${mongodbDir}
''}
'';
}

View File

@ -1,69 +0,0 @@
{createManagedProcess, writeTextFile, openssh, stateDir, runtimeDir, tmpDir, forceDisableUserChange}:
{instanceSuffix ? "", instanceName ? "sshd${instanceSuffix}", port ? 22, extraSSHDConfig ? ""}:
let
sshdStateDir = "${stateDir}/lib/${instanceName}";
sshdConfig = writeTextFile {
name = "sshd_config";
text = ''
HostKey ${sshdStateDir}/ssh_host_rsa_key
HostKey ${sshdStateDir}/ssh_host_ecdsa_key
HostKey ${sshdStateDir}/ssh_host_ed25519_key
PidFile ${if forceDisableUserChange then tmpDir else runtimeDir}/${instanceName}.pid
${extraSSHDConfig}
'';
};
group = instanceName;
user = instanceName;
in
createManagedProcess {
name = instanceName;
inherit instanceName;
initialize = ''
mkdir -p ${sshdStateDir}
mkdir -p /var/empty
if [ ! -f ${sshdStateDir}/ssh_host_rsa_key ]
then
ssh-keygen -t rsa -f ${sshdStateDir}/ssh_host_rsa_key -N ""
fi
if [ ! -f ${sshdStateDir}/ssh_host_ecdsa_key ]
then
ssh-keygen -t ecdsa -f ${sshdStateDir}/ssh_host_ecdsa_key -N ""
fi
if [ ! -f ${sshdStateDir}/ssh_host_ed25519_key ]
then
ssh-keygen -t ed25519 -f ${sshdStateDir}/ssh_host_ed25519_key -N ""
fi
'';
process = "${openssh}/bin/sshd";
args = [ "-p" port "-f" sshdConfig ];
foregroundProcessExtraArgs = [ "-D" ];
path = [ openssh ];
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
inherit group;
homeDir = "/var/empty";
description = "SSH privilege separation user";
};
};
};
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,26 +0,0 @@
{createManagedProcess, stdenv, subversion, runtimeDir, forceDisableUserChange}:
{instanceSuffix ? "", instanceName ? "svnserve${instanceSuffix}", port ? 3690, svnBaseDir, svnGroup, postInstall ? ""}:
let
pidFile = "${runtimeDir}/${instanceName}.pid";
in
createManagedProcess {
name = instanceName;
inherit instanceName postInstall;
initialize = ''
mkdir -p ${svnBaseDir}
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chgrp ${svnGroup} ${svnBaseDir}
''}
'';
process = "${subversion.out}/bin/svnserve";
args = [ "-r" svnBaseDir "--listen-port" (toString port) ];
foregroundProcessExtraArgs = [ "--foreground" ];
daemonExtraArgs = [ "--daemon" "--pid-file" pidFile ];
overrides = {
sysvinit = {
runlevels = [ 3 4 5 ];
};
};
}

View File

@ -1,84 +0,0 @@
{createManagedProcess, stdenv, tomcat, jre, stateDir, runtimeDir, tmpDir, forceDisableUserChange, commonLibs ? []}:
{instanceSuffix ? "", instanceName ? "tomcat${instanceSuffix}", tomcatConfigFiles, postInstall ? ""}:
let
baseDir = "${stateDir}/${instanceName}";
user = instanceName;
group = instanceName;
pidFile = "${runtimeDir}/${instanceName}.pid";
in
createManagedProcess rec {
name = instanceName;
inherit instanceName user pidFile postInstall;
process = "${tomcat}/bin/catalina.sh";
args = [ "run" ];
environment = {
JRE_HOME = jre;
CATALINA_TMPDIR = tmpDir;
CATALINA_BASE = baseDir;
CATALINA_PID = pidFile;
};
initialize = ''
if [ ! -d "${baseDir}/logs" ]
then
mkdir -p ${baseDir}/logs
cd ${baseDir}
cp -av ${tomcatConfigFiles}/* .
chmod -R u+w .
mkdir -p ${baseDir}/lib
# Symlink all the given common libs files or paths into the lib/ directory
for i in ${tomcat} ${toString commonLibs}
do
if [ -f "$i" ]
then
# If the given web application is a file, symlink it into the common/lib/ directory
ln -sfn $i ${baseDir}/lib/$(basename $i)
elif [ -d "$i" ]
then
# If the given web application is a directory, then iterate over the files
# in the special purpose directories and symlink them into the tomcat tree
for j in $i/lib/*
do
ln -sfn $j ${baseDir}/lib/$(basename $j)
done
fi
done
${stdenv.lib.optionalString (!forceDisableUserChange) ''
chown -R ${user}:${group} ${baseDir}
''}
fi
'';
credentials = {
groups = {
"${group}" = {};
};
users = {
"${user}" = {
inherit group;
homeDir = baseDir;
createHomeDir = true;
description = "Tomcat user";
};
};
};
overrides = {
sysvinit = {
instructions.start = {
activity = "Starting";
instruction = ''
${initialize}
${tomcat}/bin/startup.sh
'';
};
runlevels = [ 3 4 5 ];
};
};
}