Make it possible to also define functions for overrides, supervisord always accepts a pidfile parameter

README.md

@@ -271,6 +271,102 @@ with process manager-specific parameters.
 In the above example, we define an override to specify the `runlevels`. Runlevels
 is a concept only supported by sysvinit scripts.
 
+Defining process manager-specific overrides
+-------------------------------------------
+As described in the previous section, the `createManagedProcess` abstraction only
+works with high-level concepts that are easily generalizable to all kinds of
+process managers.
+
+The attribute set of parameters passed to the `createManagedProcess` function
+gets translated to an attribute set of parameters for the corresponding
+process manager-specific abstraction functions, e.g. `createSystemVInitScript`,
+`createSupervisordProgram`, `createSystemdService` etc.
+
+We can change the content of the generated attribute set, allowing you to get
+access to any property of a process manager backend including properties for
+which the `createManagedProcess` function does provide any high-level concepts.
+
+An override can be be an attribute set that simply overrides or augments the
+process manager-specific parameter attribute set:
+
+```nix
+{createManagedProcess, tmpDir}:
+{port, instanceSuffix ? "", instanceName ? "webapp${instanceSuffix}"}:
+
+let
+  webapp = import ../../webapp;
+in
+createManagedProcess {
+  name = instanceName;
+  description = "Simple web application";
+  inherit instanceName;
+
+  # This expression can both run in foreground or daemon mode.
+  # The process manager can pick which mode it prefers.
+  process = "${webapp}/bin/webapp";
+  daemonArgs = [ "-D" ];
+
+  environment = {
+    PORT = port;
+    PID_FILE = "${tmpDir}/${instanceName}.pid";
+  };
+
+  overrides = {
+    sysvinit.runlevels = [ 3 4 5 ];
+    systemd = {
+      Service.Restart = "always";
+    };
+  };
+}
+```
+
+In the above example case, we use an override to define in which runlevels the
+service should start (a sysvinit specific concept), and when systemd is used,
+the service gets restarted automatically when it stops (which is not a universal
+property all process managers support, but systemd does).
+
+It is also possible to write an override as a function which is more powerful --
+you can also delete and augment existing parameters with additional information,
+if desired:
+
+```nix
+{createManagedProcess, tmpDir}:
+{port, instanceSuffix ? "", instanceName ? "webapp${instanceSuffix}"}:
+
+let
+  webapp = import ../../webapp;
+in
+createManagedProcess {
+  name = instanceName;
+  description = "Simple web application";
+  inherit instanceName;
+
+  # This expression can both run in foreground or daemon mode.
+  # The process manager can pick which mode it prefers.
+  process = "${webapp}/bin/webapp";
+  daemonArgs = [ "-D" ];
+
+  environment = {
+    PORT = port;
+    PID_FILE = "${tmpDir}/${instanceName}.pid";
+  };
+
+  overrides = {
+    sysvinit.runlevels = [ 3 4 5 ];
+    systemd = systemdArgs: systemdArgs // {
+      Service = systemdArgs.Service // {
+        ExecStart = "${systemdArgs.Service.ExecStart} -D";
+        Type = "forking";
+      };
+    };
+  };
+}
+```
+
+In the above example, I modify the generated systemd arguments in such a way
+that the service runs in daemon mode and it is managed as a daemon (by default,
+the systemd generator prefers to work with foreground processes).
+
 Writing a constructors expression
 ---------------------------------
 As shown in the previous sections, a process configuration is a nested function.

examples/services-agnostic/supervisord.nix

@@ -13,9 +13,8 @@ createManagedProcess {
     ${initialize}
   '';
   process = "${supervisor}/bin/supervisord";
-  args = [ "--configuration" configFile "--logfile" logFile ];
+  args = [ "--configuration" configFile "--logfile" logFile "--pidfile" pidFile ];
   foregroundProcessExtraArgs = [ "--nodaemon" ];
-  daemonExtraArgs = [ "--pidfile" pidFile ];
 
   overrides = {
     sysvinit = {

nixproc/create-compound-process/create-compound-process.nix

@@ -17,6 +17,7 @@ in
 , cacheDir ? "${stateDir}/cache"
 , tmpDir ? (if stateDir == "/var" then "/tmp" else "${stateDir}/tmp")
 , forceDisableUserChange ? false
+, path ? []
 , extraParams ? {}
 , overrides ? {}
 }:
@@ -45,7 +46,7 @@ let
   commonTools = (import ../../tools { inherit pkgs system; }).common;
 in
 createManagedProcess (pkgs.lib.recursiveUpdate {
-  inherit name overrides;
+  inherit name path overrides;
   initialize = ''
     ${commonTools}/bin/nixproc-init-state --state-dir "${stateDir}" --log-dir "${logDir}" --runtime-dir "${runtimeDir}" --cache-dir "${cacheDir}" --tmp-dir "${tmpDir}" ${pkgs.lib.optionalString forceDisableUserChange "--force-disable-user-change"}
   '';

nixproc/create-compound-process/generate-supervisord-args.nix

@@ -24,8 +24,7 @@ let
 in
 {
   process = "${pkgs.pythonPackages.supervisor}/bin/supervisord";
-  args = [ "--configuration" "${profile}/supervisord.conf" "--logfile" logFile ];
-  daemonExtraArgs = [ "--pidfile" pidFile ];
+  args = [ "--configuration" "${profile}/supervisord.conf" "--logfile" logFile "--pidfile" pidFile ];
   foregroundProcessExtraArgs = [ "--nodaemon" ];
   inherit pidFile;
 }

nixproc/create-managed-process/agnostic/generate-bsdrc-script.nix

@@ -23,16 +23,22 @@
 
 # TODO: umask
 
-createBSDRCScript (stdenv.lib.recursiveUpdate ({
-  inherit name environment path directory nice dependencies;
-  inherit user instanceName credentials postInstall;
+let
+  generatedTargetSpecificArgs = {
+    inherit name environment path directory nice dependencies;
+    inherit user instanceName credentials postInstall;
 
-  command = if daemon != null then daemon else foregroundProcess;
-  commandIsDaemon = daemon != null;
-  commandArgs = if daemon != null then daemonArgs else foregroundProcessArgs;
+    command = if daemon != null then daemon else foregroundProcess;
+    commandIsDaemon = daemon != null;
+    commandArgs = if daemon != null then daemonArgs else foregroundProcessArgs;
+  } // stdenv.lib.optionalAttrs (pidFile != null) {
+    inherit pidFile;
+  } // stdenv.lib.optionalAttrs (initialize != "") {
+    commands.start.pre = initialize;
+  };
 
-} // stdenv.lib.optionalAttrs (pidFile != null) {
-  inherit pidFile;
-} // stdenv.lib.optionalAttrs (initialize != "") {
-  commands.start.pre = initialize;
-}) overrides)
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+in
+createBSDRCScript targetSpecificArgs

nixproc/create-managed-process/agnostic/generate-cygrunsrv-params.nix

@@ -35,24 +35,30 @@ let
   generateForegroundProxy = import ./generate-foreground-proxy.nix {
     inherit stdenv writeTextFile;
   };
-in
-createCygrunsrvParams (stdenv.lib.recursiveUpdate ({
-  inherit name environment dependencies postInstall;
 
-  environmentPath = path;
+  generatedTargetSpecificArgs = {
+    inherit name environment dependencies postInstall;
 
-  path = if foregroundProcess != null then
-    if initialize == "" then foregroundProcess
+    environmentPath = path;
+
+    path = if foregroundProcess != null then
+      if initialize == "" then foregroundProcess
+      else generateForegroundProxy {
+        wrapDaemon = false;
+        executable = foregroundProcess;
+        inherit name initialize runtimeDir pidFile stdenv;
+      }
     else generateForegroundProxy {
-      wrapDaemon = false;
-      executable = foregroundProcess;
+      wrapDaemon = true;
+      executable = daemon;
       inherit name initialize runtimeDir pidFile stdenv;
-    }
-  else generateForegroundProxy {
-    wrapDaemon = true;
-    executable = daemon;
-    inherit name initialize runtimeDir pidFile stdenv;
+    };
+
+    args = if foregroundProcess != null then foregroundProcessArgs else daemonArgs;
   };
 
-  args = if foregroundProcess != null then foregroundProcessArgs else daemonArgs;
-}) overrides)
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+in
+createCygrunsrvParams targetSpecificArgs

nixproc/create-managed-process/agnostic/generate-docker-container.nix

@@ -72,7 +72,7 @@ let
 
   _environment = util.appendPathToEnvironment {
     inherit environment;
-    path = basePackages ++ path;
+    path = basePackages ++ path ++ [ "/" ]; # Also give permission to /bin to allow any package added to contents can be used
   };
 
   credentialsSpec = util.createCredentialsOrNull {
@@ -83,7 +83,7 @@ let
     inherit user forceDisableUserChange;
   };
 
-  dockerImage = dockerTools.buildImage (stdenv.lib.recursiveUpdate {
+  generatedDockerImageArgs = {
     inherit name;
     tag = "latest";
 
@@ -100,12 +100,30 @@ let
     } // stdenv.lib.optionalAttrs (_user != null) {
       User = _user;
     };
-  } overrides.image or {});
+  };
+
+  dockerImageArgs =
+    if overrides ? image
+    then
+      if builtins.isFunction overrides.image then overrides.image generatedDockerImageArgs
+      else stdenv.lib.recursiveUpdate generatedDockerImageArgs overrides.image
+    else generatedDockerImageArgs;
+
+  dockerImage = dockerTools.buildImage dockerImageArgs;
+
+  generatedDockerContainerArgs = {
+    inherit name dockerImage postInstall cmd dependencies;
+    dockerImageTag = "${name}:latest";
+    useHostNixStore = true;
+    useHostNetwork = true;
+    mapStateDirVolume = stateDir;
+  };
+
+  dockerContainerArgs =
+    if overrides ? container
+    then
+      if builtins.isFunction overrides.container then overrides.container generatedDockerContainerArgs
+      else stdenv.lib.recursiveUpdate generatedDockerContainerArgs overrides.container
+    else generatedDockerContainerArgs;
 in
-createDockerContainer (stdenv.lib.recursiveUpdate {
-  inherit name dockerImage postInstall cmd dependencies;
-  dockerImageTag = "${name}:latest";
-  useHostNixStore = true;
-  useHostNetwork = true;
-  mapStateDirVolume = stateDir;
-} overrides.container or {})
+createDockerContainer dockerContainerArgs

nixproc/create-managed-process/agnostic/generate-launchd-daemon.nix

@@ -52,7 +52,7 @@ let
   });
   ProgramArguments = [ Program ] ++ (if foregroundProcess != null then foregroundProcessArgs else daemonArgs);
 
-  daemonConfig = createLaunchdDaemon (stdenv.lib.recursiveUpdate ({
+  generatedTargetSpecificArgs = {
     inherit name credentials postInstall Program;
   } // stdenv.lib.optionalAttrs (ProgramArguments != [ Program ]) {
     inherit ProgramArguments;
@@ -68,7 +68,13 @@ let
     Nice = nice;
   } // stdenv.lib.optionalAttrs (user != null) {
     UserName = user;
-  }) overrides);
+  };
+
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+
+  daemonConfig = createLaunchdDaemon targetSpecificArgs;
 in
 if dependencies == [] then daemonConfig else
 builtins.trace "WARNING: dependencies have been specified for process: ${name}, but launchd has no notion of process dependencies. Proper activation ordering cannot be guaranteed!" daemonConfig

nixproc/create-managed-process/agnostic/generate-process-script.nix

@@ -69,34 +69,40 @@ let
       inherit pidFilesDir;
     }
     else throw "I don't know how to start this process!";
-in
-createProcessScript (stdenv.lib.recursiveUpdate ({
-  inherit name dependencies credentials postInstall;
 
-  process = writeTextFile {
-    name = "${name}-process-wrapper";
-    executable = true;
-    text = ''
-      #! ${stdenv.shell} -e
-    ''
-    + util.printShellEnvironmentVariables {
-      environment = _environment;
-      allowSystemPath = true;
-    }
-    + stdenv.lib.optionalString (umask != null) ''
-      umask ${umask}
-    ''
-    + stdenv.lib.optionalString (directory != null) ''
-      cd ${directory}
-    ''
-    + stdenv.lib.optionalString (nice != null) ''
-      nice -n ${toString nice}
-    ''
-    + stdenv.lib.optionalString (initialize != null) ''
-      ${initialize}
-    ''
-    + "exec ${invocationCommand}";
+  generatedTargetSpecificArgs = {
+    inherit name dependencies credentials postInstall;
+
+    process = writeTextFile {
+      name = "${name}-process-wrapper";
+      executable = true;
+      text = ''
+        #! ${stdenv.shell} -e
+      ''
+      + util.printShellEnvironmentVariables {
+        environment = _environment;
+        allowSystemPath = true;
+      }
+      + stdenv.lib.optionalString (umask != null) ''
+        umask ${umask}
+      ''
+      + stdenv.lib.optionalString (directory != null) ''
+        cd ${directory}
+      ''
+      + stdenv.lib.optionalString (nice != null) ''
+        nice -n ${toString nice}
+      ''
+      + stdenv.lib.optionalString (initialize != null) ''
+        ${initialize}
+      ''
+      + "exec ${invocationCommand}";
+    };
+  } // stdenv.lib.optionalAttrs (_pidFile != null) {
+    pidFile = _pidFile;
   };
-} // stdenv.lib.optionalAttrs (_pidFile != null) {
-  pidFile = _pidFile;
-}) overrides)
+
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+in
+createProcessScript targetSpecificArgs

nixproc/create-managed-process/agnostic/generate-supervisord-program.nix

@@ -47,15 +47,21 @@ let
     } // stdenv.lib.optionalAttrs (pidFile != null) {
       inherit pidFile;
     })) + " ${stdenv.lib.escapeShellArgs daemonArgs}";
+
+  generatedTargetSpecificArgs = {
+    inherit name command path environment dependencies credentials postInstall;
+  } // stdenv.lib.optionalAttrs (umask != null) {
+    inherit umask;
+  } // stdenv.lib.optionalAttrs (nice != null) {
+    inherit nice;
+  } // stdenv.lib.optionalAttrs (pidFile != null) {
+    inherit pidFile;
+  } // stdenv.lib.optionalAttrs (user != null) {
+    inherit user;
+  };
+
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
 in
-createSupervisordProgram (stdenv.lib.recursiveUpdate ({
-  inherit name command path environment dependencies credentials postInstall;
-} // stdenv.lib.optionalAttrs (umask != null) {
-  inherit umask;
-} // stdenv.lib.optionalAttrs (nice != null) {
-  inherit nice;
-} // stdenv.lib.optionalAttrs (pidFile != null) {
-  inherit pidFile;
-} // stdenv.lib.optionalAttrs (user != null) {
-  inherit user;
-}) overrides)
+createSupervisordProgram targetSpecificArgs

nixproc/create-managed-process/agnostic/generate-systemd-service.nix

@@ -25,29 +25,35 @@ let
   generatePreStartScript = import ./generate-prestart-script.nix {
     inherit stdenv writeTextFile;
   };
-in
-createSystemdService (stdenv.lib.recursiveUpdate {
-  inherit name path environment dependencies credentials postInstall;
 
-  Unit = {
-    Description = description;
-  };
-  Service = {
-    ExecStart = if foregroundProcess != null then "${foregroundProcess} ${stdenv.lib.escapeShellArgs foregroundProcessArgs}" else "${daemon} ${stdenv.lib.escapeShellArgs daemonArgs}";
-    Type = if foregroundProcess != null then "simple" else "forking";
-  } // stdenv.lib.optionalAttrs (initialize != "") {
-    ExecStartPre = stdenv.lib.optionalString (user != null) "+" + generatePreStartScript {
-      inherit name initialize;
+  generatedTargetSpecificArgs = {
+    inherit name path environment dependencies credentials postInstall;
+
+    Unit = {
+      Description = description;
+    };
+    Service = {
+      ExecStart = if foregroundProcess != null then "${foregroundProcess} ${stdenv.lib.escapeShellArgs foregroundProcessArgs}" else "${daemon} ${stdenv.lib.escapeShellArgs daemonArgs}";
+      Type = if foregroundProcess != null then "simple" else "forking";
+    } // stdenv.lib.optionalAttrs (initialize != "") {
+      ExecStartPre = stdenv.lib.optionalString (user != null) "+" + generatePreStartScript {
+        inherit name initialize;
+      };
+    } // stdenv.lib.optionalAttrs (directory != null) {
+      WorkingDirectory = directory;
+    } // stdenv.lib.optionalAttrs (umask != null) {
+      UMask = umask;
+    } // stdenv.lib.optionalAttrs (nice != null) {
+      Nice = nice;
+    } // stdenv.lib.optionalAttrs (foregroundProcess == null && pidFile != null) {
+      PIDFile = pidFile;
+    } // stdenv.lib.optionalAttrs (user != null) {
+      User = user;
     };
-  } // stdenv.lib.optionalAttrs (directory != null) {
-    WorkingDirectory = directory;
-  } // stdenv.lib.optionalAttrs (umask != null) {
-    UMask = umask;
-  } // stdenv.lib.optionalAttrs (nice != null) {
-    Nice = nice;
-  } // stdenv.lib.optionalAttrs (foregroundProcess == null && pidFile != null) {
-    PIDFile = pidFile;
-  } // stdenv.lib.optionalAttrs (user != null) {
-    User = user;
   };
-} overrides)
+
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+in
+createSystemdService targetSpecificArgs

nixproc/create-managed-process/agnostic/generate-sysvinit-script.nix

@@ -21,13 +21,20 @@
 , postInstall
 }:
 
-createSystemVInitScript (stdenv.lib.recursiveUpdate ({
-  inherit name description path environment directory umask nice dependencies credentials;
-  inherit instanceName initialize user postInstall;
+let
+  generatedTargetSpecificArgs = {
+    inherit name description path environment directory umask nice dependencies credentials;
+    inherit instanceName initialize user postInstall;
 
-  process = if daemon != null then daemon else foregroundProcess;
-  processIsDaemon = daemon != null;
-  args = if daemon != null then daemonArgs else foregroundProcessArgs;
-} // stdenv.lib.optionalAttrs (pidFile != null) {
-  inherit pidFile;
-}) overrides)
+    process = if daemon != null then daemon else foregroundProcess;
+    processIsDaemon = daemon != null;
+    args = if daemon != null then daemonArgs else foregroundProcessArgs;
+  } // stdenv.lib.optionalAttrs (pidFile != null) {
+    inherit pidFile;
+  };
+
+  targetSpecificArgs =
+    if builtins.isFunction overrides then overrides generatedTargetSpecificArgs
+    else stdenv.lib.recursiveUpdate generatedTargetSpecificArgs overrides;
+in
+createSystemVInitScript targetSpecificArgs

nixproc/create-managed-process/docker/setup.nix

@@ -6,6 +6,10 @@
   # Initialize common state directories
   ${commonTools}/bin/nixproc-init-state --state-dir ${stateDir} --runtime-dir ${runtimeDir}
 
+  # Always create /tmp because it is needed quite often
+  mkdir -p /tmp
+  chmod 1777 /tmp
+
   ${stdenv.lib.optionalString (!forceDisableUserChange && credentialsSpec != null) ''
     export PATH=$PATH:${findutils}/bin:${glibc.bin}/bin
     ${dysnomia}/bin/dysnomia-addgroups ${credentialsSpec}

nixproc/create-multi-process-image/configure-bashrc.nix

@@ -0,0 +1,26 @@
+''
+  mkdir -p /root
+  cat > /root/.bashrc << "EOF"
+  alias ls='ls --color=auto'
+
+  if [ -n "$PS1" ]
+  then
+      if [ "$TERM" != "dumb" -o -n "$INSIDE_EMACS" ]
+      then
+          PROMPT_COLOR="1;31m"
+          let $UID && PROMPT_COLOR="1;32m"
+          if [ -n "$INSIDE_EMACS" -o "$TERM" == "eterm" -o "$TERM" == "eterm-color" ]
+          then
+              # Emacs term mode doesn't support xterm title escape sequence (\e]0;)
+              PS1="\n\[\033[$PROMPT_COLOR\][\u@\h:\w]\\$\[\033[0m\] "
+          else
+              PS1="\n\[\033[$PROMPT_COLOR\][\[\e]0;\u@\h: \w\a\]\u@\h:\w]\\$\[\033[0m\] "
+          fi
+          if test "$TERM" = "xterm"
+          then
+              PS1="\[\033]2;\h:\u:\w\007\]$PS1"
+          fi
+      fi
+  fi
+  EOF
+''

nixproc/create-multi-process-image/create-multi-process-image.nix

@@ -46,31 +46,9 @@ dockerTools.buildImage ({
   runAsRoot =
     setupProcessManagement
     + processManagerArgs.runAsRoot
-    + stdenv.lib.optionalString interactive ''
-      mkdir -p /root
-      cat > /root/.bashrc << "EOF"
-      alias ls='ls --color=auto'
-
-      if [ -n "$PS1" ]
-      then
-          if [ "$TERM" != "dumb" -o -n "$INSIDE_EMACS" ]
-          then
-              PROMPT_COLOR="1;31m"
-              let $UID && PROMPT_COLOR="1;32m"
-              if [ -n "$INSIDE_EMACS" -o "$TERM" == "eterm" -o "$TERM" == "eterm-color" ]
-              then
-                  # Emacs term mode doesn't support xterm title escape sequence (\e]0;)
-                  PS1="\n\[\033[$PROMPT_COLOR\][\u@\h:\w]\\$\[\033[0m\] "
-              else
-                  PS1="\n\[\033[$PROMPT_COLOR\][\[\e]0;\u@\h: \w\a\]\u@\h:\w]\\$\[\033[0m\] "
-              fi
-              if test "$TERM" = "xterm"
-              then
-                  PS1="\[\033]2;\h:\u:\w\007\]$PS1"
-              fi
-          fi
-      fi
-      EOF
+    + stdenv.lib.optionalString interactive import ./configure-bashrc.nix
+    +
+    ''
 
       ${runAsRoot}
     '';

release.nix

@@ -12,6 +12,10 @@ rec {
   };
 
   tests = {
+    multi-process-images = import ./tests/multi-process-images.nix {
+      inherit nixpkgs;
+    };
+
     webapps-agnostic = {
       config = import ./tests/webapps-agnostic-config.nix {
         inherit nixpkgs;
@@ -55,7 +59,10 @@ rec {
     name = "nix-processmgmt";
     constituents = builtins.attrValues tools
       ++ builtins.attrValues tests.webapps-agnostic
-      ++ [ tests.webapps-sysvinit ];
+      ++ [
+        tests.webapps-sysvinit
+        tests.multi-process-images
+      ];
     meta.description = "Release-critical builds";
   };
 }