Move some common operations to the util module, optimize testcases by caching their build results on the host system

nixproc/create-managed-process/agnostic/generate-docker-container.nix

@@ -25,6 +25,10 @@
 # nice unsupported
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
   generateForegroundProxy = import ./generate-foreground-proxy.nix {
     inherit stdenv writeTextFile;
   };
@@ -64,15 +68,16 @@ let
   # Instead, we mount the host system's Nix store so that the software is still accessible inside the container.
   cmdWithoutContext = map (arg: if builtins.isAttrs arg then builtins.unsafeDiscardStringContext arg else toString arg) cmd;
 
-  _path = basePackages ++ path;
-
-  _environment = {
-    PATH = builtins.concatStringsSep ":" (map(package: "${package}/bin" ) _path);
-  } // environment;
+  _environment = util.appendPathToEnvironment {
+    inherit environment;
+    path = basePackages ++ path;
+  };
 
   credentialsSpec = if credentials == {} || forceDisableUserChange then null else createCredentials credentials;
 
-  _user = if forceDisableUserChange then null else user;
+  _user = util.determineUser {
+    inherit user forceDisableUserChange;
+  };
 
   dockerImage = dockerTools.buildImage (stdenv.lib.recursiveUpdate {
     inherit name;

nixproc/create-managed-process/agnostic/generate-process-script.nix

@@ -28,6 +28,10 @@ in
 }:
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
   _path = basePackages ++ [ daemonPkg ] ++ path;
 
   _environment = {
@@ -43,7 +47,14 @@ let
           else "${tmpDir}/${instanceName}.pid"
     else pidFile;
 
-  _user = if forceDisableUserChange then null else user;
+  _user = util.determineUser {
+    inherit user forceDisableUserChange;
+  };
+
+  pidFilesDir = util.determinePIDFilesDir {
+    user = _user;
+    inherit runtimeDir tmpDir;
+  };
 in
 createProcessScript (stdenv.lib.recursiveUpdate ({
   inherit name dependencies credentials postInstall;
@@ -76,9 +87,15 @@ createProcessScript (stdenv.lib.recursiveUpdate ({
     ''
     + (if (daemon != null) then ''
       exec ${stdenv.lib.optionalString (_user != null) "su ${_user} -c '"} ${daemon} ${stdenv.lib.escapeShellArgs daemonArgs} ${stdenv.lib.optionalString (_user != null) "'"}
-    '' else if (foregroundProcess != null) then ''
-      exec daemon -U -i ${if _pidFile == null then "-P ${runtimeDir} -n $(basename ${foregroundProcess})" else "-F ${_pidFile}"} ${stdenv.lib.optionalString (nice != null) "-n ${nice}"} ${stdenv.lib.optionalString (_user != null) "-u ${_user}"} -- ${foregroundProcess} ${stdenv.lib.escapeShellArgs foregroundProcessArgs}
-    '' else throw "I don't know how to start this process!");
+    '' else if (foregroundProcess != null) then util.daemonizeForegroundProcess {
+      daemon = "daemon";
+      process = foregroundProcess;
+      args = foregroundProcessArgs;
+      pidFile = _pidFile;
+      user = _user;
+      inherit pidFilesDir;
+    }
+    else throw "I don't know how to start this process!");
   };
 } // stdenv.lib.optionalAttrs (_pidFile != null) {
   pidFile = _pidFile;

nixproc/create-managed-process/bsdrc/create-bsdrc-script.nix

@@ -80,15 +80,21 @@ name
 assert command == null -> commands ? start && commands ? stop;
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
   extraCommands = builtins.attrNames (removeAttrs commands builtinCommands);
 
-  _user = if forceDisableUserChange then null else user;
+  _user = util.determineUser {
+    inherit user forceDisableUserChange;
+  };
 
   _command = if commandIsDaemon then command else "daemon";
   _commandArgs = if commandIsDaemon then commandArgs else
-   stdenv.lib.optionals (pidFile != null) [ "-p" pidFile ]
-   ++ [ command ]
-   ++ commandArgs;
+    stdenv.lib.optionals (pidFile != null) [ "-p" pidFile ]
+    ++ [ command ]
+    ++ commandArgs;
 
   _requires = map (dependency: dependency.name) dependencies ++ requires;
 
@@ -188,7 +194,7 @@ let
     ) (builtins.attrNames commands)
     + stdenv.lib.optionalString (path != []) ''
 
-      PATH="${builtins.concatStringsSep ":" (map(package: "${package}/bin") path)}:$PATH"
+      PATH="${util.composePathEnvVariable { inherit path; }}"
       export PATH
     ''
     + "\n"

nixproc/create-managed-process/cygrunsrv/create-cygrunsrv-params.nix

@@ -42,9 +42,14 @@ name
 }:
 
 let
-  _environment = stdenv.lib.optionalAttrs (environmentPath != []) {
-    PATH = builtins.concatStringsSep ":" (map (package: "${package}/bin") environmentPath); # Augment path environment variable, if applicable
-  } // environment;
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
+  _environment = util.appendPathToEnvironment {
+    inherit environment;
+    path = environmentPath;
+  };
 
   cygrunsrvConfig = writeTextFile {
     name = "${prefix}${name}-cygrunsrv-params";

nixproc/create-managed-process/launchd/create-launchd-daemon.nix

@@ -23,9 +23,14 @@ name
 }@args:
 
 let
-  environment = stdenv.lib.optionalAttrs (path != []) {
-    PATH = builtins.concatStringsSep ":" (map (package: "${package}/bin") path); # Augment path environment variable, if applicable
-  } // stdenv.lib.mapAttrs (name: value: toString value) args.EnvironmentVariables or {}; # Convert all environment variables to strings
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
+  environment = util.appendPathToEnvironment {
+    environment = stdenv.lib.mapAttrs (name: value: toString value) args.EnvironmentVariables or {}; # Convert all environment variables to strings
+    inherit path;
+  };
 
   label = if args ? Label then args.Label else "${prefix}${name}";
 

nixproc/create-managed-process/supervisord/create-supervisord-program.nix

@@ -24,6 +24,10 @@ name
 }@params:
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
   properties = removeAttrs params ([ "name" "command" "useProxy" "pidFile" "path" "environment" "dependencies" "credentials" "postInstall" ] ++ stdenv.lib.optional forceDisableUserChange "user");
 
   priority = if dependencies == [] then 1
@@ -31,9 +35,10 @@ let
 
   _command = (stdenv.lib.optionalString useProxy "${supervisor}/bin/pidproxy ${runtimeDir}/${pidFile} ") + command;
 
-  _environment = {
-    PATH = builtins.concatStringsSep ":" (map (package: "${package}/bin") (basePackages ++ path));
-  } // environment;
+  _environment = util.appendPathToEnvironment {
+    inherit environment;
+    path = basePackages ++ path;
+  };
 
   confFile = writeTextFile {
     name = "${name}.conf";

nixproc/create-managed-process/systemd/create-systemd-service.nix

@@ -31,11 +31,16 @@ name
 }@args:
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
   sections = removeAttrs args [ "name" "environment" "dependencies" "path" "credentials" "postInstall" ];
 
-  _environment = {
-    PATH = builtins.concatStringsSep ":" (map (package: "${package}/bin") (basePackages ++ path));
-  } // environment;
+  _environment = util.appendPathToEnvironment {
+    inherit environment;
+    path = basePackages ++ path;
+  };
 
   generateEnvironmentVariables = environment:
     stdenv.lib.concatMapStrings (name:

nixproc/create-managed-process/sysvinit/create-sysvinit-script.nix

@@ -92,6 +92,13 @@ name
 }:
 
 let
+  util = import ../util {
+    inherit (stdenv) lib;
+  };
+
+  isCommonActivity = {activityName}:
+    activityName == "start" && activityName == "stop" && activityName == "reload" && activityName == "restart" && activityName == "status" && activityName == "*";
+
   # Enumerates the activities in a logical order -- the common activities first, then the remaining activities in alphabetical order
   enumerateActivities = activities:
     stdenv.lib.optional (activities ? start) "start"
@@ -99,10 +106,17 @@ let
     ++ stdenv.lib.optional (activities ? reload) "reload"
     ++ stdenv.lib.optional (activities ? restart) "restart"
     ++ stdenv.lib.optional (activities ? status) "status"
-    ++ builtins.filter (activityName: activityName != "start" && activityName != "stop" && activityName != "reload" && activityName != "restart" && activityName != "status" && activityName != "*") (builtins.attrNames activities)
+    ++ builtins.filter (activityName: !isCommonActivity { inherit activityName; }) (builtins.attrNames activities)
     ++ stdenv.lib.optional (activities ? "*") "*";
 
-  _user = if forceDisableUserChange then null else user;
+  _user = util.determineUser {
+    inherit user forceDisableUserChange;
+  };
+
+  pidFilesDir = util.determinePIDFilesDir {
+    user = _user;
+    inherit runtimeDir tmpDir;
+  };
 
   _instructions = (stdenv.lib.optionalAttrs (process != null) {
     start = {
@@ -110,7 +124,11 @@ let
       instruction =
         initialize +
         (if processIsDaemon then "${startDaemon} ${stdenv.lib.optionalString (pidFile != null) "-f -p ${pidFile}"} ${stdenv.lib.optionalString (nice != null) "-n ${nice}"} ${stdenv.lib.optionalString (_user != null) "$(type -p su) ${_user} -c '"}${process} ${stdenv.lib.escapeShellArgs args} ${stdenv.lib.optionalString (_user != null) "'"}"
-        else "${startProcessAsDaemon} -U -i ${if pidFile == null then "-P ${runtimeDir} -n $(basename ${process})" else "-F ${pidFile}"} ${stdenv.lib.optionalString (_user != null) "-u ${_user}"} -- ${process} ${toString args}");
+        else util.daemonizeForegroundProcess {
+          daemon = startProcessAsDaemon;
+          user = _user;
+          inherit process args pidFile pidFilesDir;
+        });
     };
     stop = {
       activity = "Stopping";
@@ -150,9 +168,9 @@ let
   _defaultStop = if runlevels != [] then stdenv.lib.subtractLists _defaultStart supportedRunlevels
     else defaultStop;
 
-  _environment = stdenv.lib.optionalAttrs (path != []) {
-    PATH = builtins.concatStringsSep ":" (map (package: "${package}/bin") path) + ":$PATH";
-  } // environment;
+  _environment = util.appendPathToEnvironment {
+    inherit environment path;
+  };
 
   initdScript = writeTextFile {
     inherit name;
@@ -182,14 +200,10 @@ let
     + stdenv.lib.optionalString (directory != null) ''
       cd ${directory}
     ''
-    + stdenv.lib.concatMapStrings (name:
-        let
-          value = builtins.getAttr name _environment;
-        in
-        ''
-          export ${name}=${stdenv.lib.escapeShellArg value}
-        ''
-      ) (builtins.attrNames _environment)
+    + util.printShellEnvironmentVariables {
+      environment = _environment;
+      allowSystemPath = true;
+    }
     + ''
 
       case "$1" in

nixproc/create-managed-process/util/default.nix

@@ -0,0 +1,69 @@
+{lib}:
+
+rec {
+  /*
+   * Composes a PATH environment variable from a collection of packages by
+   * translating their paths to bin/ sub folders
+   */
+  composePathEnvVariable = {path}:
+    builtins.concatStringsSep ":" (map (package: "${package}/bin") path);
+
+  /*
+   * Appends the bin/ sub folders in the packages in path as a PATH environment
+   * variable to the environment.
+   */
+  appendPathToEnvironment = {environment, path}:
+    lib.optionalAttrs (path != []) {
+      PATH = composePathEnvVariable {
+        inherit path;
+      };
+    } // environment;
+
+  /*
+   * Prints escaped export statements that configure environment variables
+   *
+   * Parameters:
+   * environment: attribute set in which the keys are the environment variables names and the values to environment variable values
+   * allowSystemPath: whether to allow access to the original system PATH
+   */
+  printShellEnvironmentVariables = {environment, allowSystemPath ? true}:
+    lib.concatMapStrings (name:
+      let
+        value = builtins.getAttr name environment;
+      in
+      ''
+        export ${name}=${lib.escapeShellArg value}${lib.optionalString (allowSystemPath && name == "PATH") ":$PATH"}
+      ''
+    ) (builtins.attrNames environment);
+
+  /*
+   * Determines the actual user name
+   */
+  determineUser = {user, forceDisableUserChange}:
+    if forceDisableUserChange then null else user;
+
+  /*
+   * Determines the preferred directory in which PID files should be stored.
+   * For privileged users it is in the runtime dir, unprivileged users use the
+   * temp dir.
+   */
+  determinePIDFilesDir = {user, runtimeDir, tmpDir}:
+    if user == null then runtimeDir else tmpDir;
+
+  /*
+   * Auto-generates the path to the preferred PID file if none has been
+   * specified.
+   */
+  autoGeneratePIDFilePath = {pidFile, instanceName, pidFilesDir}:
+    if pidFile == null then "${pidFilesDir}/${instanceName}.pid" else pidFile;
+
+  /*
+   * Creates a shell command invocation that deamonizes a foreground process by
+   * using libslack's daemon command.
+   */
+  daemonizeForegroundProcess = {daemon, process, args, pidFile ? null, pidFilesDir, user ? null}:
+    "${daemon} --unsafe --inherit"
+    + (if pidFile == null then " --pidfiles ${pidFilesDir} --name $(basename ${process})" else " --pidfile ${pidFile}")
+    + lib.optionalString (user != null) " --user ${user}"
+    + " -- ${process} ${lib.escapeShellArgs args}";
+}

tests/webapps-agnostic-docker.nix

@@ -54,16 +54,6 @@ makeTest {
       virtualisation.memorySize = 8192;
       virtualisation.diskSize = 4096;
 
-      users.extraUsers = {
-        webapp = {
-          uid = 1000;
-          group = "users";
-          shell = "/bin/sh";
-          description = "Unprivileged user";
-          home = "/var/empty";
-        };
-      };
-
       # We can't download any substitutes in a test environment. To make tests
       # faster, we disable substitutes so that Nix does not waste any time by
       # attempting to download them.
@@ -114,6 +104,11 @@ makeTest {
 
     machine.wait_for_unit("nix-process-docker")
 
+    machine.succeed("mkdir -p /home/sbu")
+    machine.succeed(
+        "cp ${/home/sbu/dysnomia-0.10pre1234.tar.gz} /home/sbu/dysnomia-0.10pre1234.tar.gz"
+    )
+
     # Deploy the system with foreground webapp processes
 
     machine.succeed(

tests/webapps-agnostic-supervisord.nix

@@ -25,6 +25,14 @@ let
     exprFile = ../examples/webapps-agnostic/processes.nix;
   };
 
+  processesEnvAdvanced = import ../nixproc/create-managed-process/supervisord/build-supervisord-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-advanced.nix;
+  };
+
+  processesEnvEmpty = import ../nixproc/create-managed-process/supervisord/build-supervisord-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-empty.nix;
+  };
+
   tools = import ../tools {};
 
   nix-processmgmt = ./..;
@@ -36,21 +44,10 @@ makeTest {
     {pkgs, ...}:
 
     {
-      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ supervisordProcessEnv processesEnvForeground processesEnvDaemon processesEnvAuto ];
+      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ supervisordProcessEnv processesEnvForeground processesEnvDaemon processesEnvAuto processesEnvAdvanced processesEnvEmpty ];
       virtualisation.writableStore = true;
       virtualisation.memorySize = 1024;
 
-      users.extraUsers = {
-        unprivileged = {
-          uid = 1000;
-          group = "users";
-          shell = "/bin/sh";
-          description = "Unprivileged user";
-          home = "/home/unprivileged";
-          createHome = true;
-        };
-      };
-
       # We can't download any substitutes in a test environment. To make tests
       # faster, we disable substitutes so that Nix does not waste any time by
       # attempting to download them.

tests/webapps-agnostic-systemd.nix

@@ -21,6 +21,14 @@ let
     exprFile = ../examples/webapps-agnostic/processes.nix;
   };
 
+  processesEnvAdvanced = import ../nixproc/create-managed-process/systemd/build-systemd-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-advanced.nix;
+  };
+
+  processesEnvEmpty = import ../nixproc/create-managed-process/systemd/build-systemd-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-empty.nix;
+  };
+
   tools = import ../tools {};
 
   nix-processmgmt = ./..;
@@ -32,21 +40,10 @@ makeTest {
     {pkgs, ...}:
 
     {
-      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ processesEnvForeground processesEnvDaemon processesEnvAuto ];
+      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ processesEnvForeground processesEnvDaemon processesEnvAuto processesEnvAdvanced processesEnvEmpty ];
       virtualisation.writableStore = true;
       virtualisation.memorySize = 1024;
 
-      users.extraUsers = {
-        unprivileged = {
-          uid = 1000;
-          group = "users";
-          shell = "/bin/sh";
-          description = "Unprivileged user";
-          home = "/home/unprivileged";
-          createHome = true;
-        };
-      };
-
       # We can't download any substitutes in a test environment. To make tests
       # faster, we disable substitutes so that Nix does not waste any time by
       # attempting to download them.

tests/webapps-agnostic-sysvinit.nix

@@ -28,6 +28,32 @@ let
     exprFile = ../examples/webapps-agnostic/processes.nix;
   };
 
+  processesEnvUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes.nix;
+    stateDir = "/home/unprivileged/var";
+    forceDisableUserChange = true;
+  };
+
+  processesEnvAdvanced = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-advanced.nix;
+  };
+
+  processesEnvAdvancedUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-advanced.nix;
+    stateDir = "/home/unprivileged/var";
+    forceDisableUserChange = true;
+  };
+
+  processesEnvEmpty = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-empty.nix;
+  };
+
+  processesEnvEmptyUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-agnostic/processes-empty.nix;
+    stateDir = "/home/unprivileged/var";
+    forceDisableUserChange = true;
+  };
+
   tools = import ../tools {};
 
   nix-processmgmt = ./..;
@@ -39,7 +65,18 @@ makeTest {
     {pkgs, ...}:
 
     {
-      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ webappUnprivilegedForegroundMode webappUnprivilegedDaemonMode webappUnprivilegedAutoMode processesEnv ];
+      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [
+        webappUnprivilegedForegroundMode
+        webappUnprivilegedDaemonMode
+        webappUnprivilegedAutoMode
+        processesEnv
+        processesEnvUnprivileged
+        processesEnvAdvanced
+        processesEnvAdvancedUnprivileged
+        processesEnvEmpty
+        processesEnvEmptyUnprivileged
+      ];
+
       virtualisation.writableStore = true;
 
       users.extraUsers = {

tests/webapps-sysvinit.nix

@@ -12,6 +12,32 @@ let
     exprFile = ../examples/webapps-sysvinit/processes.nix;
   };
 
+  processesEnvUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-sysvinit/processes.nix;
+    forceDisableUserChange = true;
+    stateDir = "/home/unprivileged/var";
+  };
+
+  processesEnvAdvanced = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-sysvinit/processes-advanced.nix;
+  };
+
+  processesEnvAdvancedUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-sysvinit/processes-advanced.nix;
+    forceDisableUserChange = true;
+    stateDir = "/home/unprivileged/var";
+  };
+
+  processesEnvEmpty = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-sysvinit/processes-empty.nix;
+  };
+
+  processesEnvEmptyUnprivileged = import ../nixproc/create-managed-process/sysvinit/build-sysvinit-env.nix {
+    exprFile = ../examples/webapps-sysvinit/processes-empty.nix;
+    forceDisableUserChange = true;
+    stateDir = "/home/unprivileged/var";
+  };
+
   tools = import ../tools {};
 
   nix-processmgmt = ./..;
@@ -23,7 +49,16 @@ makeTest {
     {pkgs, ...}:
 
     {
-      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [ webappUnprivileged processesEnv ];
+      virtualisation.pathsInNixDB = [ pkgs.stdenv ] ++ pkgs.coreutils.all ++ [
+        webappUnprivileged
+        processesEnv
+        processesEnvUnprivileged
+        processesEnvAdvanced
+        processesEnvAdvancedUnprivileged
+        processesEnvEmpty
+        processesEnvEmptyUnprivileged
+      ];
+
       virtualisation.writableStore = true;
 
       users.extraUsers = {