From d347c04fd613cd3188b0c8765d64f2fa690b6943 Mon Sep 17 00:00:00 2001 From: Sander van der Burg Date: Thu, 11 Feb 2021 23:23:02 +0100 Subject: [PATCH] Initial implementation of a buildImage function with configurable build steps, and an implementation of a mutable image building function --- README.md | 8 ++- examples/multi-process-image/default.nix | 5 +- .../disnix/generate-disnix-image-args.nix | 46 ------------ .../disnix/image-steps/disnix-dynamic.nix | 40 +++++++++++ .../disnix/image-steps/disnix-static.nix | 33 +++++++++ .../backends/disnix/image-steps/disnix.nix | 5 ++ .../disnix/image-steps/dynamic-steps.nix | 5 ++ .../disnix/image-steps/static-steps.nix | 5 ++ .../s6-rc/image-steps/dynamic-steps.nix | 4 ++ .../s6-rc/image-steps/s6-rc-dynamic.nix | 19 +++++ .../s6-rc/image-steps/s6-rc-static.nix | 22 ++++++ .../s6-rc.nix} | 23 +++--- .../s6-rc/image-steps/static-steps.nix | 4 ++ .../generate-supervisord-image-args.nix | 15 ---- .../supervisord/image-steps/dynamic-steps.nix | 4 ++ .../supervisord/image-steps/static-steps.nix | 4 ++ .../image-steps/supervisord-dynamic.nix | 21 ++++++ .../image-steps/supervisord-static.nix | 20 ++++++ .../supervisord/image-steps/supervisord.nix | 8 +++ .../sysvinit/image-steps/dynamic-steps.nix | 5 ++ .../sysvinit/image-steps/static-steps.nix | 5 ++ .../sysvinit/image-steps/sysvinit-dynamic.nix | 29 ++++++++ .../sysvinit/image-steps/sysvinit-static.nix | 37 ++++++++++ .../sysvinit/image-steps/sysvinit.nix | 5 ++ .../create-image-from-steps.nix | 8 +++ .../create-multi-process-image-universal.nix | 13 ++++ .../create-multi-process-image.nix | 30 ++++++++ ...-mutable-multi-process-image-universal.nix | 13 ++++ .../create-mutable-multi-process-image.nix | 31 ++++++++ .../create-nix-image.nix | 18 +++++ .../generate-config-from-steps.nix | 7 ++ .../create-image-from-steps/steps/basic.nix | 18 +++++ .../steps/bootstrap-init.nix | 31 ++++++++ .../create-image-from-steps/steps/init.nix | 7 ++ .../steps/interactive.nix | 37 ++++++++++ nixproc/create-image-from-steps/steps/man.nix | 11 +++ .../steps/nix-processmgmt-dynamic.nix | 70 +++++++++++++++++++ .../steps/nix-processmgmt-static.nix | 14 ++++ .../steps/nix-support.nix | 59 ++++++++++++++++ .../create-image-from-steps/steps/su-pam.nix | 19 +++++ .../configure-bashrc.nix | 26 ------- .../create-multi-process-image-universal.nix | 11 --- .../create-multi-process-image.nix | 55 --------------- 43 files changed, 677 insertions(+), 173 deletions(-) delete mode 100644 nixproc/backends/disnix/generate-disnix-image-args.nix create mode 100644 nixproc/backends/disnix/image-steps/disnix-dynamic.nix create mode 100644 nixproc/backends/disnix/image-steps/disnix-static.nix create mode 100644 nixproc/backends/disnix/image-steps/disnix.nix create mode 100644 nixproc/backends/disnix/image-steps/dynamic-steps.nix create mode 100644 nixproc/backends/disnix/image-steps/static-steps.nix create mode 100644 nixproc/backends/s6-rc/image-steps/dynamic-steps.nix create mode 100644 nixproc/backends/s6-rc/image-steps/s6-rc-dynamic.nix create mode 100644 nixproc/backends/s6-rc/image-steps/s6-rc-static.nix rename nixproc/backends/s6-rc/{generate-s6-rc-image-args.nix => image-steps/s6-rc.nix} (67%) create mode 100644 nixproc/backends/s6-rc/image-steps/static-steps.nix delete mode 100644 nixproc/backends/supervisord/generate-supervisord-image-args.nix create mode 100644 nixproc/backends/supervisord/image-steps/dynamic-steps.nix create mode 100644 nixproc/backends/supervisord/image-steps/static-steps.nix create mode 100644 nixproc/backends/supervisord/image-steps/supervisord-dynamic.nix create mode 100644 nixproc/backends/supervisord/image-steps/supervisord-static.nix create mode 100644 nixproc/backends/supervisord/image-steps/supervisord.nix create mode 100644 nixproc/backends/sysvinit/image-steps/dynamic-steps.nix create mode 100644 nixproc/backends/sysvinit/image-steps/static-steps.nix create mode 100644 nixproc/backends/sysvinit/image-steps/sysvinit-dynamic.nix create mode 100644 nixproc/backends/sysvinit/image-steps/sysvinit-static.nix create mode 100644 nixproc/backends/sysvinit/image-steps/sysvinit.nix create mode 100644 nixproc/create-image-from-steps/create-image-from-steps.nix create mode 100644 nixproc/create-image-from-steps/create-multi-process-image-universal.nix create mode 100644 nixproc/create-image-from-steps/create-multi-process-image.nix create mode 100644 nixproc/create-image-from-steps/create-mutable-multi-process-image-universal.nix create mode 100644 nixproc/create-image-from-steps/create-mutable-multi-process-image.nix create mode 100644 nixproc/create-image-from-steps/create-nix-image.nix create mode 100644 nixproc/create-image-from-steps/generate-config-from-steps.nix create mode 100644 nixproc/create-image-from-steps/steps/basic.nix create mode 100644 nixproc/create-image-from-steps/steps/bootstrap-init.nix create mode 100644 nixproc/create-image-from-steps/steps/init.nix create mode 100644 nixproc/create-image-from-steps/steps/interactive.nix create mode 100644 nixproc/create-image-from-steps/steps/man.nix create mode 100644 nixproc/create-image-from-steps/steps/nix-processmgmt-dynamic.nix create mode 100644 nixproc/create-image-from-steps/steps/nix-processmgmt-static.nix create mode 100644 nixproc/create-image-from-steps/steps/nix-support.nix create mode 100644 nixproc/create-image-from-steps/steps/su-pam.nix delete mode 100644 nixproc/create-multi-process-image/configure-bashrc.nix delete mode 100644 nixproc/create-multi-process-image/create-multi-process-image-universal.nix delete mode 100644 nixproc/create-multi-process-image/create-multi-process-image.nix diff --git a/README.md b/README.md index 39c35d3..12fb9a5 100644 --- a/README.md +++ b/README.md @@ -935,9 +935,8 @@ To construct such as an image, we can evaluate a Nix expression (e.g. }: let - createMultiProcessImage = import ../../nixproc/create-multi-process-image/create-multi-process-image.nix { - inherit pkgs system; - inherit (pkgs) dockerTools stdenv; + createMultiProcessImage = import ../../nixproc/create-image-from-steps/create-multi-process-image-universal.nix { + inherit pkgs; }; in createMultiProcessImage { @@ -946,6 +945,7 @@ createMultiProcessImage { exprFile = ../webapps-agnostic/processes.nix; processManager = "supervisord"; # sysvinit, disnix, s6-rc are also valid options interactive = true; # the default option + manpages = false; # the default option forceDisableUserChange = false; # the default option } ``` @@ -964,6 +964,8 @@ with the following parameters: this setting has been enabled, a `.bashrc` will be configured to make the bash shell usable, and a number of additional packages will be installed for file and process management operations. +* We can also optionally install `man` in the container so that you can access + manual pages. By default, it is disabled * It is also possible to adjust the state settings in the processes model. With `forceDisableUserChange` we can disable user creation and user switching. It is also possible to control the other state variables, such diff --git a/examples/multi-process-image/default.nix b/examples/multi-process-image/default.nix index b74cca1..c4ad05b 100644 --- a/examples/multi-process-image/default.nix +++ b/examples/multi-process-image/default.nix @@ -5,9 +5,8 @@ }: let - createMultiProcessImage = import ../../nixproc/create-multi-process-image/create-multi-process-image-universal.nix { - inherit pkgs system; - inherit (pkgs) dockerTools stdenv; + createMultiProcessImage = import ../../nixproc/create-image-from-steps/create-multi-process-image-universal.nix { + inherit pkgs; }; in createMultiProcessImage { diff --git a/nixproc/backends/disnix/generate-disnix-image-args.nix b/nixproc/backends/disnix/generate-disnix-image-args.nix deleted file mode 100644 index a6d6fd6..0000000 --- a/nixproc/backends/disnix/generate-disnix-image-args.nix +++ /dev/null @@ -1,46 +0,0 @@ -{pkgs, system, exprFile, stateDir, runtimeDir, forceDisableUserChange, extraParams}: - -let - sysvinitTools = (import ../../../tools { - inherit pkgs system; - }).sysvinit; - - generateCompoundProxy = import ../util/generate-compound-proxy.nix { - inherit (pkgs) stdenv writeTextFile; - }; - - disnixDataDir = "${pkgs.disnix}/share/disnix"; - - profile = import ./build-disnix-env.nix { - inherit pkgs system exprFile stateDir runtimeDir forceDisableUserChange extraParams disnixDataDir; - }; - - emptyProfile = import ./build-disnix-env.nix { - inherit pkgs system stateDir runtimeDir forceDisableUserChange extraParams disnixDataDir; - exprFile = null; - }; - - script = generateCompoundProxy { - path = [ pkgs.dysnomia pkgs.disnix ]; - startCommand = "disnix-activate ${profile}"; - stopCommand = "disnix-activate -o ${profile} ${emptyProfile}"; - }; -in -{ - runAsRoot = pkgs.lib.optionalString (!forceDisableUserChange) '' - ${pkgs.gnused}/bin/sed -i -e "s/CREATE_MAIL_SPOOL=yes/CREATE_MAIL_SPOOL=no/" /etc/default/useradd - - mkdir -p /etc/pam.d - cat > /etc/pam.d/su < /bin/bootstrap < /bin/bootstrap < /root/.bashrc << "EOF" + alias ls='ls --color=auto' + + if [ -n "$PS1" ] + then + if [ "$TERM" != "dumb" -o -n "$INSIDE_EMACS" ] + then + PROMPT_COLOR="1;31m" + let $UID && PROMPT_COLOR="1;32m" + if [ -n "$INSIDE_EMACS" -o "$TERM" == "eterm" -o "$TERM" == "eterm-color" ] + then + # Emacs term mode doesn't support xterm title escape sequence (\e]0;) + PS1="\n\[\033[$PROMPT_COLOR\][\u@\h:\w]\\$\[\033[0m\] " + else + PS1="\n\[\033[$PROMPT_COLOR\][\[\e]0;\u@\h: \w\a\]\u@\h:\w]\\$\[\033[0m\] " + fi + if test "$TERM" = "xterm" + then + PS1="\[\033]2;\h:\u:\w\007\]$PS1" + fi + fi + fi + EOF + ''; +} diff --git a/nixproc/create-image-from-steps/steps/man.nix b/nixproc/create-image-from-steps/steps/man.nix new file mode 100644 index 0000000..d38f54b --- /dev/null +++ b/nixproc/create-image-from-steps/steps/man.nix @@ -0,0 +1,11 @@ +{pkgs, common, input, result}: + +result // pkgs.lib.optionalAttrs (input ? manpages && input.manpages) { + contents = result.contents ++ (with pkgs; [ + man groff gzip + ]); + + config = result.config or {} // { + Env = result.config.Env or [] ++ [ "MANPATH=/share/man:/nix/var/nix/profiles/default/share/man" ]; + }; +} diff --git a/nixproc/create-image-from-steps/steps/nix-processmgmt-dynamic.nix b/nixproc/create-image-from-steps/steps/nix-processmgmt-dynamic.nix new file mode 100644 index 0000000..6860832 --- /dev/null +++ b/nixproc/create-image-from-steps/steps/nix-processmgmt-dynamic.nix @@ -0,0 +1,70 @@ +{pkgs, common, input, result}: + +let + commonTools = (import ../../../tools { + inherit pkgs; + inherit (common) system; + }).common; + + # If no processes.nix parameter was provided, generate a template + templateFile = pkgs.writeTextFile { + name = "processes.nix"; + text = '' + { pkgs ? import { inherit system; } + , system ? builtins.currentSystem + , stateDir ? "/var" + , runtimeDir ? "''${stateDir}/run" + , logDir ? "''${stateDir}/log" + , cacheDir ? "''${stateDir}/cache" + , tmpDir ? (if stateDir == "/var" then "/tmp" else "''${stateDir}/tmp") + , forceDisableUserChange ? false + , processManager + }: + + let + nix-processmgmt-services = builtins.fetchGit { + url = https://github.com/svanderburg/nix-processmgmt-services.git; + ref = "master"; + }; + + sharedConstructors = import "''${nix-processmgmt-services}/examples/services-agnostic/constructors.nix" { + inherit pkgs stateDir runtimeDir logDir cacheDir tmpDir forceDisableUserChange processManager; + }; + in + rec { + /*nginx = rec { + port = 8080; + + pkg = sharedConstructors.nginxReverseProxyHostBased { + webapps = []; + inherit port; + } {}; + };*/ + } + ''; + }; +in +result // { + contents = result.contents or [] + ++ [ pkgs.dysnomia commonTools ]; + + runAsRoot = result.runAsRoot or "" + '' + nixproc-init-state --state-dir ${input.stateDir} --runtime-dir ${input.runtimeDir} + + # Provide a processes.nix expression + mkdir -p /etc/nixproc + '' + (if input ? exprFile then '' + cp ${input.exprFile} /etc/nixproc/processes.nix + '' else '' + cp ${templateFile} /etc/nixproc/processes.nix + '') + + '' + chmod 644 /etc/nixproc/processes.nix + ''; + + config = result.config or {} // { + Env = result.config.Env or [] ++ [ + "NIXPROC_PROCESSES=/etc/nixproc/processes.nix" + ]; + }; +} diff --git a/nixproc/create-image-from-steps/steps/nix-processmgmt-static.nix b/nixproc/create-image-from-steps/steps/nix-processmgmt-static.nix new file mode 100644 index 0000000..283872d --- /dev/null +++ b/nixproc/create-image-from-steps/steps/nix-processmgmt-static.nix @@ -0,0 +1,14 @@ +{pkgs, common, input, result}: + +let + commonTools = (import ../../../tools { + inherit pkgs; + inherit (common) system; + }).common; +in +result // { + runAsRoot = result.runAsRoot or "" + '' + # Initialize common state directories + ${commonTools}/bin/nixproc-init-state --state-dir ${input.stateDir} --runtime-dir ${input.runtimeDir} + ''; +} diff --git a/nixproc/create-image-from-steps/steps/nix-support.nix b/nixproc/create-image-from-steps/steps/nix-support.nix new file mode 100644 index 0000000..d66c474 --- /dev/null +++ b/nixproc/create-image-from-steps/steps/nix-support.nix @@ -0,0 +1,59 @@ +{pkgs, common, input, result}: + +let + mkDbExtraCommand = contents: let + contentsList = if builtins.isList contents then pkgs.lib.unique contents else [ contents ]; in + '' + echo "Generating the nix database..." + echo "Warning: only the database of the deepest Nix layer is loaded." + echo " If you want to use nix commands in the container, it would" + echo " be better to only have one layer that contains a nix store." + + export NIX_REMOTE=local?root=$PWD + # A user is required by nix + # https://github.com/NixOS/nix/blob/9348f9291e5d9e4ba3c4347ea1b235640f54fd79/src/libutil/util.cc#L478 + export USER=nobody + ${pkgs.nix}/bin/nix-store --load-db < ${pkgs.closureInfo {rootPaths = contentsList;}}/registration + + mkdir -p nix/var/nix/gcroots/docker/ + for i in ${pkgs.lib.concatStringsSep " " contentsList} + do + ln -s $i nix/var/nix/gcroots/docker/$(basename $i) + done; + ''; +in +result // rec { + contents = result.contents or [] ++ (with pkgs; [ + # Nix + nix.out + # Needed for SSL authentication + cacert + # Needed for fetchgit + git + # Needed for downloading compressed tarballs + gnutar gzip bzip2 xz + ]); + + extraCommands = result.extraCommands or "" + + mkDbExtraCommand contents; + + runAsRoot = result.runAsRoot or "" + '' + # Initialize groups for Nix + groupadd -g 30000 nixbld + for i in $(seq 1 30) + do + groupadd -g $((30000 + i)) nixbld$i + useradd -d /var/empty -c "Nix build user $i" -u $((30000 + i)) -g nixbld$i -G nixbld nixbld$i + done + ''; + + config = result.config or {} // { + Env = result.config.Env or [] ++ [ + "NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels" + "USER=root" + "PATH=/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin" + ]; + }; +} diff --git a/nixproc/create-image-from-steps/steps/su-pam.nix b/nixproc/create-image-from-steps/steps/su-pam.nix new file mode 100644 index 0000000..a0ba171 --- /dev/null +++ b/nixproc/create-image-from-steps/steps/su-pam.nix @@ -0,0 +1,19 @@ +{pkgs, common, input, result}: + +result // { + runAsRoot = result.runAsRoot or "" + '' + ${pkgs.gnused}/bin/sed -i -e "s/CREATE_MAIL_SPOOL=yes/CREATE_MAIL_SPOOL=no/" /etc/default/useradd + + mkdir -p /etc/pam.d + cat > /etc/pam.d/su < /root/.bashrc << "EOF" - alias ls='ls --color=auto' - - if [ -n "$PS1" ] - then - if [ "$TERM" != "dumb" -o -n "$INSIDE_EMACS" ] - then - PROMPT_COLOR="1;31m" - let $UID && PROMPT_COLOR="1;32m" - if [ -n "$INSIDE_EMACS" -o "$TERM" == "eterm" -o "$TERM" == "eterm-color" ] - then - # Emacs term mode doesn't support xterm title escape sequence (\e]0;) - PS1="\n\[\033[$PROMPT_COLOR\][\u@\h:\w]\\$\[\033[0m\] " - else - PS1="\n\[\033[$PROMPT_COLOR\][\[\e]0;\u@\h: \w\a\]\u@\h:\w]\\$\[\033[0m\] " - fi - if test "$TERM" = "xterm" - then - PS1="\[\033]2;\h:\u:\w\007\]$PS1" - fi - fi - fi - EOF -'' diff --git a/nixproc/create-multi-process-image/create-multi-process-image-universal.nix b/nixproc/create-multi-process-image/create-multi-process-image-universal.nix deleted file mode 100644 index f4482dc..0000000 --- a/nixproc/create-multi-process-image/create-multi-process-image-universal.nix +++ /dev/null @@ -1,11 +0,0 @@ -{dockerTools, stdenv, pkgs, system}: - -import ./create-multi-process-image.nix { - inherit dockerTools stdenv pkgs system; - generators = { - disnix = ../backends/disnix/generate-disnix-image-args.nix; - s6-rc = ../backends/s6-rc/generate-s6-rc-image-args.nix; - supervisord = ../backends/supervisord/generate-supervisord-image-args.nix; - sysvinit = ../backends/sysvinit/generate-sysvinit-image-args.nix; - }; -} diff --git a/nixproc/create-multi-process-image/create-multi-process-image.nix b/nixproc/create-multi-process-image/create-multi-process-image.nix deleted file mode 100644 index 9cffa90..0000000 --- a/nixproc/create-multi-process-image/create-multi-process-image.nix +++ /dev/null @@ -1,55 +0,0 @@ -{dockerTools, stdenv, pkgs, system, generators}: - -{ interactive ? true -, exprFile -, extraParams ? {} -, contents ? [] -, runAsRoot ? "" -, config ? {} -, processManager -, stateDir ? "/var" -, runtimeDir ? "${stateDir}/run" -, forceDisableUserChange ? false -, ... -}@args: - -let - # Determine which parameters can be propagated to buildImage and which are customizations - buildImageFormalArgs = builtins.functionArgs dockerTools.buildImage; - buildImageArgs = removeAttrs (builtins.intersectAttrs buildImageFormalArgs args) [ "contents" "runAsRoot" "config" ]; - - commonTools = (import ../../tools { inherit pkgs; }).common; - - generateImageArgsModule = if builtins.hasAttr processManager generators - then builtins.getAttr processManager generators - else throw "Cannot use process manager: ${processManager} in a multi-process container!"; - - processManagerArgs = import generateImageArgsModule { - inherit exprFile stateDir runtimeDir forceDisableUserChange extraParams pkgs system; - }; - - setupProcessManagement = import ../backends/docker/setup.nix { - inherit (pkgs) dockerTools stdenv dysnomia findutils glibc; - inherit (processManagerArgs) credentialsSpec; - inherit commonTools stateDir runtimeDir forceDisableUserChange; - }; -in -dockerTools.buildImage ({ - contents = stdenv.lib.optionals interactive [ pkgs.glibc.bin pkgs.bashInteractive pkgs.coreutils pkgs.gnugrep pkgs.findutils pkgs.procps pkgs.utillinux pkgs.less ] - ++ processManagerArgs.contents - ++ contents; - - runAsRoot = - setupProcessManagement - + processManagerArgs.runAsRoot - + stdenv.lib.optionalString interactive import ./configure-bashrc.nix - + - '' - - ${runAsRoot} - ''; - - config = stdenv.lib.recursiveUpdate { - Cmd = processManagerArgs.cmd; - } config; -} // buildImageArgs)