Native evaluation (rather than by subprocess)

nix_actor.nimble

@@ -1,4 +1,4 @@
-version = "20230611"
+version = "20230613"
 author = "Emery Hemingway"
 description = "Syndicated Nix Actor"
 license = "Unlicense"

src/Tupfile

@@ -1,3 +1,3 @@
 include_rules
-: nix_actor.nim | $(SYNDICATE_PROTOCOL) ./<protocol> |> !nim_bin |> {bin}
-: {bin} |> !assert_built |>
+: foreach *.nim | $(SYNDICATE_PROTOCOL) ./<protocol> |> !nim_bin |> {bin}
+: foreach {bin} |> !assert_built |>

src/nix_actor.nim

@@ -1,19 +1,57 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[json, osproc, parseutils, strutils, tables]
+import std/[json, os, osproc, parseutils, strutils, tables]
 import eris/memory_stores
 import preserves, preserves/jsonhooks
 import syndicate
 from syndicate/protocols/dataspace import Observe
-import ./nix_actor/protocol
 import ./nix_actor/[clients, daemons]
-
+import ./nix_actor/libnix/[libexpr, main, store]
+import ./nix_actor/protocol
 
 type
   Value = Preserve[void]
   Observe = dataspace.Observe[Ref]
 
+proc toPreserve(val: libexpr.ValueObj | libExpr.ValuePtr; state: EvalState; E = void): Preserve[E] {.gcsafe.} =
+  ## Convert a Nix value to a Preserves value.
+  case val.kind
+  of nThunk:
+    result = initRecord[E]("thunk")
+  of nInt:
+    result = val.integer.toPreserve(E)
+  of nFloat:
+    result = val.fpoint.toPreserve(E)
+  of nBool:
+    result = val.boolean.toPreserve(E)
+  of nString:
+    result = val.shallowString.toPreserve(E)
+  of nPath:
+    result = toSymbol($val.path, E)
+  of nNull:
+    result = initRecord[E]("null")
+  of nAttrs:
+    result = initDictionary(E)
+    for key, val in val.pairs:
+      result[symbolString(state, key).toSymbol(E)] = val.toPreserve(state, E)
+  of nList:
+    result = initSequence(0, E)
+    for e in val.items:
+      result.sequence.add(e.toPreserve(state, E))
+  of nFunction:
+    result = initRecord[E]("func")
+  of nExternal:
+    result = initRecord[E]("external")
+
+proc eval(state: EvalState; code: string): Value =
+  ## Evaluate Nix `code` to a Preserves value.
+  var nixVal: libexpr.ValueObj
+  let expr = state.parseExprFromString(code, getCurrentDir())
+  state.eval(expr, nixVal)
+  state.forceValueDeep(nixVal)
+  nixVal.toPreserve(state, void)
+
 proc parseArgs(args: var seq[string]; opts: AttrSet) =
   for sym, val in opts:
     add(args, "--" & $sym)
@@ -71,15 +109,6 @@ proc instantiate(instantiate: Instantiate): Value =
   var execOutput = strip execProcess(cmd, args = args, options = {poUsePath})
   execOutput.toPreserve
 
-proc eval(eval: Eval): Value =
-  const cmd = "nix"
-  var args = @["eval", "--expr", eval.expr]
-  parseArgs(args, eval.options)
-  var execOutput = strip execProcess(cmd, args = args, options = {poUsePath})
-  if execOutput != "":
-    var js = parseJson(execOutput)
-    result = js.toPreserve
-
 proc bootNixFacet(turn: var Turn; ds: Ref): Facet =
   # let store = openStore()
   result = inFacet(turn) do (turn: var Turn):
@@ -100,14 +129,6 @@ proc bootNixFacet(turn: var Turn; ds: Ref): Facet =
         ass.result = instantiate(ass)
         discard publish(turn, ds, ass)
 
-    during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Value):
-      var ass = Eval(expr: e)
-      if not fromPreserve(ass.options, unpackLiterals(o)):
-        stderr.writeLine "invalid options ", o
-      else:
-        ass.result = eval(ass)
-        discard publish(turn, ds, ass)
-
     #[
     during(turn, ds, ?Observe(pattern: !Narinfo) ?? {0: grabLit()}) do (path: string):
       narinfo(turn, ds, path)
@@ -121,15 +142,28 @@ type
   DaemonSideArgs {.preservesDictionary.} = object
     `daemon-socket`: string
 
+main.initNix()
+libexpr.initGC()
+
 runActor("main") do (root: Ref; turn: var Turn):
-  let store = newMemoryStore()
+  let
+    erisStore = newMemoryStore()
+    nixStore = openStore()
+    nixState = newEvalState(nixStore)
   connectStdio(root, turn)
 
   during(turn, root, ?RefArgs) do (ds: Ref):
     discard bootNixFacet(turn, ds)
 
+    during(turn, ds, ?Observe(pattern: !Eval) ?? {0: grabLit(), 1: grabDict()}) do (e: string, o: Assertion):
+      var ass = Eval(expr: e)
+      doAssert fromPreserve(ass.options, unpackLiterals(o))
+        # unused options
+      ass.result = eval(nixState, ass.expr)
+      discard publish(turn, ds, ass)
+
     during(turn, root, ?ClientSideArgs) do (socketPath: string):
-      bootClientSide(turn, ds, store, socketPath)
+      bootClientSide(turn, ds, erisStore, socketPath)
 
     during(turn, root, ?DaemonSideArgs) do (socketPath: string):
-      bootDaemonSide(turn, ds, store, socketPath)
+      bootDaemonSide(turn, ds, erisStore, socketPath)

src/nix_actor.nim.cfg

@@ -0,0 +1 @@
+backend:cpp

src/nix_actor/clients.nim

@@ -62,7 +62,9 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy
     await send(client, "0.0.0")
     await sendWorkEnd(client)
   while not client.socket.isClosed:
-    let wop = await recvWord(client.socket)
+    let
+      w = await recvWord(client.socket)
+      wop = WorkerOperation(w)
     case wop
 
     of wopAddToStore:
@@ -147,7 +149,7 @@ proc serveClient(facet: Facet; ds: Ref; store: ErisStore; client: Session) {.asy
       # all options from the client are ingored
 
     else:
-      let msg = "unhandled worker op " & $wop.int
+      let msg = "unhandled worker op " & $wop
       await sendNext(client, msg)
       await sendWorkEnd(client)
       close(client.socket)

src/nix_actor/daemons.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[asyncdispatch, asyncnet, sets, streams, strutils]
+import std/[asyncdispatch, asyncnet, sets, strutils]
 from std/algorithm import sort
 
 import eris
@@ -9,18 +9,9 @@ import preserves, syndicate
 from syndicate/protocols/dataspace import Observe
 import ./protocol, ./sockets
 
-type Value = Preserve[void]
-
-proc merge(items: varargs[Value]): Value =
-  # TODO: just a hack, not a proper imlementation
-  # https://preserves.dev/preserves.html#appendix-merging-values
-  result = initDictionary()
-  for e in items:
-    for (key, val) in e.pairs:
-      result[key] = val
-  cannonicalize(result)
-
-type Observe = dataspace.Observe[Ref]
+type
+  Value = Preserve[void]
+  Observe = dataspace.Observe[Ref]
 
 proc recvError(daemon: Session): Future[string] {.async.} =
   discard #[typ]# await recvString(daemon)
@@ -94,7 +85,7 @@ proc connectDaemon(daemon: Session; socketPath: string) {.async.} =
 
 proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async.} =
   var miss = Missing(targets: targets)
-  await send(daemon, wopQueryMissing)
+  await send(daemon, Word wopQueryMissing)
   await send(daemon, miss.targets)
   await recvWork(daemon)
   miss.willBuild = await recvStringSet(daemon)
@@ -106,7 +97,7 @@ proc queryMissing(daemon: Session; targets: StringSeq): Future[Missing] {.async.
 
 proc queryPathInfo(daemon: Session; path: string): Future[LegacyPathAttrs] {.async.} =
   var info: LegacyPathAttrs
-  await send(daemon, wopQueryPathInfo)
+  await send(daemon, Word wopQueryPathInfo)
   await send(daemon, path)
   await recvWork(daemon)
   let valid = await recvWord(daemon)
@@ -137,10 +128,8 @@ proc recvLegacyPathAttrs(daemon: Session): Future[AddToStoreAttrs] {.async.} =
   return info
 
 proc addToStore(daemon: Session; store: ErisStore; request: AddToStoreClientAttrs): Future[(string, AddToStoreAttrs)] {.async.} =
-  let
-    erisCap = parseCap(request.eris)
-    stream = newErisStream(store, erisCap)
-  await send(daemon, wopAddToStore)
+  let erisCap = parseCap(request.eris)
+  await send(daemon, Word wopAddToStore)
   await send(daemon, request.name)
   await send(daemon, string request.`ca-method`)
   await send(daemon, request.references)

src/nix_actor/libnix/libexpr.nim

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import ./stdpuspus, ./store
+
+{.passC: staticExec("pkg-config --cflags nix-expr").}
+{.passL: staticExec("pkg-config --libs nix-expr").}
+
+proc parentDir(path: string): string =
+  var i = path.high
+  while path[i] != '/': dec(i)
+  path[0..i]
+
+{.passC: "-I" & parentDir(currentSourcePath).}
+
+type
+  NixInt* = int64
+  NixFloat* = float64
+
+  ValueKind* {.importcpp: "nix::ValueType", header: "value.hh".} = enum
+    nThunk,
+    nInt,
+    nFloat,
+    nBool,
+    nString,
+    nPath,
+    nNull,
+    nAttrs,
+    nList,
+    nFunction,
+    nExternal,
+  Value* = ValueObj | ValuePtr
+  ValuePtr* = ptr ValueObj
+  ValueObj* {.importcpp: "nix::Value", header: "value.hh".} = object
+    integer*: NixInt
+    boolean*: bool
+    string: StringContext
+    path*: cstring
+    fpoint*: NixFloat
+    attrs: Bindings
+  StringContext = object
+    s: cstring
+  Symbol* {.importcpp: "nix::Symbol", header: "symbol-table.hh".} = object
+    discard
+  Attr {.importcpp: "nix::Attr", header: "attr-set.hh".} = object
+    name: Symbol
+    value: ValuePtr
+  Bindings = ptr BindginsObj
+  BindginsObj {.importcpp: "nix::Bindings", header: "attr-set.hh".} = object
+    discard
+
+proc kind*(val: Value): ValueKind {.importcpp: "#.type()".}
+
+proc shallowString*(val: Value): string =
+  if val.kind != nString:
+    raise newException(FieldDefect, "Value not an attribute set")
+  $val.string.s
+
+proc size(bindings: Bindings): csize_t {.importcpp.}
+
+proc `[]`(b: Bindings; i: Natural): Attr {.importcpp: "(*#)[#]".}
+
+iterator pairs*(val: Value): (Symbol, ValuePtr) =
+  if val.kind != nAttrs:
+    raise newException(FieldDefect, "Value not an attribute set")
+  for i in 0..<val.attrs.size():
+    let attr = val.attrs[i]
+    yield (attr.name, attr.value)
+
+proc listSize(val: Value): csize_t {.importcpp.}
+
+proc listElems(val: Value): ptr UncheckedArray[ValuePtr] {.importcpp.}
+
+iterator items*(val: Value): ValuePtr =
+  if val.kind != nList:
+    raise newException(FieldDefect, "Value not a list")
+  for i in 0..<val.listSize:
+    yield val.listElems()[i]
+
+type
+  ExprObj {.importcpp: "nix::Expr", header: "nixexpr.hh".} = object
+    discard
+  Expr* = ptr ExprObj
+  EvalState* {.importcpp: "nix::ref<nix::EvalState>", header: "eval.hh".} = object
+    discard
+
+proc newEvalState*(store: Store): EvalState {.
+  importcpp: "nix::newEvalState(@)", header: "seepuspus.hh".}
+
+proc parseExprFromString*(state: EvalState; s, basePath: cstring): Expr {.
+  importcpp: "#->parseExprFromString(@)".}
+
+proc eval*(state: EvalState; expr: Expr; value: var ValueObj) {.
+  importcpp: "#->eval(@)".}
+
+proc forceValueDeep*(state: EvalState; value: var ValueObj) {.
+  importcpp: "#->forceValueDeep(@)".}
+
+proc stringView(state: EvalState; sym: Symbol): StringView {.
+  importcpp: "((std::string_view)#->symbols[#])".}
+
+proc symbolString*(state: EvalState; sym: Symbol): string = $stringView(state, sym)
+
+proc initGC*() {.importcpp: "nix::initGC", header: "eval.hh".}

src/nix_actor/libnix/main.nim

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+{.passC: staticExec("pkg-config --cflags nix-main").}
+{.passL: staticExec("pkg-config --libs nix-main").}
+
+proc initNix*() {.importcpp: "nix::initNix", header: "shared.hh".}

src/nix_actor/libnix/seepuspus.hh

@@ -0,0 +1,21 @@
+#pragma once
+#include "eval.hh"
+
+namespace nix {
+
+  ref<EvalState> newEvalState(ref<Store> store)
+  {
+    auto searchPath = Strings();
+    auto evalState =
+      #if HAVE_BOEHMGC
+        std::allocate_shared<EvalState>(
+          traceable_allocator<EvalState>(), searchPath, store, store)
+      #else
+        std::make_shared<EvalState>(
+          searchPath, store, store)
+      #endif
+        ;
+    return ref<EvalState>(evalState);
+  }
+
+}

src/nix_actor/libnix/stdpuspus.nim

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+type StringView* {.importcpp: "std::string_view", header: "<string>".} = object
+
+proc toStringView*(s: pointer; count: int): StringView {.
+  importcpp: "std::string_view(static_cast<const char *>(#), #)", constructor.}
+
+proc toStringView*(s: string): StringView {.inline.} =
+  if s.len == 0: toStringView(nil, 0)
+  else: toStringView(unsafeAddr s[0], s.len)
+
+proc toStringView*(buf: openarray[byte]): StringView {.inline.} =
+  if buf.len == 0: toStringView(nil, 0)
+  else: toStringView(unsafeAddr buf[0], buf.len)
+
+proc toStringView*(sv: StringView): StringView {.inline.} = sv
+
+proc data(sv: StringView): pointer {.importcpp.}
+proc size(sv: StringView): csize_t {.importcpp.}
+
+proc `$`*(sv: StringView): string =
+  result = newString(sv.size)
+  copyMem(addr result[0], sv.data, result.len)

src/nix_actor/libnix/store.nim

@@ -23,7 +23,7 @@ var nixVersion* {.importc: "nix::nixVersion", header: "globals.hh".}: StdString
 proc isDerivation*(path: StorePath): bool {.importcpp.}
 
 type
-  Store {.importcpp: "nix::ref<nix::Store>", header: "store-api.hh".} = object
+  Store* {.importcpp: "nix::ref<nix::Store>", header: "store-api.hh".} = object
     discard
 
 proc ensurePath*(store: Store; path: StorePath) {.importcpp.}

src/nix_actor/sockets.nim

@@ -32,41 +32,51 @@ const
   STDERR_STOP_ACTIVITY* = 0x53544F50
   STDERR_RESULT* = 0x52534C54
 
-  wopIsValidPath* = 1
-  wopHasSubstitutes* = 3
-  wopQueryReferrers* = 6
-  wopAddToStore* = 7
-  wopBuildPaths* = 9
-  wopEnsurePath* = 10
-  wopAddTempRoot* = 11
-  wopAddIndirectRoot* = 12
-  wopSyncWithGC* = 13
-  wopFindRoots* = 14
-  wopSetOptions* = 19
-  wopCollectGarbage* = 20
-  wopQuerySubstitutablePathInfo* = 21
-  wopQueryAllValidPaths* = 23
-  wopQueryFailedPaths* = 24
-  wopClearFailedPaths* = 25
-  wopQueryPathInfo* = 26
-  wopQueryPathFromHashPart* = 29
-  wopQuerySubstitutablePathInfos* = 30
-  wopQueryValidPaths* = 31
-  wopQuerySubstitutablePaths* = 32
-  wopQueryValidDerivers* = 33
-  wopOptimiseStore* = 34
-  wopVerifyStore* = 35
-  wopBuildDerivation* = 36
-  wopAddSignatures* = 37
-  wopNarFromPath* = 38
-  wopAddToStoreNar* = 39
-  wopQueryMissing* = 40
-  wopQueryDerivationOutputMap* = 41
-  wopRegisterDrvOutput* = 42
-  wopQueryRealisation* = 43
-  wopAddMultipleToStore* = 44
-  wopAddBuildLog* = 45
-  wopBuildPathsWithResults* = 46
+type WorkerOperation* = enum
+  wopInvalid = 0,
+  wopIsValidPath = 1,
+  wopHasSubstitutes = 3,
+  wopQueryPathHash = 4, # obsolete
+  wopQueryReferences = 5, # obsolete
+  wopQueryReferrers = 6,
+  wopAddToStore = 7,
+  wopAddTextToStore = 8, # obsolete since 1.25, Nix 3.0. Use wopAddToStore
+  wopBuildPaths = 9,
+  wopEnsurePath = 10,
+  wopAddTempRoot = 11,
+  wopAddIndirectRoot = 12,
+  wopSyncWithGC = 13,
+  wopFindRoots = 14,
+  wopExportPath = 16, # obsolete
+  wopQueryDeriver = 18, # obsolete
+  wopSetOptions = 19,
+  wopCollectGarbage = 20,
+  wopQuerySubstitutablePathInfo = 21,
+  wopQueryDerivationOutputs = 22, # obsolete
+  wopQueryAllValidPaths = 23,
+  wopQueryFailedPaths = 24,
+  wopClearFailedPaths = 25,
+  wopQueryPathInfo = 26,
+  wopImportPaths = 27, # obsolete
+  wopQueryDerivationOutputNames = 28, # obsolete
+  wopQueryPathFromHashPart = 29,
+  wopQuerySubstitutablePathInfos = 30,
+  wopQueryValidPaths = 31,
+  wopQuerySubstitutablePaths = 32,
+  wopQueryValidDerivers = 33,
+  wopOptimiseStore = 34,
+  wopVerifyStore = 35,
+  wopBuildDerivation = 36,
+  wopAddSignatures = 37,
+  wopNarFromPath = 38,
+  wopAddToStoreNar = 39,
+  wopQueryMissing = 40,
+  wopQueryDerivationOutputMap = 41,
+  wopRegisterDrvOutput = 42,
+  wopQueryRealisation = 43,
+  wopAddMultipleToStore = 44,
+  wopAddBuildLog = 45,
+  wopBuildPathsWithResults = 46,
 
 type
   ProtocolError* = object of IOError