Remove records module

Redundant with toPreserve and fromPreserve.

src/preserves.nim

@@ -10,16 +10,14 @@ from std/macros import hasCustomPragma, getCustomPragmaVal
 type
   PreserveKind* = enum
     pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkBigInteger, pkString, pkByteString, pkSymbol,
-        pkRecord, pkSequence, pkSet, pkDictionary
+        pkRecord, pkSequence, pkSet, pkDictionary, pkEmbedded
 
 const
   atomKinds* = {pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkBigInteger, pkString, pkByteString, pkSymbol}
   compoundKinds* = {pkRecord, pkSequence, pkSet, pkDictionary}
 
 type
-  DictEntry = tuple[key: Preserve, val: Preserve]
+  Preserve*[E = void] {.acyclic.} = object
-
-  Preserve* {.acyclic.} = object
     case kind*: PreserveKind
     of pkBoolean:
       bool*: bool
@@ -38,18 +36,24 @@ type
     of pkSymbol:
       symbol*: string
     of pkRecord:
-      record*: seq[Preserve] # label is last
+      record*: seq[Preserve[E]] # label is last
     of pkSequence:
-      sequence*: seq[Preserve]
+      sequence*: seq[Preserve[E]]
     of pkSet:
-      set*: seq[Preserve]
+      set*: seq[Preserve[E]]
         # TODO: HashSet
     of pkDictionary:
-      dict*: seq[DictEntry]
+      dict*: seq[DictEntry[E]]
         # TODO: Tables
+    of pkEmbedded:
+      embed*: E
     embedded*: bool
+      ## Flag to mark embedded Preserves
 
-proc `==`*(x, y: Preserve): bool =
+  DictEntry[E] = tuple[key: Preserve[E], val: Preserve[E]]
+
+
+proc `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
   ## Check `x` and `y` for equivalence.
   if x.kind == y.kind and x.embedded == y.embedded:
     case x.kind
@@ -79,6 +83,12 @@ proc `==`*(x, y: Preserve): bool =
       result = x.set == y.set
     of pkDictionary:
       result = x.dict == y.dict
+    of pkEmbedded:
+      when A is B:
+        when A is void:
+          result = true
+        else:
+          result = x.embed == y.embed
 
 proc `<`(x, y: string | seq[byte]): bool =
   for i in 0 .. min(x.high, y.high):
@@ -86,7 +96,7 @@ proc `<`(x, y: string | seq[byte]): bool =
     if x[i] != y[i]: return false
   x.len < y.len
 
-proc `<`*(x, y: Preserve): bool =
+proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
   ## Preserves have a total order over Values. Check if `x` is ordered before `y`.
   if x.embedded != y.embedded:
     result = y.embedded
@@ -138,6 +148,9 @@ proc `<`*(x, y: Preserve): bool =
           if x.dict[i].val < y.dict[i].val: return true
           if x.dict[i].val != y.dict[i].val: return false
       result = x.dict.len < y.dict.len
+    of pkEmbedded:
+      when (not A is void) and (A is B):
+        result = x.embed < y.embed
 
 proc hash*(pr: Preserve): Hash =
   ## Produce a `Hash` of `pr` for use with a `HashSet` or `Table`.
@@ -172,6 +185,8 @@ proc hash*(pr: Preserve): Hash =
   of pkDictionary:
     for (key, val) in pr.dict.items:
       h = h !& hash(key) !& hash(val)
+  of pkEmbedded:
+      h = h !& hash(pr.embed)
   !$h
 
 proc `[]`*(pr: Preserve; i: int): Preserve =
@@ -224,39 +239,37 @@ proc `[]=`*(pr: var Preserve; key, val: Preserve) =
       return
   pr.dict.add((key, val, ))
 
-proc symbol*(s: string; E = void): Preserve {.inline.} =
+proc symbol*[E](s: string): Preserve[E] {.inline.} =
   ## Create a Preserves symbol value.
-  Preserve(kind: pkSymbol, symbol: s)
+  Preserve[E](kind: pkSymbol, symbol: s)
 
-proc initRecord*(label: Preserve; arity = 0): Preserve =
+proc initRecord*[E](label: Preserve[E]; arity: Natural = 0): Preserve[E] =
   ## Create a Preserves record value.
-  result = Preserve(kind: pkRecord, record: newSeq[Preserve](arity.succ))
+  result = Preserve[E](kind: pkRecord, record: newSeq[Preserve[E]](arity.succ))
   result.record[arity] = label
 
-proc initRecord*(label: Preserve; args: varargs[Preserve]): Preserve =
+proc initRecord*[E](label: Preserve[E]; args: varargs[Preserve[E]]): Preserve[E] =
   ## Create a Preserves record value.
-  result = Preserve(kind: pkRecord,
+  result = Preserve[E](kind: pkRecord,
-      record: newSeqOfCap[Preserve](1+args.len))
+      record: newSeqOfCap[Preserve[E]](1+args.len))
   for arg in args:
     result.record.add(arg)
   result.record.add(label)
 
-proc initRecord*(label: string; args: varargs[Preserve, toPreserve]): Preserve =
+proc initSequence*[E](len: Natural = 0): Preserve[E] =
-  ## Convert ``label`` to a symbol and create a new record.
-  runnableExamples:
-    assert($initRecord("foo", 1, 2.0) == "<foo 1 2.0f>")
-  initRecord(symbol(label), args)
-
-proc initSequence*(len = 0): Preserve =
   ## Create a Preserves sequence value.
-  Preserve(kind: pkSequence, record: newSeq[Preserve](len))
+  Preserve[E](kind: pkSequence, sequence: newSeq[Preserve[E]](len))
 
-proc initSet*(): Preserve = Preserve(kind: pkSet)
+proc initSet*[E](): Preserve[E] = Preserve[E](kind: pkSet)
   ## Create a Preserves set value.
 
-proc initDictionary*(): Preserve = Preserve(kind: pkDictionary)
+proc initDictionary*[E](): Preserve[E] = Preserve[E](kind: pkDictionary)
   ## Create a Preserves dictionary value.
 
+proc embed*[E](e: E): Preserve[E] =
+  ## Create a Preserves value that embeds ``e``.
+  Preserve[E](kind: pkEmbedded, embed: e)
+
 proc len*(pr: Preserve): int =
   ## Return the shallow count of values in ``pr``, that is the number of
   ## fields in a record, items in a sequence, items in a set, or key-value pairs
@@ -327,8 +340,10 @@ proc isSet*(pr: Preserve): bool {.inline.} = pr.kind == pkSet
 proc isDictionary*(pr: Preserve): bool {.inline.} = pr.kind == pkDictionary
   ## Check if ``pr`` is a Preserves dictionary.
 
-func isEmbedded*(pr: Preserve): bool {.inline.} = pr.embedded
+func isEmbedded*[E](pr: Preserve[E]): bool {.inline.} =
   ## Check if ``pr`` is an embedded value.
+  when E is void: pr.embedded # embedded Preserves
+  else: pr.kind == pkEmbedded # embedded Nim
 
 proc label*(pr: Preserve): Preserve {.inline.} =
   ## Return the label of record value.
@@ -366,7 +381,7 @@ proc readVarint(s: Stream): int =
       break
     shift.inc 7
 
-proc write*(str: Stream; pr: Preserve) =
+proc write*[E](str: Stream; pr: Preserve[E]) =
   ## Write the binary-encoding of a Preserves value to a stream.
   if pr.embedded: str.write(0x86'u8)
   case pr.kind:
@@ -460,15 +475,19 @@ proc write*(str: Stream; pr: Preserve) =
       str.write(key)
       str.write(value)
     str.write(0x84'u8)
+  of pkEmbedded:
+    when not E is void:
+      str.write(0x86'u8)
+      str.write(pr.embed.toPreserve)
 
-proc encode*(pr: Preserve): seq[byte] =
+proc encode*[E](pr: Preserve[E]): seq[byte] =
   ## Return the binary-encoding of a Preserves value.
   let s = newStringStream()
   s.write pr
   s.setPosition 0
   result = cast[seq[byte]](s.readAll)
 
-proc decodePreserves*(s: Stream): Preserve =
+proc decodePreserves*(s: Stream; E = void): Preserve[E] =
   ## Decode a Preserves value from a binary-encoded stream.
   proc assertStream(check: bool) =
     if not check:
@@ -476,61 +495,61 @@ proc decodePreserves*(s: Stream): Preserve =
   const endMarker = 0x84
   let tag = s.readUint8()
   case tag
-  of 0x80: result = Preserve(kind: pkBoolean, bool: false)
+  of 0x80: result = Preserve[E](kind: pkBoolean, bool: false)
-  of 0x81: result = Preserve(kind: pkBoolean, bool: true)
+  of 0x81: result = Preserve[E](kind: pkBoolean, bool: true)
   of 0x82:
     when system.cpuEndian == bigEndian:
-      result = Preserve(kind: pkFloat, float: s.readFloat32())
+      result = Preserve[E](kind: pkFloat, float: s.readFloat32())
     else:
-      result = Preserve(kind: pkFloat)
+      result = Preserve[E](kind: pkFloat)
       var be = s.readFloat32()
       swapEndian32(result.float.addr, be.addr)
   of 0x83:
     when system.cpuEndian == bigEndian:
-      result = Preserve(kind: pkDouble, double: s.readFloat64())
+      result = Preserve[E](kind: pkDouble, double: s.readFloat64())
     else:
-      result = Preserve(kind: pkDouble)
+      result = Preserve[E](kind: pkDouble)
       var be = s.readFloat64()
       swapEndian64(result.double.addr, be.addr)
   of 0x86:
-    result = decodePreserves(s)
+    result = decodePreserves(s, E)
     result.embedded = true
   of 0xb1:
-    result = Preserve(kind: pkString)
+    result = Preserve[E](kind: pkString)
     let len = s.readVarint()
     result.string = s.readStr(len)
   of 0xb2:
-    result = Preserve(kind: pkByteString)
+    result = Preserve[E](kind: pkByteString)
     let len = s.readVarint()
     result.bytes = cast[seq[byte]](s.readStr(len))
   of 0xb3:
     let len = s.readVarint()
-    result = Preserve(kind: pkSymbol, symbol: s.readStr(len))
+    result = Preserve[E](kind: pkSymbol, symbol: s.readStr(len))
   of 0xb4:
-    result = Preserve(kind: pkRecord)
+    result = Preserve[E](kind: pkRecord)
-    var label = decodePreserves(s)
+    var label = decodePreserves(s, E)
     while s.peekUint8() != endMarker:
-      result.record.add decodePreserves(s)
+      result.record.add decodePreserves(s, E)
     result.record.add(move label)
     discard s.readUint8()
   of 0xb5:
-    result = Preserve(kind: pkSequence)
+    result = Preserve[E](kind: pkSequence)
     while s.peekUint8() != endMarker:
-      result.sequence.add decodePreserves(s)
+      result.sequence.add decodePreserves(s, E)
     discard s.readUint8()
   of 0xb6:
-    result = Preserve(kind: pkSet)
+    result = Preserve[E](kind: pkSet)
     while s.peekUint8() != endMarker:
-      incl(result, decodePreserves(s))
+      incl(result, decodePreserves(s, E))
     discard s.readUint8()
   of 0xb7:
-    result = Preserve(kind: pkDictionary)
+    result = Preserve[E](kind: pkDictionary)
     while s.peekUint8() != endMarker:
-      result[decodePreserves(s)] = decodePreserves(s)
+      result[decodePreserves(s, E)] = decodePreserves(s, E)
     discard s.readUint8()
   of 0xb0:
     let len = s.readVarint()
-    result = Preserve(kind: pkBigInteger, bigint: initBigint 0)
+    result = Preserve[E](kind: pkBigInteger, bigint: initBigint 0)
     for _ in 1..len:
       result.bigint = (result.bigint shl 8) + s.readUint8().int32
   of endMarker:
@@ -539,29 +558,29 @@ proc decodePreserves*(s: Stream): Preserve =
     case 0xf0 and tag
     of 0x90:
       var n = tag.BiggestInt
-      result = Preserve(kind: pkSignedInteger,
+      result = Preserve[E](kind: pkSignedInteger,
           int: n - (if n > 0x9c: 0xa0 else: 0x90))
     of 0xa0:
       let len = (tag.int and 0x0f) + 1
       if len <= 8:
-        result = Preserve(kind: pkSignedInteger, int: s.readUint8().BiggestInt)
+        result = Preserve[E](kind: pkSignedInteger, int: s.readUint8().BiggestInt)
         if (result.int and 0x80) != 0: result.int.dec(0x100)
         for i in 1..<len:
           result.int = (result.int shl 8) or s.readUint8().BiggestInt
       else:
-        result = Preserve(kind: pkBigInteger)
+        result = Preserve[E](kind: pkBigInteger)
         for i in 0..<len:
           result.bigint = (result.bigint shl 8) + s.readUint8().int32
     else:
       assertStream(false)
 
-proc decodePreserves*(s: string): Preserve =
+proc decodePreserves*(s: string; E = void): Preserve[E] =
   ## Decode a string of binary-encoded Preserves.
-  s.newStringStream.decodePreserves
+  decodePreserves(s.newStringStream, E)
 
-proc decodePreserves*(s: seq[byte]): Preserve =
+proc decodePreserves*(s: seq[byte]; E = void): Preserve[E] =
   ## Decode a byte-string of binary-encoded Preserves.
-  cast[string](s).decodePreserves
+  decodePreserves(cast[string](s), E)
 
 template record*(label: string) {.pragma.}
   ## Serialize this object or tuple as a record. See ``toPreserve``.
@@ -569,61 +588,77 @@ template record*(label: string) {.pragma.}
 template unpreservable*() {.pragma.}
   ## Pragma to forbid a type from being converted by ``toPreserve``.
 
-proc toPreserve*[T](x: T): Preserve =
+proc toPreserve*[T](x: T; E = void): Preserve[E] =
   ## Serializes ``x`` to Preserves. Can be customized by defining
-  ## ``toPreserveHook(x: T)`` in the calling scope.
+  ## ``toPreserveHook(x: T; E: typedesc)`` in the calling scope.
-  when (T is Preserve): result = x
+  ## Any ``toPreserveHook`` that does not compile will be discarded;
-  elif compiles(toPreserveHook(x)):
+  ## *Write tests for your hooks!*
-    result = toPreserveHook(x)
+  when (T is Preserve[E]): result = x
+  elif compiles(toPreserveHook(x, E)):
+    result = toPreserveHook(x, E)
   elif T is Bigint:
-    result = Preserve(kind: pkBigInteger, bigint: x)
+    result = Preserve[E](kind: pkBigInteger, bigint: x)
   elif T is seq[byte]:
-    result = Preserve(kind: pkByteString, bytes: x)
+    result = Preserve[E](kind: pkByteString, bytes: x)
   elif T is array | seq:
-    result = Preserve(kind: pkSequence, sequence: newSeqOfCap[Preserve](x.len))
+    result = Preserve[E](kind: pkSequence, sequence: newSeqOfCap[Preserve[E]](x.len))
-    for v in x.items: result.sequence.add(toPreserve(v))
+    for v in x.items: result.sequence.add(toPreserve(v, E))
   elif T is bool:
-    result = Preserve(kind: pkBoolean, bool: x)
+    result = Preserve[E](kind: pkBoolean, bool: x)
   elif T is distinct:
-    result = toPreserve(x.distinctBase)
+    result = toPreserve(x.distinctBase, E)
   elif T is float:
-    result = Preserve(kind: pkFloat, float: x)
+    result = Preserve[E](kind: pkFloat, float: x)
   elif T is float64:
-    result = Preserve(kind: pkDouble, double: x)
+    result = Preserve[E](kind: pkDouble, double: x)
   elif T is object | tuple:
     when T.hasCustomPragma(unpreservable): {.fatal: "unpreservable type".}
     elif T.hasCustomPragma(record):
-      result = Preserve(kind: pkRecord)
+      result = Preserve[E](kind: pkRecord)
-      for _, f in x.fieldPairs: result.record.add(toPreserve(f))
+      for _, f in x.fieldPairs: result.record.add(toPreserve(f, E))
-      result.record.add(symbol(T.getCustomPragmaVal(record)))
+      result.record.add(symbol[E](T.getCustomPragmaVal(record)))
     else:
-      result = Preserve(kind: pkDictionary)
+      result = Preserve[E](kind: pkDictionary)
       for k, v in x.fieldPairs:
-        result[symbol(k)] = toPreserve(v)
+        result[symbol[E](k)] = toPreserve(v, E)
   elif T is Ordinal:
-    result = Preserve(kind: pkSignedInteger, int: x.ord.BiggestInt)
+    result = Preserve[E](kind: pkSignedInteger, int: x.ord.BiggestInt)
   elif T is ptr | ref:
-    if system.`==`(x, nil): result = symbol("null")
+    if system.`==`(x, nil): result = symbol[E]("null")
-    else: result = toPreserve(x[])
+    else: result = toPreserve(x[], E)
   elif T is string:
-    result = Preserve(kind: pkString, string: x)
+    result = Preserve[E](kind: pkString, string: x)
   elif T is SomeInteger:
-    result = Preserve(kind: pkSignedInteger, int: x.BiggestInt)
+    result = Preserve[E](kind: pkSignedInteger, int: x.BiggestInt)
   else:
     raiseAssert("unpreservable type" & $T)
 
-proc toPreserveHook*[T](set: HashSet[T]): Preserve =
+proc toPreserveHook*[A](pr: Preserve[A]; E: typedesc): Preserve[E] =
+  ## Hook for converting ``Preserve`` values with different embedded types.
+  if pr.kind == pkEmbedded:
+    when E is void:
+      result = toPreserve(pr.embed, E)
+    else:
+      result = Preserve[E](pk: pr.kind, embed: (E)pr.embed)
+  else:
+    result = cast[Preserve[E]](pr)
+
+proc toPreserveHook*[T](set: HashSet[T]; E: typedesc): Preserve[E] =
   ## Hook for preserving ``HashSet``.
-  Preserve(kind: pkSet, set: set.map(toPreserve))
+  result = Preserve[E](kind: pkSet, set: newSeqOfCap[Preserve[E]](set.len))
+  for e in set: result.incl toPreserve(e, E)
 
-proc toPreserveHook*[A, B](table: Table[A, B]|TableRef[A, B]): Preserve =
+proc toPreserveHook*[A, B](table: Table[A, B]|TableRef[A, B], E: typedesc): Preserve[E] =
   ## Hook for preserving ``Table``.
-  result = initDictionary()
+  result = initDictionary[E]()
-  for k, v in table.pairs: result[k.toPreserve] = v.toPreserve
+  for k, v in table.pairs: result[toPreserve(k, E)] = toPreserve(v, E)
 
-proc fromPreserve*[T](v: var T; pr: Preserve): bool =
+proc fromPreserve*[T, E](v: var T; pr: Preserve[E]): bool =
   ## Inplace version of `preserveTo`. Returns ``true`` on
   ## a complete match, otherwise returns ``false``.
+  ## Can be customized with `fromPreserveHook[E](x: T; var pr: Preserve[E]): bool`.
+  ## Any ``fromPreserveHook`` that does not compile will be discarded;
+  ## *Write tests for your hooks!*
   # TODO: {.raises: [].}
   runnableExamples:
     import preserves, preserves/parse
@@ -637,6 +672,8 @@ proc fromPreserve*[T](v: var T; pr: Preserve): bool =
   when T is Value:
     v = pr
     result = true
+  elif T is E:
+    result = pr.embed
   elif compiles(fromPreserveHook(v, pr)):
     result = fromPreserveHook(v, pr)
   elif T is Bigint:
@@ -695,7 +732,7 @@ proc fromPreserve*[T](v: var T; pr: Preserve): bool =
       for key, val in v.fieldPairs:
         inc fieldCount
         for (pk, pv) in pr.dict.items:
-          var sym = symbol(key)
+          var sym = symbol[E](key)
           if sym == pk:
             result = result and fromPreserve(val, pv)
             break
@@ -706,7 +743,7 @@ proc fromPreserve*[T](v: var T; pr: Preserve): bool =
       v = (T)pr.int
       result = true
   elif T is ref:
-    if pr != symbol("null"):
+    if pr != symbol[E]("null"):
       new v
       result = fromPreserve(v[], pr)
   elif T is string:
@@ -735,13 +772,28 @@ proc preserveTo*(pr: Preserve; T: typedesc): Option[T] =
   if fromPreserve(v, pr):
     result = some(move v)
 
-proc fromPreserveHook*[A,B,E](t: var Table[A,B]|TableRef[A,B]; pr: Preserve): bool =
+proc fromPreserveHook*[T, E](set: var HashSet[T]; pr: Preserve[E]): bool =
-  if pr.isDictionary:
+  ## Hook for preserving ``HashSet``.
-    for k, v in pr.pairs:
+  if pr.kind == pkSet:
-      t[preserveTo(k, A)] = preserveTo(v, B)
+    result = true
-      result = true
+    set.init(pr.set.len)
+    var e: T
+    for pe in pr.set:
+      result = fromPreserve(e, pe)
+      if not result: break
+      set.incl(move e)
 
-proc concat(result: var string; pr: Preserve) =
+proc fromPreserveHook*[A,B,E](t: var (Table[A,B]|TableRef[A,B]); pr: Preserve[E]): bool =
+  if pr.isDictionary:
+    result = true
+    var a: A
+    var b: B
+    for (k, v) in pr.dict:
+      result = fromPreserve(a, k) and fromPreserve(b, v)
+      if not result: break
+      t[move a] = move b
+
+proc concat[E](result: var string; pr: Preserve[E]) =
   if pr.embedded: result.add("#!")
   case pr.kind:
   of pkBoolean:
@@ -804,6 +856,20 @@ proc concat(result: var string; pr: Preserve) =
       result.concat(value)
       inc i
     result.add('}')
+  of pkEmbedded:
+    when not E is void:
+      result.add("#!")
+      result.add($pr.embed)
 
 proc `$`*(pr: Preserve): string = concat(result, pr)
   ## Generate the textual representation of ``pr``.
+
+when isMainModule:
+  block:
+    var t: Table[int, string]
+    var pr = t.toPreserveHook(void)
+    assert fromPreserveHook(t, pr)
+  block:
+    var h: HashSet[string]
+    var pr = h.toPreserveHook(void)
+    assert fromPreserveHook(h, pr)

src/preserves/jsonhooks.nim

@@ -4,31 +4,31 @@
 import std/[json, tables]
 import ../preserves
 
-proc toPreserveHook*(js: JsonNode): Preserve =
+proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] =
   case js.kind
   of JString:
-    result = Preserve(kind: pkString, string: js.str)
+    result = Preserve[E](kind: pkString, string: js.str)
   of JInt:
-    result = Preserve(kind: pkSignedInteger, int: js.num)
+    result = Preserve[E](kind: pkSignedInteger, int: js.num)
   of JFloat:
-    result = Preserve(kind: pkDouble, double: js.fnum)
+    result = Preserve[E](kind: pkDouble, double: js.fnum)
   of JBool:
     result = case js.bval
-      of false: symbol"false"
+      of false: symbol[E]"false"
-      of true: symbol"true"
+      of true: symbol[E]"true"
   of JNull:
-    result = symbol"null"
+    result = symbol[E]"null"
   of JObject:
-    result = Preserve(kind: pkDictionary)
+    result = Preserve[E](kind: pkDictionary)
     for key, val in js.fields.pairs:
-      result[Preserve(kind: pkString, string: key)] = toPreserveHook(val)
+      result[Preserve[E](kind: pkString, string: key)] = toPreserveHook(val, E)
   of JArray:
-    result = Preserve(kind: pkSequence,
+    result = Preserve[E](kind: pkSequence,
-        sequence: newSeq[Preserve](js.elems.len))
+        sequence: newSeq[Preserve[E]](js.elems.len))
     for i, e in js.elems:
-      result.sequence[i] = toPreserveHook(e)
+      result.sequence[i] = toPreserveHook(e, E)
 
-proc fromPreserveHook*(js: var JsonNode; prs: Preserve): bool =
+proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
   case prs.kind:
   of pkBoolean:
     js = newJBool(prs.bool)
@@ -67,6 +67,16 @@ proc fromPreserveHook*(js: var JsonNode; prs: Preserve): bool =
   else: return false
   true
 
-proc toJsonHook*(pr: Preserve): JsonNode =
+proc toJsonHook*[E](pr: Preserve[E]): JsonNode =
-  if not fromPreserve(result, pr):
+  if not fromPreserveHook(result, pr):
     raise newException(ValueError, "cannot convert Preserves value to JSON")
+
+proc fromJsonHook*[E](pr: var Preserve[E]; js: JsonNode) =
+  pr = toPreserveHook(js, E)
+
+when isMainModule:
+  var js = JsonNode()
+  var pr = js.toPreserveHook(void)
+  assert fromPreserveHook(js, pr)
+  fromJsonHook(pr, js)
+  js = toJsonHook(pr)

src/preserves/parse.nim

@@ -6,14 +6,15 @@ import npeg
 import ../preserves, ./pegs
 
 type
-  Frame = tuple[value: Preserve, pos: int]
+  Value = Preserve[void]
+  Frame = tuple[value: Value, pos: int]
   Stack = seq[Frame]
 
 proc shrink(stack: var Stack; n: int) = stack.setLen(stack.len - n)
 
-template pushStack(v: Preserve) = stack.add((v, capture[0].si))
+template pushStack(v: Value) = stack.add((v, capture[0].si))
 
-proc parsePreserves*(text: string): Preserve {.gcsafe.} =
+proc parsePreserves*(text: string): Preserve[void] {.gcsafe.} =
   const pegParser = peg("Document", stack: Stack):
     # Override rules from pegs.nim
 
@@ -21,7 +22,7 @@ proc parsePreserves*(text: string): Preserve {.gcsafe.} =
 
     Preserves.Record <- Preserves.Record:
       var
-        record: seq[Preserve]
+        record: seq[Value]
         labelOff: int
       while stack[labelOff].pos < capture[0].si:
         inc labelOff
@@ -29,18 +30,18 @@ proc parsePreserves*(text: string): Preserve {.gcsafe.} =
         record.add(move stack[i].value)
       record.add(move stack[labelOff].value)
       stack.shrink record.len
-      pushStack Preserve(kind: pkRecord, record: move record)
+      pushStack Value(kind: pkRecord, record: move record)
 
     Preserves.Sequence <- Preserves.Sequence:
-      var sequence: seq[Preserve]
+      var sequence: seq[Value]
       for frame in stack.mitems:
         if frame.pos > capture[0].si:
           sequence.add(move frame.value)
       stack.shrink sequence.len
-      pushStack Preserve(kind: pkSequence, sequence: move sequence)
+      pushStack Value(kind: pkSequence, sequence: move sequence)
 
     Preserves.Dictionary <- Preserves.Dictionary:
-      var prs = Preserve(kind: pkDictionary)
+      var prs = Value(kind: pkDictionary)
       for i in countDown(stack.high.pred, 0, 2):
         if stack[i].pos < capture[0].si: break
         prs[move stack[i].value] = stack[i.succ].value
@@ -48,7 +49,7 @@ proc parsePreserves*(text: string): Preserve {.gcsafe.} =
       pushStack prs
 
     Preserves.Set <- Preserves.Set:
-      var prs = Preserve(kind: pkSet)
+      var prs = Value(kind: pkSet)
       for frame in stack.mitems:
         if frame.pos > capture[0].si:
           prs.incl(move frame.value)
@@ -57,36 +58,36 @@ proc parsePreserves*(text: string): Preserve {.gcsafe.} =
 
     Preserves.Boolean <- Preserves.Boolean:
       case $0
-      of "#f": pushStack Preserve(kind: pkBoolean)
+      of "#f": pushStack Value(kind: pkBoolean)
-      of "#t": pushStack Preserve(kind: pkBoolean, bool: true)
+      of "#t": pushStack Value(kind: pkBoolean, bool: true)
       else: discard
 
     Preserves.Float <- Preserves.Float:
-      pushStack Preserve(kind: pkFloat, float: parseFloat($1))
+      pushStack Value(kind: pkFloat, float: parseFloat($1))
 
     Preserves.Double <- Preserves.Double:
-      pushStack Preserve(kind: pkDouble)
+      pushStack Value(kind: pkDouble)
       let i = stack.high
       discard parseBiggestFloat($0, stack[i].value.double)
 
     Preserves.SignedInteger <- Preserves.SignedInteger:
-      pushStack Preserve(kind: pkSignedInteger, int: parseInt($0))
+      pushStack Value(kind: pkSignedInteger, int: parseInt($0))
 
     Preserves.String <- Preserves.String:
-      pushStack Preserve(kind: pkString, string: unescape($0))
+      pushStack Value(kind: pkString, string: unescape($0))
 
     Preserves.charByteString <- Preserves.charByteString:
       let s = unescape($1)
-      pushStack Preserve(kind: pkByteString, bytes: cast[seq[byte]](s))
+      pushStack Value(kind: pkByteString, bytes: cast[seq[byte]](s))
 
     Preserves.hexByteString <- Preserves.hexByteString:
-      pushStack Preserve(kind: pkByteString, bytes: cast[seq[byte]](parseHexStr($1)))
+      pushStack Value(kind: pkByteString, bytes: cast[seq[byte]](parseHexStr($1)))
 
     Preserves.b64ByteString <- Preserves.b64ByteString:
-      pushStack Preserve(kind: pkByteString, bytes: cast[seq[byte]](base64.decode($1)))
+      pushStack Value(kind: pkByteString, bytes: cast[seq[byte]](base64.decode($1)))
 
     Preserves.Symbol <- Preserves.Symbol:
-      pushStack Preserve(kind: pkSymbol, symbol: $0)
+      pushStack Value(kind: pkSymbol, symbol: $0)
 
     Preserves.Embedded <- Preserves.Embedded:
       var v = stack.pop.value
@@ -94,7 +95,7 @@ proc parsePreserves*(text: string): Preserve {.gcsafe.} =
       pushStack v
 
     Preserves.Compact <- Preserves.Compact:
-      pushStack decodePreserves(stack.pop.value.bytes)
+      pushStack decodePreserves(stack.pop.value.bytes, void)
 
   var stack: Stack
   let match = pegParser.match(text, stack)

src/preserves/private/preserves_schema_nim.nim

@@ -1,13 +1,16 @@
 # SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[os, strutils, tables]
+import std/[os, strutils, sets, tables]
 
 import compiler/[ast, idents, renderer, lineinfos]
 
 import ../../preserves, ../schemas
 
-type TypeTable = OrderedTable[string, PNode]
+type
+  Value = Preserve[void]
+  TypeSpec = tuple[node: PNode, embeddable: bool]
+  TypeTable = OrderedTable[string, TypeSpec]
 
 proc add(parent, child: PNode): PNode {.discardable.} =
   parent.sons.add child
@@ -52,31 +55,50 @@ proc ident(sn: SchemaNode): PNode =
     raiseAssert("no ident for " & $sn.kind & " " & $sn)
   s.ident.accQuote
 
-proc typeIdent(sn: SchemaNode): PNode =
+proc parameterize(node: PNode; embeddable: bool): PNode =
+  if embeddable: nn(nkBracketExpr, node, ident"E")
+  else: node
+
+proc parameterize(spec: TypeSpec): PNode =
+  parameterize(spec.node, spec.embeddable)
+
+proc isPreserve(n: PNode): bool =
+  n.kind == nkBracketExpr and
+    n.renderTree == "Preserve[E]"
+
+proc preserveIdent(): Pnode =
+  nn(nkBracketExpr, ident"Preserve", ident"E")
+
+proc orEmbed(x: var TypeSpec; y: TypeSpec) =
+  x.embeddable = x.embeddable or y.embeddable
+
+proc isEmbeddable(scm: Schema; sn: SchemaNode; seen: var HashSet[string]): bool =
   case sn.kind
-  of snkAtom:
+  of snkAtom, snkLiteral: discard
-    case sn.atom
+  of snkAlt:
-    of akBool: ident"bool"
+    result = isEmbeddable(scm, sn.altBranch, seen)
-    of akFloat: ident"float32"
+  of snkAny: result = true
-    of akDouble: ident"float64"
-    of akInt: ident"BiggestInt"
-    of akString: ident"string"
-    of akBytes: nn(nkBracketExpr, ident"seq", ident"byte")
-    of akSymbol: ident"string" # TODO: distinct string type for symbols?
-  of snkNamed:
-    sn.pattern.typeIdent
   of snkRef:
-    var id = ident sn.refPath[sn.refPath.high]
+    if sn.refPath.len == 1:
-    for i in countDown(sn.refPath.high.pred, 0):
+      let name = sn.refPath[0]
-      id = nn(nkDotExpr, ident(sn.refPath[i]), id)
+      if name notin seen: # loop detection
-    id
+        seen.incl name
+        result = isEmbeddable(scm, scm.definitions[name], seen)
+      else:
+        result = false
+    # TODO: cross-module types
+  of snkEmbedded:
+    result = isEmbeddable(scm, sn.embed, seen)
+  of snkNamed:
+    result = isEmbeddable(scm, sn.pattern, seen)
   else:
-    ident"Preserve"
+    for bn in sn.nodes:
+      result = isEmbeddable(scm, bn, seen)
+      if result: break
 
-proc toExport(n: sink PNode): PNode =
+proc isEmbeddable(scm: Schema; sn: SchemaNode): bool =
-  nkPostFix.newNode.add(ident"*", n)
+  var seen: HashSet[string]
-
+  isEmbeddable(scm, sn, seen)
-proc newEmpty(): PNode = newNode(nkEmpty)
 
 proc isConst(scm: Schema; sn: SchemaNode): bool =
   case sn.kind
@@ -86,7 +108,46 @@ proc isConst(scm: Schema; sn: SchemaNode): bool =
       result = isConst(scm, scm.definitions[sn.refPath[0]])
   else: discard
 
-proc literal(scm: Schema; sn: SchemaNode): Preserve =
+proc isSymbolEnum(scm: Schema; sn: SchemaNode): bool =
+  case sn.kind
+  of snkRef:
+    if sn.refPath.len == 1:
+      result = isSymbolEnum(scm, scm.definitions[sn.refPath[0]])
+  of snkOr:
+    for bn in sn.nodes:
+      if bn.altBranch.kind != snkLiteral or
+          bn.altBranch.value.kind != pkSymbol:
+        return false
+    result = true
+  else: discard
+
+proc typeIdent(scm: Schema; sn: SchemaNode): TypeSpec =
+  case sn.kind
+  of snkAtom:
+    case sn.atom
+    of akBool: (ident"bool", false)
+    of akFloat: (ident"float32", false)
+    of akDouble: (ident"float64", false)
+    of akInt: (ident"BiggestInt", false)
+    of akString: (ident"string", false)
+    of akBytes: (nn(nkBracketExpr, ident"seq", ident"byte"), false)
+    of akSymbol: (ident"string", false) # TODO: distinct string type for symbols?
+  of snkNamed:
+    typeIdent(scm, sn.pattern)
+  of snkRef:
+    var id = ident sn.refPath[sn.refPath.high]
+    for i in countDown(sn.refPath.high.pred, 0):
+      id = nn(nkDotExpr, ident(sn.refPath[i]), id)
+    (id, isEmbeddable(scm, sn))
+  else:
+    (preserveIdent(), true)
+
+proc toExport(n: sink PNode): PNode =
+  nkPostFix.newNode.add(ident"*", n)
+
+proc newEmpty(): PNode = newNode(nkEmpty)
+
+proc literal(scm: Schema; sn: SchemaNode): Value =
   case sn.kind
   of snkLiteral: result = sn.value
   of snkRef:
@@ -97,14 +158,6 @@ proc literal(scm: Schema; sn: SchemaNode): Preserve =
   else:
     raiseAssert("not convertable to a literal: " & $sn)
 
-proc isSymbolEnum(sn: SchemaNode): bool =
-  if sn.kind == snkOr:
-    for bn in sn.nodes:
-      if bn.altBranch.kind != snkLiteral or
-          bn.altBranch.value.kind != pkSymbol:
-        return false
-    result = true
-
 proc toEnumTy(sn: SchemaNode): PNode =
   result = nkEnumTy.newNode.add newEmpty()
   for bn in sn.nodes:
@@ -118,7 +171,22 @@ proc toEnumDef(name: string; sn: SchemaNode): PNode =
       newEmpty(),
       sn.toEnumTy)
 
-proc typeDef(sn: SchemaNode; name: string; ty: PNode): PNode =
+proc embeddingParams(embeddable: bool): PNode =
+  if embeddable:
+    nn(nkGenericParams, nn(nkIdentDefs, ident"E", newEmpty(), ident"void"))
+  else:
+    newEmpty()
+
+proc identDef(sn: SchemaNode; a, b: PNode; embeddable: bool): PNode =
+  if embeddable and b.kind != nkBracketExpr:
+    nn(nkIdentDefs, a, nn(nkBracketExpr, b, ident"E"), newEmpty())
+  else:
+    nn(nkIdentDefs, a, b, newEmpty())
+
+proc identDef(sn: SchemaNode; l: PNode; ts: TypeSpec): PNode =
+  identDef(sn, l, ts.node, ts.embeddable)
+
+proc typeDef(sn: SchemaNode; name: string; ty: PNode; embeddable: bool): PNode =
   case sn.kind
   of snkRecord:
     nn(nkTypeDef,
@@ -128,22 +196,24 @@ proc typeDef(sn: SchemaNode; name: string; ty: PNode): PNode =
         nn(nkExprColonExpr,
           ident"record",
           PNode(kind: nkStrLit, strVal: sn.nodes[0].value.symbol)))),
-      newEmpty(),
+      embeddingParams(embeddable),
       ty)
   else:
-    nn(nkTypeDef, name.ident.toExport, newEmpty(), ty)
+    nn(nkTypeDef, name.ident.toExport, embeddingParams(embeddable), ty)
 
-proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""): PNode =
+proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""):
+    TypeSpec =
+  if name in known: return known[name]
   case sn.kind
   of snkOr:
-    if sn.isSymbolEnum:
+    if isSymbolEnum(scm, sn):
-      result = sn.toEnumTy
+      result.node = sn.toEnumTy
     else:
       let
         enumName = name.nimIdentNormalize & "Kind"
         enumIdent = ident(enumName)
       if enumName notin known:
-        known[enumName] = toEnumDef(enumName, sn)
+        known[enumName] = (toEnumDef(enumName, sn), false)
       let recCase = nkRecCase.newNode.add(
         nkIdentDefs.newNode.add(
           "kind".ident.toExport,
@@ -152,15 +222,19 @@ proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""): PN
       for bn in sn.nodes:
         assert(bn.kind == snkAlt, $bn.kind)
         doAssert(name != "", " no name for " & $sn)
-        var memberTypeIdent: PNode
+        var memberType: TypeSpec
         if bn.altbranch.kind == snkRef:
-          memberTypeIdent = bn.altBranch.typeIdent
+          memberType = typeIdent(scm, bn.altBranch)
         else:
           let memberTypeName = name & bn.altLabel.nimIdentNormalize
-          memberTypeIdent = ident memberTypeName
+          memberType.node = ident memberTypeName
           if memberTypeName notin known:
             let ty = nimTypeOf(scm, known, bn.altBranch, memberTypeName)
-            known[memberTypeName] = typeDef(bn.altBranch, memberTypeName, ty)
+            orEmbed memberType, ty
+            orEmbed result, memberType
+            known[memberTypeName] = (
+              typeDef(bn.altBranch, memberTypeName, ty.node, ty.embeddable),
+              ty.embeddable)
         var recList = nkRecList.newNode
         case bn.altBranch.kind
         of snkRecord:
@@ -169,44 +243,36 @@ proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""): PN
           of 2:
             if not isConst(scm, bn.altBranch.nodes[1]):
               let label = bn.ident
-              recList.add nkIdentDefs.newNode.add(
+              let branch = typeIdent(scm, bn.altBranch.nodes[1])
-                label.toExport,
+              orEmbed result, branch
-                nimTypeOf(scm, known, bn.altBranch.nodes[1], $label),
+              recList.add identDef(bn, label.toExport, branch)
-                newEmpty())
           else:
-            recList.add nkIdentDefs.newNode.add(
+            recList.add identDef(bn, bn.ident.toExport, memberType)
-              bn.ident.toExport,
-              memberTypeIdent,
-              newEmpty())
         else:
           if isConst(scm, bn.altBranch):
             recList.add nkDiscardStmt.newNode.add(newEmpty())
           else:
             let label = bn.ident
-            recList.add(nkIdentDefs.newNode.add(
+            let branch = typeIdent(scm, bn.altBranch)
-              label.toExport,
+            orEmbed result, branch
-              nimTypeOf(scm, known, bn.altBranch, $label),
+            recList.add identDef(bn, label.toExport, branch)
-              newEmpty()))
         let disc = nkDotExpr.newNode.add(
           enumIdent, bn.altLabel.nimIdentNormalize.ident.accQuote)
         if recList.len == 0:
-          recList.add nkIdentDefs.newNode.add(
+          recList.add identDef(bn, bn.ident.toExport, memberType)
-            bn.ident.toExport,
-            memberTypeIdent,
-            newEmpty())
         recCase.add nkOfBranch.newNode.add(disc, recList)
-      result = nn(nkRefTy, nn(nkObjectTy,
+      result.node = nn(nkRefTy, nn(nkObjectTy,
         newEmpty(),
         newEmpty(),
         nn(nkRecList, recCase)))
   of snkAny:
-    result = ident"Preserve"
+    result = (ident"Preserve", true)
   of snkAtom:
-    result = typeIdent(sn)
+    result = typeIdent(scm, sn)
   of snkEmbedded:
     result = nimTypeOf(scm, known, sn.embed)
   of snkLiteral:
-    result = case sn.value.kind # nearly verbatim from ../../preserves/src/preserves.nim
+    result.node = case sn.value.kind # nearly verbatim from ../../preserves/src/preserves.nim
     of pkBoolean: ident"bool"
     of pkFloat: ident"float32"
     of pkDouble: ident"float64"
@@ -216,30 +282,37 @@ proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""): PN
     of pkByteString: nn(
       nkBracketExpr, ident"seq", ident"byte")
     of pkSymbol: ident"string"
-    of pkRecord: ident"Preserve"
+    of pkRecord: preserveIdent()
     of pkSequence: nn(
-      nkBracketExpr, ident"seq", ident"Preserve")
+      nkBracketExpr, ident"seq", preserveIdent())
     of pkSet: nn(
-      nkBracketExpr, ident"HashSet", ident"Preserve")
+      nkBracketExpr, ident"HashSet", preserveIdent())
     of pkDictionary: nn(
-      nkBracketExpr, ident"Table", ident"Preserve", ident"Preserve")
+      nkBracketExpr, ident"TableRef", preserveIdent(), preserveIdent())
+    of pkEmbedded:
+      raiseAssert "this should never happen"
   of snkSequenceOf:
-    result = nkBracketExpr.newNode.add(
+    result = nimTypeOf(scm, known, sn.child)
+    result.node = nkBracketExpr.newNode.add(
         ident"seq",
-        nimTypeOf(scm, known, sn.child))
+        parameterize(result))
   of snkSetOf:
-    result = nkBracketExpr.newNode.add(
+    result = nimTypeOf(scm, known, sn.child)
+    result.node = nkBracketExpr.newNode.add(
         ident"HashedSet",
-        nimTypeOf(scm, known, sn.child))
+        parameterize(result))
   of snkDictOf:
-    result = nkBracketExpr.newNode.add(
+    let
-        ident"Table",
+      key = nimTypeOf(scm, known, sn.nodes[0])
-        nimTypeOf(scm, known, sn.nodes[0]),
+      val = nimTypeOf(scm, known, sn.nodes[1])
-        nimTypeOf(scm, known, sn.nodes[1]))
+    orEmbed result, key
+    orEmbed result, val
+    result.node = nkBracketExpr.newNode.add(
+      ident"TableRef", parameterize(key), parameterize(val))
   of snkRecord:
     case sn.nodes.len
     of 0, 1:
-      result = nn(nkObjectTy,
+      result.node = nn(nkObjectTy,
         newEmpty(),
         newEmpty(),
         nn(nkRecList, nn(nkDiscardStmt, newEmpty())))
@@ -247,62 +320,55 @@ proc nimTypeOf(scm: Schema; known: var TypeTable; sn: SchemaNode; name = ""): PN
       let recList = nkRecList.newNode()
       for i, field in sn.nodes:
         if i > 0:
-          let id = field.ident
+          let
-          recList.add nkIdentDefs.newNode.add(
+            id = field.ident
-            id.toExport,
+            fieldType = nimTypeOf(scm, known, field, $id)
-            nimTypeOf(scm, known, field, $id),
+          orEmbed result, fieldType
-            newEmpty())
+          recList.add identDef(sn, id.toExport, fieldType)
-      result = nn(nkRefTy, nn(nkObjectTy,
+      result.node = nn(nkRefTy, nn(nkObjectTy,
         newEmpty(),
         newEmpty(),
         recList))
   of snkTuple:
-    # TODO: the variable part
+    result.node = nkTupleTy.newNode
-    result = nkTupleTy.newNode
     for tn in sn.nodes:
       if not isConst(scm, sn):
-        result.add nkIdentDefs.newNode.add(
+        let fieldType = nimTypeOf(scm, known, tn)
-          tn.ident, nimTypeOf(scm, known, tn), newEmpty())
+        orEmbed result, fieldType
+        result.node.add identDef(sn, tn.ident, fieldType)
   of snkVariableTuple:
-    result = nkTupleTy.newNode
+    result.node = nkTupleTy.newNode
     for i, tn in sn.nodes:
       if not isConst(scm, sn):
+        let fieldType = nimTypeOf(scm, known, tn)
+        orEmbed result, fieldType
         if i == sn.nodes.high:
-          result.add nkIdentDefs.newNode.add(
+          result.node.add identDef(
+            tn,
             tn.ident,
-            nn(nkBracketExpr, ident"seq", nimTypeOf(scm, known, tn)),
+            nn(nkBracketExpr, ident"seq", fieldType.node),
-            newEmpty())
+            fieldType.embeddable)
         else:
-          result.add nkIdentDefs.newNode.add(
+          result.node.add identDef(tn, tn.ident, fieldType)
-            tn.ident, nimTypeOf(scm, known, tn), newEmpty())
   of snkDictionary:
-    result = nkTupleTy.newNode
+    result.node = nkTupleTy.newNode
     for i in countup(0, sn.nodes.high, 2):
       let id = ident(sn.nodes[i+0])
-      result.add nkIdentDefs.newNode.add(
+      let fieldType = nimTypeOf(scm, known, sn.nodes[i+1], $id)
-        id,
+      orEmbed result, fieldType
-        nimTypeOf(scm, known, sn.nodes[i+1], $id),
+      result.node.add identDef(sn.nodes[i+1], id, fieldType)
-        newEmpty())
   of snkNamed:
     result = nimTypeOf(scm, known, sn.pattern, name)
   of snkRef:
-    if sn.refPath.len == 1:
+    result = typeIdent(scm, sn)
-      let
-        refName = sn.refPath[0]
-        refDef = scm.definitions[refName]
-      case refDef.kind
-      of snkDictOf:
-        result = nimTypeOf(scm, known, refDef, refName)
-      else: result = typeIdent(sn)
-    else:
-      result = typeIdent(sn)
   else:
-    result = nkCommentStmt.newNode
+    result.node = nkCommentStmt.newNode
-    result.comment = result.comment & "Missing type generator for " & $sn.kind & " " & $sn
+    result.node.comment = result.node.comment &
+      "Missing type generator for " & $sn.kind & " " & $sn
 
 proc exportIdent(id: string): PNode = nn(nkPostFix, ident"*", ident(id))
 
-proc generateConstProcs(result: var seq[PNode]; name: string; def: SchemaNode) =
+proc generateConstProcs(result: var seq[PNode]; scm: Schema, name: string; def: SchemaNode) =
   case def.kind
   of snkLiteral:
     var stmts = nn(nkStmtList)
@@ -311,7 +377,7 @@ proc generateConstProcs(result: var seq[PNode]; name: string; def: SchemaNode) =
       discard stmts.add newIntNode(nkIntLit, def.value.int)
     of pkSymbol:
       discard stmts.add nn(nkCall,
-        ident"symbol",
+        nn(nkBracketExpr, ident"symbol", ident"E"),
         PNode(kind: nkStrLit, strVal: def.value.symbol))
     else:
       raiseAssert("conversion of " & $def & " to a Nim literal is not implemented")
@@ -320,8 +386,8 @@ proc generateConstProcs(result: var seq[PNode]; name: string; def: SchemaNode) =
     let constProc= nn(nkProcDef,
       exportIdent(procId),
       newEmpty(),
-      newEmpty(),
+      nn(nkGenericParams, nn(nkIdentDefs, ident"E", newEmpty(), ident"void")),
-      nn(nkFormalParams, ident"Preserve"),
+      nn(nkFormalParams, preserveIdent()),
       newEmpty(),
       newEmpty(),
       stmts)
@@ -329,18 +395,18 @@ proc generateConstProcs(result: var seq[PNode]; name: string; def: SchemaNode) =
     result.add constProc
   else: discard
 
-proc toNimLit(sn: SchemaNode): PNode =
+proc nimLit(scm: Schema; sn: SchemaNode): PNode =
   assert(sn.kind == snkLiteral, $sn)
   case sn.value.kind
   of pkSymbol:
-    nkCall.newNode.add(
+    nn(nkCall,
-      ident"symbol",
+      nn(nkBracketExpr, ident"symbol", ident"E"),
       PNode(kind: nkStrLit, strVal: sn.value.symbol))
   else:
     raiseAssert("no Nim literal for " & $sn)
 
 proc literalToPreserveCall(pr: Preserve): PNode =
-  var prConstr = nn(nkObjConstr, ident"Preserve")
+  var prConstr = nn(nkObjConstr, preserveIdent())
   proc constr(kind, field: string; lit: PNode) =
     prConstr.add nn(nkExprColonExpr, ident"kind", ident(kind))
     prConstr.add nn(nkExprColonExpr, ident(field), lit)
@@ -370,8 +436,7 @@ proc tupleConstructor(scm: Schema; sn: SchemaNode; ident: PNode): Pnode =
       seqBracket.add literalToPreserveCall(literal(scm, field))
     elif sn.kind == snkTuple or i < sn.nodes.high:
       seqBracket.add nn(nkCall,
-        ident"toPreserve",
+        ident"toPreserve", nn(nkDotExpr, ident, field.ident), ident"E")
-        nn(nkDotExpr, ident, field.ident))
   let seqConstr = nn(nkPrefix, ident"@", seqBracket)
   let colonExpr = nn(nkExprColonExpr, ident"sequence")
   if sn.kind == snkTuple:
@@ -383,10 +448,11 @@ proc tupleConstructor(scm: Schema; sn: SchemaNode; ident: PNode): Pnode =
       nn(nkDotExpr,
         nn(nkCall, ident"toPreserve",
           nn(nkDotExpr,
-            ident, sn.nodes[sn.nodes.high].ident)),
+            ident, sn.nodes[sn.nodes.high].ident),
+          ident"E"),
         ident"sequence"))
   nn(nkObjConstr,
-    ident"Preserve",
+    preserveIdent(),
     nn(nkExprColonExpr, ident"kind", ident"pkSequence"),
     colonExpr)
 
@@ -397,7 +463,7 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
       enumId = name.ident
       paramId = ident"v"
       orStmts = nn(nkStmtList)
-    if sn.isSymbolEnum:
+    if isSymbolEnum(scm, sn):
       let caseStmt = nn(nkCaseStmt, paramId)
       for bn in sn.nodes:
         caseStmt.add nn(nkOfBranch,
@@ -405,7 +471,8 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
             enumId,
             bn.altLabel.nimIdentNormalize.ident.accQuote),
           nn(nkCall,
-            ident"symbol", PNode(kind: nkStrLit, strVal: $bn.altLabel)))
+            nn(nkBracketExpr, ident"symbol", ident"E"),
+            PNode(kind: nkStrLit, strVal: $bn.altLabel)))
       orStmts.add caseStmt
     else:
       let caseStmt = nn(nkCaseStmt, nn(nkDotExpr, paramId, ident"kind"))
@@ -420,13 +487,13 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
           else:
             stmts.add nn(nkCall,
               ident"toPreserve",
-              nn(nkDotExpr, paramId, fieldId))
+              nn(nkDotExpr, paramId, fieldId), ident"E")
         of snkTuple, snkVariableTuple:
           stmts.add tupleConstructor(scm, sn, nn(nkDotExpr, paramId, fieldId))
         of snkAtom, snkSequenceOf:
           stmts.add nn(nkCall,
             ident"toPreserve",
-            nn(nkDotExpr, paramId, fieldId))
+            nn(nkDotExpr, paramId, fieldId), ident"E")
         else:
           raiseAssert("no case statement for " & $sn.kind & " " & $sn)
       for bn in sn.nodes:
@@ -443,41 +510,43 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
       newEmpty(),
       newEmpty(),
       nn(nkFormalParams,
-        ident"Preserve",
+        preserveIdent(),
         nn(nkIdentDefs,
-          paramId, ident(name), newEmpty())),
+          paramId, ident(name), newEmpty()),
+        nn(nkIdentDefs,
+          ident"E", ident"typedesc", newEmpty())),
       newEmpty(),
       newEmpty(),
       orStmts)
   of snkRecord:
     var
-      params = nn(nkFormalParams, ident"Preserve")
+      params = nn(nkFormalParams, preserveIdent())
       initRecordCall = nn(nkCall,
-        ident"initRecord",
+        nn(nkBracketExpr, ident"initRecord", ident"E"),
-        sn.nodes[0].toNimLit)
+        nimLit(scm, sn.nodes[0]))
     for i, field in sn.nodes:
       if i > 0:
         let
           id = field.ident
-        var fieldType  = field.typeIdent
+        var (fieldType, embeddable) = typeIdent(scm, field)
-        if fieldType.kind != nkIdent or fieldType.ident.s != "Preserve":
+        if not fieldType.isPreserve:
           fieldType =
             nn(nkInfix,
               ident"|",
               fieldType,
-              ident"Preserve")
+              preserveIdent())
         params.add nn(nkIdentDefs,
           id, fieldType, newEmpty())
         initRecordCall.add(
-          nn(nkCall, ident"toPreserve", id))
+          nn(nkCall, ident"toPreserve", id, ident"E"))
     var procId = name
     procId[0] = procId[0].toLowerAscii
     let cmt = nkCommentStmt.newNode
     cmt.comment = "Preserves constructor for ``" & name & "``."
     result.add nn(nkProcDef,
-      exportIdent(procId),
+      procId.ident.accQuote.toExport,
-      newEmpty(),
       newEmpty(),
+      nn(nkGenericParams, nn(nkIdentDefs, ident"E", newEmpty(), ident"void")),
       params,
       newEmpty(),
       newEmpty(),
@@ -486,21 +555,24 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
     block:
       let paramId = name.toLowerAscii.ident.accQuote
       initRecordCall = nn(nkCall,
-        ident"initRecord",
+        nn(nkBracketExpr, ident"initRecord", ident"E"),
-        sn.nodes[0].toNimLit)
+        nimLit(scm, sn.nodes[0]))
       for i, field in sn.nodes:
         if i > 0:
           initRecordCall.add nn(nkCall,
             ident"toPreserve",
-            nn(nkDotExpr, paramId, field.ident))
+            nn(nkDotExpr, paramId, field.ident),
+            ident"E")
       result.add nn(nkProcDef,
         exportIdent("toPreserveHook"),
         newEmpty(),
         newEmpty(),
         nn(nkFormalParams,
-          ident"Preserve",
+          preserveIdent(),
           nn(nkIdentDefs,
-            paramId, ident(name), newEmpty())),
+            paramId, ident(name), newEmpty()),
+          nn(nkIdentDefs,
+            ident"E", ident"typedesc", newEmpty())),
         newEmpty(),
         newEmpty(),
         nn(nkStmtList, initRecordCall))
@@ -511,9 +583,11 @@ proc generateProcs(result: var seq[PNode]; scm: Schema; name: string; sn: Schema
       newEmpty(),
       newEmpty(),
       nn(nkFormalParams,
-        ident"Preserve",
+        preserveIdent(),
         nn(nkIdentDefs,
-          paramId, ident(name), newEmpty())),
+          paramId, ident(name), newEmpty()),
+        nn(nkIdentDefs,
+          ident"E", ident"typedesc", newEmpty())),
       newEmpty(),
       newEmpty(),
       nn(nkStmtList, tupleConstructor(scm, sn, paramId)))
@@ -534,9 +608,6 @@ proc collectRefImports(imports: PNode; sn: SchemaNode) =
       collectRefImports(imports, child)
 
 proc collectRefImports(imports: PNode; scm: Schema) =
-  if scm.embeddedType.contains {'.'}:
-    let m = split(scm.embeddedType, '.', 1 )[0]
-    imports.add ident(m)
   for _, def in scm.definitions:
     collectRefImports(imports, def)
 
@@ -548,35 +619,39 @@ proc generateNimFile*(scm: Schema; path: string) =
     megaType: PNode
   for name, def in scm.definitions.pairs:
     if isConst(scm, def):
-      generateConstProcs(procs, name, def)
+      generateConstProcs(procs, scm, name, def)
     else:
       var name = name
       name[0] = name[0].toUpperAscii
+      var defIdent = parameterize(ident(name), isEmbeddable(scm, def))
       if megaType.isNil:
-        megaType = ident(name)
+        megaType = defIdent
       else:
         megaType = nn(nkInfix,
-          ident"|", megaType, ident(name))
+          ident"|", megaType, defIdent)
-      let t = nimTypeOf(scm, knownTypes, def, name)
+      let (t, embeddable) =
+        if def.kind == snkAny: (preserveIdent(), true)
+        else: nimTypeOf(scm, knownTypes, def, name)
       t.comment = "``" & $def & "``"
       case def.kind
       of snkAtom:
-        knownTypes[name] = nkTypeDef.newNode.add(
+        knownTypes[name] = (nkTypeDef.newNode.add(
-          name.ident.toExport, newEmpty(), t)
+          name.ident.toExport, embeddingParams(false), t),
+          embeddable)
       else:
         if def.kind == snkRecord:
-          knownTypes[name] = typeDef(def, name, t)
+          knownTypes[name] = (typeDef(def, name, t, embeddable), embeddable)
         else:
           case t.kind
           of nkEnumTy:
-            knownTypes[name] = nn(nkTypeDef,
+            knownTypes[name] = (nn(nkTypeDef,
               nn(nkPragmaExpr,
                 name.ident.toExport,
                 nn(nkPragma, ident"pure")),
                 newEmpty(),
-                t)
+                t), false)
           of nkDistinctTy:
-            knownTypes[name] = nn(nkTypeDef,
+            knownTypes[name] = (nn(nkTypeDef,
               nn(nkPragmaExpr,
                 name.ident.toExport,
                 nn(nkPragma,
@@ -584,12 +659,13 @@ proc generateNimFile*(scm: Schema; path: string) =
                     ident"borrow",
                     accQuote(ident".")))),
                 newEmpty(),
-                t)
+                t), embeddable)
           else:
-            knownTypes[name] = nn(nkTypeDef,
+            knownTypes[name] = (nn(nkTypeDef,
-              name.ident.toExport, newEmpty(), t)
+              name.ident.toExport, embeddingParams(embeddable), t),
+              embeddable)
       generateProcs(procs, scm, name, def)
-  for typeDef in knownTypes.values:
+  for (typeDef, _) in knownTypes.values:
     typeSection.add typeDef
   var imports = nkImportStmt.newNode.add(
     ident"std/typetraits",
@@ -598,7 +674,7 @@ proc generateNimFile*(scm: Schema; path: string) =
   procs.add nn(nkProcDef,
       "$".ident.accQuote.toExport,
       newEmpty(),
-      newEmpty(),
+      nn(nkGenericParams, nn(nkIdentDefs, ident"E", newEmpty(), newEmpty())),
       nn(nkFormalParams,
         ident"string",
         nn(nkIdentDefs,
@@ -608,11 +684,12 @@ proc generateNimFile*(scm: Schema; path: string) =
       newEmpty(),
       newEmpty(),
       nn(nkStmtList,
-        nn(nkCall, ident"$", nn(nkCall, ident"toPreserve", ident"x"))))
+        nn(nkCall, ident"$",
+          nn(nkCall, ident"toPreserve", ident"x", ident"E"))))
   procs.add nn(nkProcDef,
       "encode".ident.accQuote.toExport,
       newEmpty(),
-      newEmpty(),
+      nn(nkGenericParams, nn(nkIdentDefs, ident"E", newEmpty(), newEmpty())),
       nn(nkFormalParams,
         nn(nkBracketExpr, ident"seq", ident"byte"),
         nn(nkIdentDefs,
@@ -622,7 +699,8 @@ proc generateNimFile*(scm: Schema; path: string) =
       newEmpty(),
       newEmpty(),
       nn(nkStmtList,
-        nn(nkCall, ident"encode", nn(nkCall, ident"toPreserve", ident"x"))))
+        nn(nkCall, ident"encode", nn(nkCall,
+          ident"toPreserve", ident"x", ident"E"))))
   var module = newNode(nkStmtList).add(
       imports,
       typeSection

src/preserves/records.nim

@@ -1,64 +0,0 @@
-# SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
-# SPDX-License-Identifier: Unlicense
-
-import std/[macros, typetraits]
-import ../preserves
-
-type RecordClass* = object
-  ## Type of a preserves record.
-  label*: Preserve
-  arity*: Natural
-
-proc `$`*(rec: RecordClass): string =
-  $rec.label & "/" & $rec.arity
-
-proc isClassOf*(rec: RecordClass; val: Preserve): bool =
-  ## Compare the label and arity of ``val`` to the record type ``rec``.
-  if val.kind == pkRecord:
-    assert(val.record.len > 0)
-    result = val.label == rec.label and rec.arity == val.arity
-
-proc classOf*(val: Preserve): RecordClass =
-  ## Derive the ``RecordClass`` of ``val``.
-  if val.kind != pkRecord:
-    raise newException(Defect, "cannot derive class of non-record value " & $val)
-  assert(val.record.len > 0)
-  RecordClass(label: val.label, arity: val.arity)
-
-proc classOf*[T](x: T): RecordClass =
-  ## Derive the ``RecordClass`` of ``x``.
-  when not T.hasCustomPragma(record): {.error: "no {.record.} pragma on " & $T.}
-  result.label = preserves.symbol(T.getCustomPragmaVal(record))
-  for k, v in x.fieldPairs: inc(result.arity)
-
-proc classOf*(T: typedesc[tuple]): RecordClass =
-  ## Derive the ``RecordClass`` of ``T``.
-  when not T.hasCustomPragma(record): {.error: "no {.record.} pragma on " & $T.}
-  RecordClass(
-      label: preserves.symbol(T.getCustomPragmaVal(record)),
-      arity: tupleLen(T))
-
-proc init*(rec: RecordClass; fields: varargs[Preserve, toPreserve]): Preserve =
-  ## Initialize a new record value.
-  assert(fields.len == rec.arity, $(fields.toPreserve) & " (arity " & $fields.len & ") is not of arity " & $rec.arity)
-  result = initRecord(rec.label, fields)
-
-proc init*(T: typedesc[tuple]; fields: varargs[Preserve, toPreserve]): Preserve =
-  ## Initialize a new record value.
-  init(classOf(T), fields)
-
-proc `%`*(rec: RecordClass; fields: openArray[Preserve]): Preserve =
-  ## Initialize a simple record value.
-  init(rec, fields)
-
-proc `%`*(rec: RecordClass; field: Preserve): Preserve =
-  ## Initialize a simple record value.
-  init(rec, [field])
-
-proc `%`*[T](rec: RecordClass; field: T): Preserve =
-  ## Initialize a simple record value.
-  init(rec, [toPreserve field])
-
-proc `%`*(T: typedesc[tuple]; fields: varargs[Preserve, toPreserve]): Preserve =
-  ## Initialize a new record value.
-  `%`(classOf(T), fields)

src/preserves/schemas.nim

@@ -11,6 +11,8 @@ import npeg
 import ../preserves, ./parse, ./pegs
 
 type
+  Value = Preserve[void]
+
   Stack = seq[tuple[node: SchemaNode, pos: int]]
 
   SchemaNodeKind* = enum
@@ -52,7 +54,7 @@ type
     of snkEmbedded:
       embed*: SchemaNode
     of snkLiteral:
-      value*: Preserve
+      value*: Value
     of snkNamed:
       name*: string
       pattern*: SchemaNode
@@ -271,7 +273,7 @@ const parser = peg("Schema", p: ParseState):
 
   AltRecord <- '<' * >id * *(S * NamedPattern) * '>':
     let
-      id = SchemaNode(kind: snkLiteral, value: symbol($1))
+      id = SchemaNode(kind: snkLiteral, value: symbol[void]($1))
       n = SchemaNode(kind: snkAlt,
           altLabel: $1,
           altBranch: snkRecord.newSchemaNode.add(id).add(takeStackAt()))

tests/test_conversions.nim

@@ -1,8 +1,8 @@
 # SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[options, unittest]
+import std/[options, tables, unittest]
-import bigints, preserves, preserves/records
+import bigints, preserves
 
 suite "conversions":
   test "dictionary":
@@ -30,7 +30,15 @@ suite "conversions":
     check(prs.kind == pkRecord)
     check($prs == """<foo 1 2 <bar "ku">>""")
     check(preserveTo(prs, Foobar) == some(tup))
-    check(classOf(tup) == classOf(prs))
+
+  test "tables":
+    var a: Table[int, string]
+    for i, s in ["a", "b", "c"]: a[i] = s
+    let b = toPreserve(a)
+    check($b == """{0: "a" 1: "b" 2: "c"}""")
+    var c: Table[int, string]
+    check(fromPreserve(c, b))
+    check(a == c)
 
 suite "%":
   template check(p: Preserve; s: string) =