preserves_schema_nim: remove dead code

Use EmbeddedRef for embbeded values, unless a schema does not name
an embedded type.

.envrc

@@ -1,2 +0,0 @@
-source_env ..
-use nix

depends.tup

@@ -1 +1,2 @@
 NIM_FLAGS += --path:$(TUP_CWD)/../nim
+NIM_FLAGS += --path:$(TUP_CWD)/../npeg/src

lock.json

@@ -1,5 +1,17 @@
 {
   "depends": [
+    {
+      "method": "fetchzip",
+      "packages": [
+        "bigints"
+      ],
+      "path": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source",
+      "ref": "20231006",
+      "rev": "86ea14d31eea9275e1408ca34e6bfe9c99989a96",
+      "sha256": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4",
+      "srcDir": "src",
+      "url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz"
+    },
     {
       "method": "fetchzip",
       "packages": [

preserves.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version = "20231220"
+version = "20240108"
 author        = "Emery Hemingway"
 description   = "data model and serialization format"
 license       = "Unlicense"
@@ -11,4 +11,4 @@ bin           = @["preserves/preserves_schema_nim", "preserves/private/preserves
 
 # Dependencies
 
-requires "nim >= 2.0.0", "compiler >= 1.4.8", "https://github.com/zevv/npeg.git >= 1.2.1"
+requires "nim >= 2.0.0", "compiler >= 2.0.0", "https://github.com/zevv/npeg.git >= 1.2.1", "https://github.com/ehmry/nim-bigints.git >= 20231006"

shell.nix

@@ -1,5 +0,0 @@
-{ pkgs ? import <nixpkgs> { } }:
-pkgs.buildNimPackage {
-  name = "dummy";
-  lockFile = ./lock.json;
-}

src/Tupfile

@@ -1,2 +1,2 @@
 include_rules
-: preserves.nim |> !nim_run |>
+: preserves.nim |> !nim_check |>

src/preserves/Tupfile

@@ -1,7 +1,10 @@
 include_rules
 NIM_FLAGS += --path:$(TUP_CWD)/..
+NIM_FLAGS_preserves_schemac += -d:npegDotDir="../.."
+
 : foreach preserves_schema_nim.nim schemaparse.nim |> !nim_bin |> $(BIN_DIR)/%B | $(BIN_DIR)/<%B>
 
-DOT_FILES = ../../Document.dot ../../Schema.dot
+DOT_FILES = ../../Atom.dot ../../Document.dot ../../Schema.dot
-: schemac.nim |> !nim_bin |> $(BIN_DIR)/%B | $(DOT_FILES) $(BIN_DIR)/<%B>
+: preserves_schemac.nim |> !nim_bin |> $(BIN_DIR)/preserves-schemac | $(DOT_FILES) $(BIN_DIR)/<preserves-schemac>
 : foreach $(DOT_FILES) |> dot -Tsvg -LO %f > %o |> ../../%B-Grammer-Graph.svg
+: foreach *hooks.nim  |> !nim_run |>

src/preserves/datehooks.nim

@@ -11,10 +11,10 @@ const
   fullTimeFormat = "HH:mm:sszzz"
   dateTimeFormat = "yyyy-MM-dd'T'HH:mm:sszzz"
 
-proc toPreserveHook*(dt: DateTime; E: typedesc): Preserve[E] =
+proc toPreservesHook*(dt: DateTime): Value =
-  initRecord[E](toSymbol("rfc3339", E), toPreserve($dt, E))
+  initRecord("rfc3339", toPreserves($dt))
 
-proc fromPreserveHook*[E](dt: var DateTime; pr: Preserve[E]): bool =
+proc fromPreservesHook*(dt: var DateTime; pr: Value): bool =
   result = pr.isRecord(label, 1) and pr.record[0].isString
   if result:
     try:
@@ -39,6 +39,6 @@ runnableExamples:
   import preserves
   var a, b: DateTime
   a = now()
-  var pr = a.toPreserveHook(void)
+  var pr = a.toPreservesHook()
-  check fromPreserveHook(b, pr)
+  check b.fromPreservesHook(pr)
   check $a == $b

src/preserves/jsonhooks.nim

@@ -4,51 +4,51 @@
 import std/[json, tables]
 import ../preserves
 
-proc toPreserveHook*(js: JsonNode; E: typedesc): Preserve[E] =
+proc toPreservesHook*(js: JsonNode): Value =
   case js.kind
   of JString:
-    result = Preserve[E](kind: pkString, string: js.str)
+    result = js.str.toPreserves()
   of JInt:
-    result = Preserve[E](kind: pkSignedInteger, int: js.num)
+    result = js.num.toPreserves()
   of JFloat:
-    result = Preserve[E](kind: pkDouble, double: js.fnum)
+    result = js.fnum.toPreserves()
   of JBool:
     result = case js.bval
-      of false: toSymbol("false", E)
+      of false: toSymbol("false")
-      of true: toSymbol("true", E)
+      of true: toSymbol("true")
   of JNull:
-    result = toSymbol("null", E)
+    result = toSymbol("null")
   of JObject:
-    result = Preserve[E](kind: pkDictionary)
+    result = Value(kind: pkDictionary)
     for key, val in js.fields.pairs:
-      result[Preserve[E](kind: pkSymbol, symbol: Symbol key)] = toPreserveHook(val, E)
+      result[Value(kind: pkSymbol, symbol: Symbol key)] = toPreservesHook(val)
   of JArray:
-    result = Preserve[E](kind: pkSequence,
+    result = Value(kind: pkSequence,
-        sequence: newSeq[Preserve[E]](js.elems.len))
+        sequence: newSeq[Value](js.elems.len))
     for i, e in js.elems:
-      result.sequence[i] = toPreserveHook(e, E)
+      result.sequence[i] = toPreservesHook(e)
 
-proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
+proc fromPreservesHook*(js: var JsonNode; pr: Value): bool =
   runnableExamples:
     import std/json
     var js = JsonNode()
-    var pr = js.toPreserveHook(void)
+    var pr = js.toPreservesHook()
-    assert fromPreserveHook(js, pr)
+    assert js.fromPreservesHook(pr)
     fromJsonHook(pr, js)
     js = toJsonHook(pr)
-  case prs.kind:
+  case pr.kind:
   of pkBoolean:
-    js = newJBool(prs.bool)
+    js = newJBool(pr.bool)
   of pkFloat:
-    js = newJFloat(prs.float)
+    js = newJFloat(pr.float)
   of pkDouble:
-    js = newJFloat(prs.double)
+    js = newJFloat(pr.double)
-  of pkSignedInteger:
+  of pkRegister:
-    js = newJInt(prs.int)
+    js = newJInt(pr.register)
   of pkString:
-    js = newJString(prs.string)
+    js = newJString(pr.string)
   of pkSymbol:
-    case prs.symbol.string
+    case pr.symbol.string
     of "false":
       js = newJBool(false)
     of "true":
@@ -59,38 +59,38 @@ proc fromPreserveHook*[E](js: var JsonNode; prs: Preserve[E]): bool =
       return false
   of pkSequence:
     js = newJArray()
-    js.elems.setLen(prs.sequence.len)
+    js.elems.setLen(pr.sequence.len)
-    for i, val in prs.sequence:
+    for i, val in pr.sequence:
-      if not fromPreserveHook(js.elems[i], val):
+      if not js.elems[i].fromPreservesHook(val):
         return false
   of pkSet:
     js = newJArray()
-    js.elems.setLen(prs.set.len)
+    js.elems.setLen(pr.set.len)
     var i: int
-    for val in prs.set:
+    for val in pr.set:
-      if not fromPreserveHook(js.elems[i], val):
+      if not js.elems[i].fromPreservesHook(val):
         return false
       inc i
   of pkDictionary:
     js = newJObject()
-    for (key, val) in prs.dict.items:
+    for (key, val) in pr.dict.items:
       case key.kind
       of pkSymbol:
         var jsVal: JsonNode
-        if not fromPreserveHook(jsVal, val): return false
+        if not jsVal.fromPreservesHook(val): return false
         js[string key.symbol] = jsVal
       of pkString:
         var jsVal: JsonNode
-        if not fromPreserveHook(jsVal, val): return false
+        if not jsVal.fromPreservesHook(val): return false
         js[key.string] = jsVal
       else:
         return false
   else: return false
   true
 
-proc toJsonHook*[E](pr: Preserve[E]): JsonNode =
+proc toJsonHook*(pr: Value): JsonNode =
-  if not fromPreserveHook(result, pr):
+  if not result.fromPreservesHook(pr):
     raise newException(ValueError, "cannot convert Preserves value to JSON")
 
-proc fromJsonHook*[E](pr: var Preserve[E]; js: JsonNode) =
+proc fromJsonHook*(pr: var Value; js: JsonNode) =
-  pr = toPreserveHook(js, E)
+  pr = toPreservesHook(js)

src/preserves/pegs.nim

@@ -9,20 +9,21 @@ when defined(nimHasUsed): {.used.}
 
 grammar "Preserves":
 
-  ws <- *(' ' | '\t' | '\r' | '\n' )
+  ws <- *{ ' ', '\t', '\r', '\n' }
   commas <- *(ws * ',') * ws
+  delimiter <- { ' ', '\t', '\r', '\n', '<', '>', '[', ']', '{', '}', '#', ':', '"', '|', '@', ';', ',' } | !1
 
   Document <- Value * ws * !1
 
+  Atom <- Boolean | Float | Double | FloatRaw | DoubleRaw | SignedInteger | String | ByteString | Symbol
+
+  Collection <- Sequence | Dictionary | Set
+
   Value <-
       (ws * (Record | Collection | Atom | Embedded | Compact)) |
       (ws * Annotation) |
       (ws * '#' * @'\n' * Value)
 
-  Collection <- Sequence | Dictionary | Set
-
-  Atom <- Boolean | Float | Double | FloatRaw | DoubleRaw | SignedInteger | String | ByteString | Symbol
-
   Record <- '<' * +Value * ws * '>'
 
   Sequence <- '[' * *(commas * Value) * commas * ']'
@@ -31,37 +32,42 @@ grammar "Preserves":
 
   Set <- "#{" * *(commas * Value) * commas * '}'
 
-  Boolean <- "#f" | "#t"
+  Boolean <- '#' * {'f', 't'} * &delimiter
 
-  nat <- '0' | (Digit-'0') * *Digit
+  nat <- +Digit
-  int <- ?'-' * nat
+  int <- ?('-'|'+') * nat
   frac <- '.' * +Digit
   exp <- 'e' * ?('-'|'+') * +Digit
   flt <- int * ((frac * exp) | frac | exp)
 
-  Float <- >flt * 'f'
+  Float <- >flt * {'f','F'} * &delimiter
-  Double <- flt
+  Double <- flt * &delimiter
 
-  SignedInteger <- int
+  SignedInteger <- int * &delimiter
 
-  char <- unescaped | '|' | (escape * (escaped | '"' | ('u' * Xdigit[4])))
+  unescaped <- utf8.any - { '\x00'..'\x19', '"', '\\', '|' }
+  unicodeEscaped <- 'u' * Xdigit[4]
+  escaped <- {'\\', '/', 'b', 'f', 'n', 'r', 't'}
+  escape <- '\\'
+
+  char <- unescaped | '|' | (escape * (escaped | '"' | unicodeEscaped))
   String <- '"' * >(*char) * '"'
 
+  binunescaped <- {' '..'!', '#'..'[', ']'..'~'}
+  binchar <- binunescaped | (escape * (escaped | '"' | ('x' * Xdigit[2])))
+
   ByteString <- charByteString | hexByteString | b64ByteString
   charByteString <- "#\"" * >(*binchar) * '"'
   hexByteString <- "#x\"" * >(*(ws * Xdigit[2])) * ws * '"'
   base64char <- {'A'..'Z', 'a'..'z', '0'..'9', '+', '/', '-', '_', '='}
   b64ByteString <- "#[" * >(*(ws * base64char)) * ws * ']'
 
-  binchar <- binunescaped | (escape * (escaped | '"' | ('x' * Xdigit[2])))
+  symchar <- (utf8.any - {'\\', '|'}) | (escape * (escaped | unicodeEscaped)) | "\\|"
-  binunescaped <- {' '..'!', '#'..'[', ']'..'~'}
-
-  symchar <- (utf8.any - { 0..127, '\\', '|' }) | (escape * (escaped | ('u' * Xdigit[4]))) | "\\|"
   QuotedSymbol <- '|' * >(*symchar) * '|'
   sympunct <- {'~', '!', '$', '%', '^', '&', '*', '?', '_', '=', '+', '-', '/', '.'}
   symuchar <- utf8.any - { 0..127 }
   SymbolOrNumber <- >(+(Alpha | Digit | sympunct | symuchar))
-  Symbol <- QuotedSymbol | SymbolOrNumber
+  Symbol <- QuotedSymbol | (SymbolOrNumber * &delimiter)
 
   Embedded <- "#!" * Value
 
@@ -69,10 +75,5 @@ grammar "Preserves":
 
   Compact <- "#=" * ws * ByteString
 
-  unescaped <- utf8.any - { '\x00'..'\x19', '"', '\\', '|' }
-  unicodeEscaped <- 'u' * Xdigit[4]
-  escaped <- {'\\', '/', 'b', 'f', 'n', 'r', 't'}
-  escape <- '\\'
-
   FloatRaw <- "#xf\"" * >((ws * Xdigit[2])[4]) * ws * '"'
   DoubleRaw <- "#xd\"" * >((ws * Xdigit[2])[8]) * ws * '"'

src/preserves/preserves_schema_nim.nim

@@ -6,9 +6,10 @@
 # the schema module must be regenerated!
 # nim c --path:../../../nim --path:.. -r ./preserves_schema_nim ../../../preserves/schema/schema.bin
 
-import std/[hashes, strutils, sets, tables]
+import std/[hashes, sets, strutils, tables]
 
-# Cannot use std/macros, must uss compiler because we are generating code at run-time.
+# Cannot use std/macros, must use compiler modules because
+# we are generating code at run-time.
 import compiler/[ast, idents, renderer, lineinfos]
 
 import ../preserves, ./schema
@@ -16,10 +17,7 @@ import ../preserves, ./schema
 type
   Attribute = enum
     embedded
-      ## type contains an embedded value and
+      ## type contains an embedded value
-      ## must take an parameter
-    recursive
-      ## type is recursive and therefore must be a ref
   Attributes = set[Attribute]
   TypeSpec = object
     node: PNode
@@ -96,7 +94,7 @@ proc isEmbedded(ts: TypeSpec): bool =
 
 func isAtomic(r: Ref): bool =
   case r.name.string
-  of "bool",  "float", "double", "int", "string", "bytes", "symbol": true
+  of "bool", "float", "double", "int", "string", "bytes", "symbol": true
   else: false
 
 proc addAttrs(x: var TypeSpec; y: TypeSpec) =
@@ -115,44 +113,27 @@ proc ident(`ref`: Ref): PNode =
     dotExtend(result, `ref`.name.string.capitalizeAscii)
 
 proc deref(loc: Location; r: Ref): (Location, Definition) =
-  result[0] = loc
+  try:
-  if r.module == @[]:
+    result[0] = loc
-    result[1] = loc.bundle.modules[loc.schemaPath].field0.definitions[r.name]
+    if r.module == @[]:
-  else:
+      result[1] = loc.bundle.modules[loc.schemaPath].field0.definitions[r.name]
-    result[0].schemaPath = r.module
+    else:
-    result[1] = loc.bundle.modules[r.module].field0.definitions[r.name]
+      result[0].schemaPath = r.module
+      result[1] = loc.bundle.modules[r.module].field0.definitions[r.name]
+  except KeyError:
+    raise newException(KeyError, "reference not found in bundle: " & $r)
 
 proc hasEmbeddedType(scm: Schema): bool =
   case scm.field0.embeddedType.orKind
   of EmbeddedtypenameKind.false: false
   of EmbeddedtypenameKind.Ref: true
 
-proc embeddedIdentString(scm: Schema): string =
+proc parameterize(loc: Location; node: PNode; embeddable: bool): PNode = node
-  case scm.field0.embeddedType.orKind
-  of EmbeddedtypenameKind.false:
-    "E"
-  of EmbeddedtypenameKind.Ref:
-    doAssert $scm.field0.embeddedType.ref.name != ""
-    $scm.field0.embeddedType.ref.name
 
-proc embeddedIdent(scm: Schema): PNode =
+proc parameterize(loc: Location; spec: TypeSpec): PNode =
-  ident(embeddedIdentString(scm))
+  parameterize(loc, spec.node, spec.isEmbedded)
 
-proc preserveIdent(scm: Schema): Pnode =
+proc hash(r: Ref): Hash = r.toPreserves.hash
-  if scm.hasEmbeddedType:
-    nkBracketExpr.newTree(ident"Preserve", embeddedIdent(scm))
-  else:
-    nkBracketExpr.newTree(ident"Preserve", ident"void")
-
-proc parameterize(scm: Schema; node: PNode; embeddable: bool): PNode =
-  if embeddable and node.kind notin {nkBracketExpr}:
-    nkBracketExpr.newTree(node, scm.embeddedIdent)
-  else: node
-
-proc parameterize(scm: Schema; spec: TypeSpec): PNode =
-  parameterize(scm, spec.node, spec.isEmbedded)
-
-proc hash(r: Ref): Hash = r.toPreserve.hash
 type RefSet = HashSet[Ref]
 
 proc attrs(loc: Location; pat: Pattern; seen: RefSet): Attributes {.gcsafe.}
@@ -163,10 +144,8 @@ proc attrs(loc: Location; n: NamedAlternative|NamedPattern; seen: RefSet): Attri
 
 proc attrs(loc: Location; sp: SimplePattern; seen: RefSet): Attributes =
   case sp.orKind
-  of SimplepatternKind.atom, SimplepatternKind.lit: {}
+  of SimplepatternKind.atom, SimplepatternKind.lit, SimplepatternKind.any: {}
-  of SimplepatternKind.any, SimplepatternKind.embedded:
+  of SimplepatternKind.embedded: {embedded}
-    if loc.schema.hasEmbeddedType: {embedded}
-    else: {}
   of SimplepatternKind.seqof:
     attrs(loc, sp.seqof.pattern, seen)
   of SimplepatternKind.setof:
@@ -174,8 +153,7 @@ proc attrs(loc: Location; sp: SimplePattern; seen: RefSet): Attributes =
   of SimplepatternKind.dictof:
     attrs(loc, sp.dictof.key, seen) + attrs(loc, sp.dictof.value, seen)
   of SimplepatternKind.Ref:
-    if sp.ref in seen: {recursive}
+    if (sp.ref in seen) or sp.ref.isAtomic: {}
-    elif sp.ref.isAtomic: {}
     else:
       var
         (loc, def) = deref(loc, sp.ref)
@@ -214,32 +192,104 @@ proc attrs(loc: Location; pat: Pattern; seen: RefSet): Attributes =
   of PatternKind.CompoundPattern:
     attrs(loc, pat.compoundPattern, seen)
 
-proc attrs(loc: Location; orDef: DefinitionOr; seen: RefSet): Attributes =
+proc attrs(loc: Location; def: DefinitionOr|DefinitionAnd; seen: RefSet): Attributes =
-  result = attrs(loc, orDef.field0.pattern0, seen) + attrs(loc, orDef.field0.pattern1, seen)
+  result = attrs(loc, def.field0.pattern0, seen) + attrs(loc, def.field0.pattern1, seen)
-  for p in orDef.field0.patternN:
+  for p in def.field0.patternN:
     result = result + attrs(loc, p, seen)
 
 proc attrs(loc: Location; def: Definition; seen: RefSet): Attributes =
   case def.orKind
   of DefinitionKind.or: result = attrs(loc, def.or, seen)
-  of DefinitionKind.and:
+  of DefinitionKind.and: result = attrs(loc, def.and, seen)
-    result =
-      attrs(loc, def.and.field0.pattern0, seen) +
-      attrs(loc, def.and.field0.pattern1, seen)
-    for p in def.and.field0.patternN:
-      result = result + attrs(loc, p, seen)
   of DefinitionKind.Pattern:
     result = attrs(loc, def.pattern, seen)
 
-proc attrs(loc: Location; p: Definition|DefinitionOr|Pattern|CompoundPattern|SimplePattern): Attributes =
+proc attrs(loc: Location; p: Definition|DefinitionOr|DefinitionAnd|Pattern|CompoundPattern|SimplePattern): Attributes =
   var seen: RefSet
   attrs(loc, p, seen)
 
-proc isEmbedded(loc: Location; p: Definition|DefinitionOr|Pattern|CompoundPattern): bool =
+proc isEmbedded(loc: Location; p: Definition|DefinitionOr|DefinitionAnd|Pattern|CompoundPattern|SimplePattern): bool =
   embedded in attrs(loc, p)
 
-proc isRecursive(loc: Location; p: Definition|DefinitionOr|Pattern|CompoundPattern): bool =
+proc isRecursive(loc: Location; name: string; pat: Pattern; seen: RefSet): bool {.gcsafe.}
-  recursive in attrs(loc, p)
+
+proc isRecursive(loc: Location; name: string; def: Definition; seen: RefSet): bool {.gcsafe.}
+
+proc isRecursive(loc: Location; name: string; n: NamedAlternative|NamedPattern; seen: RefSet): bool =
+  isRecursive(loc, name, n.pattern, seen)
+
+proc isRecursive(loc: Location; name: string; sp: SimplePattern; seen: RefSet): bool =
+  case sp.orKind
+  of SimplepatternKind.embedded:
+    isRecursive(loc, name, sp.embedded.interface, seen)
+  of SimplepatternKind.Ref:
+    if sp.ref.name.string == name: true
+    elif sp.ref in seen: false
+    else:
+      var
+        (loc, def) = deref(loc, sp.ref)
+        seen = seen
+      incl(seen, sp.ref)
+      isRecursive(loc, name, def, seen)
+  else:
+    false
+    # seqof, setof, and dictof are not processed
+    # because they imply pointer indirection
+
+proc isRecursive(loc: Location; name: string; np: NamedSimplePattern; seen: RefSet): bool =
+  case np.orKind
+  of NamedSimplePatternKind.named:
+    isRecursive(loc, name, np.named.pattern, seen)
+  of NamedSimplePatternKind.anonymous:
+    isRecursive(loc, name, np.anonymous, seen)
+
+proc isRecursive(loc: Location; name: string; cp: CompoundPattern; seen: RefSet): bool =
+  case cp.orKind
+  of CompoundPatternKind.rec:
+    result =
+      isRecursive(loc, name, cp.rec.label.pattern, seen) or
+      isRecursive(loc, name, cp.rec.fields.pattern, seen)
+  of CompoundPatternKind.tuple:
+    for np in cp.tuple.patterns:
+      if result: return
+      result = isRecursive(loc, name, np.pattern, seen)
+  of CompoundPatternKind.tupleprefix:
+    result = isRecursive(loc, name, cp.tupleprefix.variable, seen)
+    for p in cp.tupleprefix.fixed:
+      if result: return
+      result = isRecursive(loc, name, p, seen)
+  of CompoundPatternKind.dict:
+    for nsp in cp.dict.entries.values:
+      if result: return
+      result = isRecursive(loc, name, nsp, seen)
+
+proc isRecursive(loc: Location; name: string; pat: Pattern; seen: RefSet): bool =
+  case pat.orKind
+  of PatternKind.SimplePattern:
+    isRecursive(loc, name, pat.simplePattern, seen)
+  of PatternKind.CompoundPattern:
+    isRecursive(loc, name, pat.compoundPattern, seen)
+
+proc isRecursive(loc: Location; name: string; def: DefinitionOr|DefinitionAnd; seen: RefSet): bool =
+  result =
+      isRecursive(loc, name, def.field0.pattern0, seen) or
+      isRecursive(loc, name, def.field0.pattern1, seen)
+  for p in def.field0.patternN:
+    if result: return
+    result = isRecursive(loc, name, p, seen)
+
+proc isRecursive(loc: Location; name: string; def: Definition; seen: RefSet): bool =
+  case def.orKind
+  of DefinitionKind.or:
+    isRecursive(loc, name, def.or, seen)
+  of DefinitionKind.and:
+    isRecursive(loc, name, def.and, seen)
+  of DefinitionKind.Pattern:
+    isRecursive(loc, name, def.pattern, seen)
+
+proc isRecursive(loc: Location; name: string; def: Definition): bool =
+  var seen: RefSet
+  isRecursive(loc, name, def, seen)
 
 proc isLiteral(loc: Location; def: Definition): bool {.gcsafe.}
 proc isLiteral(loc: Location; pat: Pattern): bool {.gcsafe.}
@@ -253,7 +303,8 @@ proc isLiteral(loc: Location; sp: SimplePattern): bool =
   of SimplepatternKind.lit:
     result = true
   of SimplepatternKind.embedded:
-    result = isLiteral(loc, sp.embedded.interface)
+    if not loc.schema.hasEmbeddedType:
+      result = isLiteral(loc, sp.embedded.interface)
   else: discard
 
 proc isLiteral(loc: Location; np: NamedPattern): bool =
@@ -277,18 +328,14 @@ proc isLiteral(loc: Location; def: Definition): bool =
 proc isRef(sp: SimplePattern): bool =
   sp.orKind == SimplePatternKind.Ref
 
-proc isRef(pat: Pattern): bool =
-  pat.orKind == PatternKind.SimplePattern and
-    pat.simplePattern.isRef
-
 proc isSimple(pat: Pattern): bool =
   pat.orKind == PatternKind.SimplePattern
 
 proc isLiteral(loc: Location; na: NamedAlternative): bool = isLiteral(loc, na.pattern)
 
-proc isSymbolEnum(loc: Location; orDef: DefinitionOr): bool =
+proc isSymbolEnum(loc: Location; def: DefinitionOr): bool =
-  result = isLiteral(loc, orDef.field0.pattern0) and isLiteral(loc, orDef.field0.pattern1)
+  result = isLiteral(loc, def.field0.pattern0) and isLiteral(loc, def.field0.pattern1)
-  for na in orDef.field0.patternN:
+  for na in def.field0.patternN:
     if not result: break
     result = isLiteral(loc, na)
 
@@ -310,6 +357,41 @@ proc isSymbolEnum(loc: Location; sp: SimplePattern): bool =
     result = isSymbolEnum(loc, def)
   else: discard
 
+proc isDictionary(loc: Location; def: Definition): bool {.gcsafe.}
+
+proc isDictionary(loc: Location; pat: Pattern): bool =
+  case pat.orKind
+  of PatternKind.SimplePattern:
+    case pat.simplePattern.orKind
+    of SimplePatternKind.Ref:
+      var (loc, def) = deref(loc, pat.simplePattern.ref)
+      result = isDictionary(loc, def)
+    of SimplePatternKind.dictof:
+      result = true
+    else: discard
+  of PatternKind.CompoundPattern:
+    case pat.compoundpattern.orKind
+    of CompoundPatternKind.dict:
+      result = true
+    else: discard
+
+proc isDictionary(loc: Location; def: Definition): bool =
+  case def.orKind
+  of DefinitionKind.Pattern:
+    result = isDictionary(loc, def.pattern)
+  of DefinitionKind.or:
+    result =
+      isDictionary(loc, def.or.field0.pattern0.pattern) and
+      isDictionary(loc, def.or.field0.pattern1.pattern)
+    for np in def.or.field0.patternN:
+      if result: result = isDictionary(loc, np.pattern)
+  of DefinitionKind.and:
+    result =
+      isDictionary(loc, def.and.field0.pattern0.pattern) and
+      isDictionary(loc, def.and.field0.pattern1.pattern)
+    for np in def.and.field0.patternN:
+      if result: result = isDictionary(loc, np.pattern)
+
 proc isAny(loc: Location; def: Definition): bool =
   case def.orKind
   of DefinitionKind.Pattern:
@@ -327,9 +409,6 @@ proc isAny(loc: Location; def: Definition): bool =
       of CompoundPatternKind.rec:
         result = not isLiteral(loc, def.pattern.compoundpattern.rec.label)
       else: discard
-  of DefinitionKind.and:
-    # not actually "any" but it will be a Preserve[E] type
-    result = true
   else: discard
 
 proc typeIdent(atom: AtomKind): PNode =
@@ -343,7 +422,6 @@ proc typeIdent(atom: AtomKind): PNode =
   of AtomKind.Symbol: ident"Symbol"
 
 proc typeIdent(loc: Location; sp: SimplePattern): TypeSpec =
-  let scm = loc.schema
   case sp.orKind
   of SimplepatternKind.atom:
     result = TypeSpec(node: typeIdent(sp.atom.atomKind))
@@ -365,16 +443,15 @@ proc typeIdent(loc: Location; sp: SimplePattern): TypeSpec =
     result.attrs = key.attrs + val.attrs
   of SimplepatternKind.Ref:
     result = TypeSpec(node: ident(sp.ref), attrs: attrs(loc, sp))
-    result.node = parameterize(scm, result)
+    result.node = parameterize(loc, result)
   of SimplepatternKind.embedded:
-    case scm.field0.embeddedType.orKind
+    if loc.schema.hasEmbeddedType:
-    of EmbeddedtypenameKind.false:
+      result = TypeSpec(node: ident"EmbeddedRef")
-      result = typeIdent(loc, sp.embedded.interface)
+    else:
-    of EmbeddedtypenameKind.Ref:
+      result = TypeSpec(node: ident"Value")
-      result = TypeSpec(node: scm.embeddedIdent())
     incl(result.attrs, embedded)
   of SimplepatternKind.any, SimplepatternKind.lit:
-    result = TypeSpec(node: preserveIdent(scm))
+    result = TypeSpec(node: ident"Value")
 
 proc typeIdent(loc: Location; pat: Pattern): TypeSpec =
   case pat.orKind
@@ -400,6 +477,7 @@ proc toStrLit(loc: Location; sp: SimplePattern): PNode =
     var (loc, def) = deref(loc, sp.ref)
     result = toStrLit(loc, def)
   of SimplePatternKind.embedded:
+    doAssert not loc.schema.hasEmbeddedType
     result = PNode(kind: nkStrLit, strVal: "#!" & toStrLit(loc, sp.embedded.interface).strVal)
   else: raiseAssert $sp
 
@@ -418,18 +496,11 @@ proc toFieldIdent(loc: Location, label: string; pat: Pattern): PNode =
 
 proc newEmpty(): PNode = newNode(nkEmpty)
 
-proc embeddingParams(scm: Schema; embeddable: bool): PNode =
+proc embeddingParams(loc: Location; embeddable: bool): PNode =
-  if embeddable:
+  newEmpty()
-    nkGenericParams.newTree(nkIdentDefs.newTree(embeddedIdent(scm), newEmpty(), newEmpty()))
-  else:
-    newEmpty()
 
 proc identDef(scm: Schema; a, b: PNode; embeddable: bool): PNode =
-  if embeddable and scm.hasEmbeddedType and b.kind notin {nkBracketExpr, nkTupleTy} and
+  nkIdentDefs.newTree(a, b, newEmpty())
-      (b.kind != nkIdent or b.ident.s != scm.embeddedIdentString):
-    nkIdentDefs.newTree(a, nkBracketExpr.newTree(b, embeddedIdent(scm)), newEmpty())
-  else:
-    nkIdentDefs.newTree(a, b, newEmpty())
 
 proc identDef(scm: Schema; l: PNode; ts: TypeSpec): PNode =
   identDef(scm, l, ts.node, ts.isEmbedded)
@@ -437,13 +508,20 @@ proc identDef(scm: Schema; l: PNode; ts: TypeSpec): PNode =
 proc label(pat: Pattern): string =
   raiseAssert "need to derive record label for " & $pat
 
-proc label(na: NamedPattern; index: int): string =
+proc label(na: NamedPattern; parentLabel: string; index: int): string =
   case na.orKind
   of NamedPatternKind.named:
     string na.named.name
   of NamedPatternKind.anonymous:
     "field" & $index
 
+proc label(nsp: NamedSimplePattern; parentLabel: string; index: int): string =
+  case nsp.orKind
+  of NamedSimplePatternKind.named:
+    string nsp.named.name
+  of NamedSimplePatternKind.anonymous:
+    parentLabel & $index
+
 proc idStr(sp: SimplePattern): string =
   if sp.orKind == SimplepatternKind.lit:
     case sp.lit.value.kind
@@ -465,15 +543,17 @@ proc idStr(np: NamedPattern): string =
   of NamedPatternKind.anonymous:
     np.anonymous.idStr
 
+proc typeDef(loc: Location; name: string; pat: SimplePattern; ty: PNode): PNode =
+    let id = name.ident.toExport
+    nkTypeDef.newTree(id, newEmpty(), ty)
+
 proc typeDef(loc: Location; name: string; pat: Pattern; ty: PNode): PNode =
   let
-    scm = loc.schema
+    embedParams = embeddingParams(loc, isEmbedded(loc, pat))
-    embedParams = embeddingParams(scm, isEmbedded(loc, pat))
     id = name.ident.toExport
   case pat.orKind
   of PatternKind.CompoundPattern:
     let pragma = newNode(nkPragma)
-    if isRecursive(loc, pat): pragma.add(ident"acyclic")
     case pat.compoundPattern.orKind
     of CompoundPatternKind.rec:
       if isLiteral(loc, pat.compoundPattern.rec.label):
@@ -498,8 +578,12 @@ proc typeDef(loc: Location; name: string; pat: Pattern; ty: PNode): PNode =
 proc typeDef(loc: Location; name: string; def: Definition; ty: PNode): PNode =
   case def.orKind
   of DefinitionKind.or:
+    var ty = ty
     let pragma = newNode(nkPragma)
-    if isRecursive(loc, def): pragma.add(ident"acyclic")
+    if isRecursive(loc, name, def):
+      doAssert ty.kind == nkObjectTy
+      pragma.add(ident"acyclic")
+      ty = nkRefTy.newTree(ty)
     pragma.add(ident"preservesOr")
     if isSymbolEnum(loc, def):
       pragma.add ident"pure"
@@ -507,21 +591,26 @@ proc typeDef(loc: Location; name: string; def: Definition; ty: PNode): PNode =
       nkPragmaExpr.newTree(
         name.ident.accQuote.toExport,
         pragma),
-      embeddingParams(loc.schema, isEmbedded(loc, def)),
+      embeddingParams(loc, isEmbedded(loc, def)),
       ty)
   of DefinitionKind.and:
+    var pragma = nkPragma.newNode
+    if isDictionary(loc, def):
+      pragma.add(ident"preservesDictionary")
     nkTypeDef.newTree(
-      name.ident.toExport,
+      nkPragmaExpr.newTree(
-      embeddingParams(loc.schema, isEmbedded(loc, def)),
+        name.ident.accQuote.toExport,
-      preserveIdent(loc.schema))
+        pragma),
+      embeddingParams(loc, isEmbedded(loc, def)),
+      ty)
   of DefinitionKind.Pattern:
     typeDef(loc, name, def.pattern, ty)
 
-proc nimTypeOf(loc: Location; known: var TypeTable; nsp: NamedSimplePattern; name = ""): TypeSpec
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; nsp: NamedSimplePattern): TypeSpec
-proc nimTypeOf(loc: Location; known: var TypeTable; pat: Pattern; name = ""): TypeSpec
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; pat: Pattern): TypeSpec
 
-proc nimTypeOf(loc: Location; known: var TypeTable; cp: CompoundPattern; name = ""): TypeSpec
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; cp: CompoundPattern): TypeSpec
-proc nimTypeOf(loc: Location; known: var TypeTable; sp: SimplePattern; name = ""): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; sp: SimplePattern): TypeSpec =
   typeIdent(loc, sp)
 
 proc addField(recList: PNode; loc: Location; known: var TypeTable; sp: SimplePattern; label: string): PNode {.discardable.} =
@@ -536,62 +625,102 @@ proc addField(recList: PNode; loc: Location; known: var TypeTable; sp: SimplePat
           ident"preservesLiteral",
           toStrLit(loc, sp))))
     recList.add identDef(scm, id, TypeSpec(node: ident"tuple[]"))
-  elif sp.orKind == SimplePatternKind.embedded and not scm.hasEmbeddedType:
+  elif sp.orKind == SimplePatternKind.embedded:
     let id = nkPragmaExpr.newTree(
       id, nkPragma.newTree(ident"preservesEmbedded"))
-    recList.add identDef(scm, id, nimTypeOf(loc, known, sp))
+    recList.add identDef(scm, id, nimTypeOf(loc, known, "", sp))
   else:
-    recList.add identDef(scm, id, nimTypeOf(loc, known, sp))
+    recList.add identDef(scm, id, nimTypeOf(loc, known, "", sp))
 
-proc addFields(recList: PNode; loc: Location; known: var TypeTable; cp: CompoundPattern; parentName: string): PNode {.discardable.} =
+proc addField(recList: PNode; loc: Location; known: var TypeTable; parentName: string; np: NamedPattern; index = 0) =
+  let
+    label = label(np, parentName, index)
+    id = label.toFieldIdent
+    pattern = np.pattern
+  if pattern.isSimple:
+    addField(recList, loc, known, pattern.simplePattern, label)
+  else:
+    var
+      typeName = parentName & capitalizeAscii(label)
+      typePath = loc.schemaPath & @[Symbol typeName]
+      fieldSpec = nimTypeOf(loc, known, label, pattern)
+    known[typePath] = typeDef(loc, typeName, pattern, fieldSpec.node)
+    recList.add identDef(loc.schema, id, ident(typeName), isEmbedded(loc, pattern))
+
+proc addField(recList: PNode; loc: Location; known: var TypeTable; parentName: string; nsp: NamedSimplePattern; index: int; optional: bool) =
+  let
+    label = label(nsp, parentName, index)
+    id = label.toFieldIdent
+    pattern = nsp.pattern
+  if pattern.isRef:
+    var node = typeIdent(loc, pattern).node
+    if optional:
+      node = nkBracketExpr.newTree(ident"Option", node)
+    recList.add identDef(loc.schema, id, node, false)
+  else:
+    var
+      typeName = parentName & capitalizeAscii(label)
+      typePath = loc.schemaPath & @[Symbol typeName]
+      fieldSpec = nimTypeOf(loc, known, label, pattern)
+    if optional:
+      fieldSpec.node = nkBracketExpr.newTree(ident"Option", fieldSpec.node)
+    known[typePath] = typeDef(loc, typeName, pattern, fieldSpec.node)
+    recList.add identDef(loc.schema, id, fieldSpec.node, false)
+
+proc addFields(recList: PNode; loc: Location; known: var TypeTable; parentName: string; cp: CompoundPattern): PNode {.discardable.} =
   let scm = loc.schema
   template addField(np: NamedPattern; index: int) =
     let
-      label = label(np, index)
+      label = label(np, parentName, index)
       id = label.toFieldIdent
       pattern = np.pattern
-    if pattern.isRef or pattern.isSimple:
+    if pattern.isSimple:
       addField(recList, loc, known, pattern.simplePattern, label)
     else:
       var
         typeName = parentName & capitalizeAscii(label)
         typePath = loc.schemaPath & @[Symbol typeName]
-        fieldSpec = nimTypeOf(loc, known, pattern, label)
+        fieldSpec = nimTypeOf(loc, known, label, pattern)
       known[typePath] = typeDef(loc, typeName, pattern, fieldSpec.node)
       recList.add identDef(scm, id, ident(typeName), isEmbedded(loc, pattern))
   case cp.orKind
   of CompoundPatternKind.rec:
-    # recList.add identDef(scm, ident(label), nimTypeOf(loc, known, cp, ""))
+    # recList.add identDef(scm, ident(label), nimTypeOf(loc, known, "", cp))
     raiseassert "unexpected record of fields " #& $cp.rec
   of CompoundPatternKind.tuple:
     for i, np in cp.tuple.patterns: addField(np, i)
   of CompoundPatternKind.tuplePrefix:
     for i, np in cp.tuplePrefix.fixed: addField(np, i)
-    let variableType = nimTypeOf(loc, known, cp.tuplePrefix.variable)
+    let variableType = nimTypeOf(loc, known, "", cp.tuplePrefix.variable)
     recList.add identDef(
       scm,
       nkPragmaExpr.newTree(
         ident(cp.tuplePrefix.variable, parentName).accQuote.toExport,
         nkPragma.newTree(ident"preservesTupleTail")),
-      parameterize(scm, variableType),
+      parameterize(loc, variableType),
       variableType.isEmbedded)
-  else: raiseAssert "not adding fields for " #& $cp
+  of CompoundPatternKind.dict:
+    for nameVal, nsp in cp.dict.entries:
+      recList.addField(loc, known, $nameVal, nsp, 0, false)
   reclist
 
-proc addFields(recList: PNode; loc: Location; known: var TypeTable; pat: Pattern; parentName: string): PNode {.discardable.} =
+proc addFields(recList: PNode; loc: Location; known: var TypeTable; name: string; pat: SimplePattern): PNode {.discardable.} =
+  addField(recList, loc, known, pat, name)
+
+proc addFields(recList: PNode; loc: Location; known: var TypeTable; parentName: string; pat: Pattern): PNode {.discardable.} =
   case pat.orKind
   of PatternKind.SimplePattern:
-    addField(recList, loc, known, pat.simplePattern, "field0")
+    discard addFields(recList, loc, known, parentName, pat.simplePattern)
   of PatternKind.CompoundPattern:
-    discard addFields(recList, loc, known, pat.compoundPattern, parentName)
+    discard addFields(recList, loc, known, parentName, pat.compoundPattern)
   reclist
 
-proc addFields(recList: PNode; loc: Location; known: var TypeTable; entries: DictionaryEntries; parentName: string): PNode {.discardable.} =
+proc addFields(recList: PNode; loc: Location; known: var TypeTable; parentName: string; entries: DictionaryEntries): PNode {.discardable.} =
   var sortedEntries =
-    initOrderedTable[Preserve[void], NamedSimplePattern](entries.len)
+    initOrderedTable[Value, NamedSimplePattern](entries.len)
   for key, val in entries.pairs:
     sortedEntries[key] = val
-  sort(sortedEntries) do (x, y: (Preserve[void], NamedSimplePattern)) -> int:
+  sort(sortedEntries) do (x, y: (Value, NamedSimplePattern)) -> int:
     cmp(x[0], y[0])
   for key, val in sortedEntries.pairs:
     doAssert(key.isSymbol)
@@ -599,44 +728,41 @@ proc addFields(recList: PNode; loc: Location; known: var TypeTable; entries: Dic
     addField(recList, loc, known, val.pattern, label)
   recList
 
-proc nimTypeOf(loc: Location; known: var TypeTable; nsp: NamedSimplePattern; name: string): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; nsp: NamedSimplePattern): TypeSpec =
   case nsp.orKind
   of NamedsimplepatternKind.named:
-    nimTypeOf(loc, known, nsp.named.pattern, string nsp.named.name)
+    nimTypeOf(loc, known, string nsp.named.name, nsp.named.pattern)
   of NamedsimplepatternKind.anonymous:
-    nimTypeOf(loc, known, nsp.anonymous, name)
+    nimTypeOf(loc, known, name, nsp.anonymous)
 
-proc nimTypeOf(loc: Location; known: var TypeTable; rec: CompoundPatternRec; name: string): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; rec: CompoundPatternRec): TypeSpec =
   if isLiteral(loc, rec.label):
     result.node = nkObjectTy.newTree(
       newEmpty(), newEmpty(),
-      newNode(nkRecList).addFields(loc, known, rec.fields.pattern, name))
+      newNode(nkRecList).addFields(loc, known, name, rec.fields.pattern))
   else:
-    result.node = preserveIdent(loc.schema)
+    result.node = ident"Value"
 
-proc nimTypeOf(loc: Location; known: var TypeTable; cp: CompoundPattern; name: string): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; cp: CompoundPattern): TypeSpec =
   case cp.orKind
   of CompoundPatternKind.`rec`:
-    result = nimTypeOf(loc, known, cp.rec, name)
+    result = nimTypeOf(loc, known, name, cp.rec)
   of CompoundPatternKind.`tuple`, CompoundPatternKind.`tupleprefix`:
     result.node = nkObjectTy.newTree(
       newEmpty(), newEmpty(),
-      newNode(nkRecList).addFields(loc, known, cp, name))
+      newNode(nkRecList).addFields(loc, known, name, cp))
   of CompoundPatternKind.`dict`:
     result.node = nkObjectTy.newTree(newEmpty(), newEmpty(),
-      newNode(nkRecList).addFields(loc, known, cp.dict.entries, name))
+      newNode(nkRecList).addFields(loc, known, name, cp.dict.entries))
-  if result.node.kind == nkObjectTy and isRecursive(loc, cp):
-    result.node = nkRefTy.newTree(result.node)
 
-proc nimTypeOf(loc: Location; known: var TypeTable; pat: Pattern; name: string): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; pat: Pattern): TypeSpec =
   case pat.orKind
   of PatternKind.SimplePattern:
-    nimTypeOf(loc, known, pat.simplePattern, name)
+    nimTypeOf(loc, known, name, pat.simplePattern)
   of PatternKind.CompoundPattern:
-    nimTypeOf(loc, known, pat.compoundPattern, name)
+    nimTypeOf(loc, known, name, pat.compoundPattern)
 
-proc nimTypeOf(loc: Location; known: var TypeTable; orDef: DefinitionOr; name: string): TypeSpec =
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; orDef: DefinitionOr): TypeSpec =
-  let scm = loc.schema
   proc toEnumTy(): PNode =
     let ty = nkEnumTy.newNode.add newEmpty()
     proc add (na: NamedAlternative) =
@@ -670,23 +796,27 @@ proc nimTypeOf(loc: Location; known: var TypeTable; orDef: DefinitionOr; name: s
       var memberType: TypeSpec
       if isLiteral(loc, na.pattern):
         memberType.node = ident"bool"
-      elif na.pattern.isRef:
+      elif na.pattern.isSimple:
-        memberType = typeIdent(loc, na.pattern)
+       memberType = typeIdent(loc, na.pattern)
       else:
         let
           memberTypeName = name & na.variantLabel.capitalizeAscii
           memberPath = loc.schemaPath & @[Symbol memberTypeName]
         memberType.node = ident memberTypeName
-        let ty = nimTypeOf(loc, known, na.pattern, memberTypeName)
+        let ty = nimTypeOf(loc, known, memberTypeName, na.pattern)
         addAttrs(memberType, ty)
         if memberPath notin known and not isLiteral(loc, na.pattern):
           known[memberPath] =
             typeDef(loc, memberTypeName, na.pattern, ty.node)
       addAttrs(result, memberType)
       memberType.node = parameterize(
-        scm, memberType.node, isEmbedded(loc, na.pattern))
+        loc, memberType.node, isEmbedded(loc, na.pattern))
+      var memberId = toFieldIdent(loc, na.variantLabel.normalize, na.pattern)
+      if isEmbedded(loc, na.pattern):
+        memberId = nkPragmaExpr.newTree(
+          memberId, nkPragma.newTree(ident"preservesEmbedded"))
       branchRecList.add nkIdentDefs.newTree(
-        toFieldIdent(loc, na.variantLabel.normalize, na.pattern),
+        memberId,
         memberType.node, newEmpty())
       recCase.add nkOfBranch.newTree(
         nkDotExpr.newTree(
@@ -699,18 +829,100 @@ proc nimTypeOf(loc: Location; known: var TypeTable; orDef: DefinitionOr; name: s
       newEmpty(),
       newEmpty(),
       nkRecList.newTree(recCase))
-    # result.attrs = attrs(loc, orDef)
-    if result.node.kind == nkObjectTy and (recursive in attrs(loc, orDef)):
-      result.node = nkRefTy.newTree(result.node)
 
-proc nimTypeOf(loc: Location; known: var TypeTable; def: Definition; name: string): TypeSpec =
+proc isAny(sp: SimplePattern): bool =
+  sp.orKind == SimplePatternKind.any
+
+proc initSimpleAny: SimplePattern =
+  SimplePattern(orKind: SimplePatternKind.any)
+
+proc asAny(nsp: NamedSimplePattern): NamedSimplePattern =
+  result = nsp
+  case result.orKind
+  of NamedSimplePatternKind.named:
+    if not result.named.pattern.isAny:
+      result.named.pattern = initSimpleAny()
+  of NamedSimplePatternKind.anonymous:
+    if not result.anonymous.isAny:
+      result.anonymous = initSimpleAny()
+
+type
+  AndEntry = tuple[pattern: NamedSimplePattern, optional: bool]
+  AndEntries = OrderedTable[Value, AndEntry]
+
+proc collect(entries: var AndEntries; loc: Location; def: Definition; optional: bool) {.gcsafe.}
+
+proc collect(entries: var AndEntries; loc: Location; pat: SimplePattern; optional: bool) =
+  case pat.orKind
+  of SimplePatternKind.Ref:
+    let (loc, def) = deref(loc, pat.ref)
+    collect(entries, loc, def, optional)
+  else:
+    raiseAssert "cannot collect dictionary entries from " & $pat
+
+proc collect(entries: var AndEntries; loc: Location; comp: CompoundPattern; optional: bool) =
+  case comp.orKind
+  of CompoundPatternKind.dict:
+    for key, nsp in comp.dict.entries.pairs:
+      if entries.hasKey(key):
+        entries[key] = (asAny nsp, optional)
+      else:
+        entries[key] = (nsp, optional)
+  else:
+    raiseAssert "cannot collect dictionary entries from " & $comp
+
+proc collect(entries: var AndEntries; loc: Location; pat: Pattern; optional: bool) =
+  case pat.orKind
+  of PatternKind.SimplePattern:
+    collect(entries, loc, pat.simplepattern, optional)
+  of PatternKind.CompoundPattern:
+    collect(entries, loc, pat.compoundpattern, optional)
+
+proc collect(entries: var AndEntries; loc: Location; def: Definition; optional: bool) =
   case def.orKind
   of DefinitionKind.or:
-    nimTypeOf(loc, known, def.or, name)
+    collect(entries, loc, def.or.field0.pattern0.pattern, true)
+    collect(entries, loc, def.or.field0.pattern1.pattern, true)
+    for np in def.or.field0.patternN:
+      collect(entries, loc, np.pattern, true)
   of DefinitionKind.and:
-    TypeSpec(node: preserveIdent(loc.schema))
+    collect(entries, loc, def.and.field0.pattern0.pattern, optional)
+    collect(entries, loc, def.and.field0.pattern1.pattern, optional)
+    for np in def.and.field0.patternN:
+      collect(entries, loc, np.pattern, optional)
   of DefinitionKind.Pattern:
-    nimTypeOf(loc, known, def.pattern, name)
+    collect(entries, loc, def.pattern, optional)
+
+proc toDef(a: DefinitionAnd): Definition =
+  Definition(orKind: DefinitionKind.and, `and`: a)
+
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; andDef: DefinitionAnd): TypeSpec =
+  var def = andDef.toDef
+  if isDictionary(loc, def):
+    var
+      recList = nkRecList.newNode
+      entries: AndEntries
+    collect(entries, loc, def, false)
+    sort(entries) do (x, y: (Value, AndEntry)) -> int:
+      preserves.cmp(x[0], y[0])
+    # TODO: sort the entries
+    var i = 0
+    for key, (nsp, opt) in entries.pairs:
+      recList.addField(loc, known, name, nsp, i, opt)
+      inc(i)
+    result.node = nkObjectTy.newTree(
+      newEmpty(), newEmpty(), recList)
+  else:
+    result.node = ident"Value"
+
+proc nimTypeOf(loc: Location; known: var TypeTable; name: string; def: Definition): TypeSpec =
+  case def.orKind
+  of DefinitionKind.or:
+    nimTypeOf(loc, known, name, def.or)
+  of DefinitionKind.and:
+    nimTypeOf(loc, known, name, def.and)
+  of DefinitionKind.Pattern:
+    nimTypeOf(loc, known, name, def.pattern)
 
 proc generateConstProcs(result: var seq[PNode]; scm: Schema, name: string; def: Definition) =
   discard
@@ -760,6 +972,8 @@ proc collectRefImports(imports: var StringSet; loc: Location; def: Definition) =
     for na in def.or.field0.patternN:
       collectRefImports(imports, loc, na.pattern)
   of DefinitionKind.and:
+    if isDictionary(loc, def):
+      incl(imports, "std/options")
     collectRefImports(imports, loc, def.and.field0.pattern0.pattern)
     collectRefImports(imports, loc, def.and.field0.pattern1.pattern)
     for np in def.and.field0.patternN:
@@ -789,20 +1003,17 @@ proc renderNimBundle*(bundle: Bundle): Table[string, string] =
     var
       typeSection = newNode nkTypeSection
       procs: seq[PNode]
-      unembeddableType, embeddableType: PNode
+      unembeddableType: PNode
     for name, def in scm.field0.definitions.pairs:
       if isLiteral(loc, def):
         generateConstProcs(procs, scm, string name, def)
       else:
         var name = string name
         name[0] = name[0].toUpperAscii
-        var defIdent = parameterize(scm, ident(name), isEmbedded(loc, def))
+        var defIdent = parameterize(loc, ident(name), isEmbedded(loc, def))
         if not isSymbolEnum(loc, def) and not isAny(loc, def):
-          if isEmbedded(loc, def):
+          mergeType(unembeddableType, defIdent)
-            mergeType(embeddableType, defIdent)
+        let typeSpec = nimTypeOf(loc, typeDefs, name, def)
-          else:
-            mergeType(unembeddableType, defIdent)
-        let typeSpec = nimTypeOf(loc, typeDefs, def, name)
         typeDefs[scmPath & @[Symbol name]] = typeDef(loc, name, def, typeSpec.node)
         generateProcs(procs, scm, name, def)
 
@@ -810,46 +1021,12 @@ proc renderNimBundle*(bundle: Bundle): Table[string, string] =
       if typepath.hasPrefix(scmPath):
         add(typeSection, typeDef)
 
-
     let imports = nkImportStmt.newNode.add(ident"preserves")
     block:
       var importSet: HashSet[string]
       collectRefImports(importSet, loc, scm)
       for module in importSet:
         add(imports, ident(module))
-    if not embeddableType.isNil:
-      let genericParams =
-        nkGenericParams.newTree(nkIdentDefs.newTree(embeddedIdent(scm), newEmpty(), newEmpty()))
-      procs.add nkProcDef.newTree(
-          "$".toFieldIdent,
-          newEmpty(),
-          genericParams,
-          nkFormalParams.newTree(
-            ident"string",
-            nkIdentDefs.newTree(
-              ident"x",
-              embeddableType,
-              newEmpty())),
-          newEmpty(),
-          newEmpty(),
-          nkStmtList.newTree(
-            nkCall.newTree(ident"$",
-              nkCall.newTree(ident"toPreserve", ident"x", embeddedIdent(scm)))))
-      procs.add nkProcDef.newTree(
-          "encode".ident.toExport,
-          newEmpty(),
-          genericParams,
-          nkFormalParams.newTree(
-            nkBracketExpr.newTree(ident"seq", ident"byte"),
-            nkIdentDefs.newTree(
-              ident"x",
-              embeddableType,
-              newEmpty())),
-          newEmpty(),
-          newEmpty(),
-          nkStmtList.newTree(
-            nkCall.newTree(ident"encode",
-              nkCall.newTree(ident"toPreserve", ident"x", embeddedIdent(scm)))))
     if not unembeddableType.isNil:
       procs.add nkProcDef.newTree(
           "$".toFieldIdent,
@@ -865,7 +1042,7 @@ proc renderNimBundle*(bundle: Bundle): Table[string, string] =
           newEmpty(),
           nkStmtList.newTree(
             nkCall.newTree(ident"$",
-              nkCall.newTree(ident"toPreserve", ident"x"))))
+              nkCall.newTree(ident"toPreserves", ident"x"))))
       procs.add nkProcDef.newTree(
           "encode".ident.toExport,
           newEmpty(),
@@ -880,7 +1057,7 @@ proc renderNimBundle*(bundle: Bundle): Table[string, string] =
           newEmpty(),
           nkStmtList.newTree(
             nkCall.newTree(ident"encode", nkCall.newTree(
-              ident"toPreserve", ident"x"))))
+              ident"toPreserves", ident"x"))))
     var module = newNode(nkStmtList).add(
         imports,
         typeSection
@@ -902,7 +1079,7 @@ when isMainModule:
       writeFile(path, txt)
       stdout.writeLine(path)
 
-  import std/[options, os, parseopt]
+  import std/[os, parseopt]
   var inputs: seq[string]
   for kind, key, val in getopt():
     case kind
@@ -918,7 +1095,6 @@ when isMainModule:
   for inputPath in inputs:
     var bundle: Bundle
     if dirExists inputPath:
-      new bundle
       for filePath in walkDirRec(inputPath, relative = true):
         var (dirPath, fileName, fileExt) = splitFile(filePath)
         if fileExt == ".prs":
@@ -934,15 +1110,14 @@ when isMainModule:
       let raw = readFile inputPath
       if raw[0] == 0xb4.char:
         var pr = decodePreserves raw
-        if not fromPreserve(bundle, pr):
+        if not fromPreserves(bundle, pr):
           var schema: Schema
-          if fromPreserve(schema, pr):
+          if fromPreserves(schema, pr):
             bundle.modules[@[Symbol fileName]] = schema
       else:
-        new bundle
         var scm = parsePreservesSchema(readFile(inputPath), dirPath)
         bundle.modules[@[Symbol fileName]] = scm
-    if bundle.isNil or bundle.modules.len == 0:
+    if bundle.modules.len == 0:
       quit "Failed to recognize " & inputPath
     else:
       writeModules(bundle)

src/preserves/preserves_schema_nim.nim.cfg

@@ -1 +0,0 @@
-threads:off

src/preserves/preserves_schemac.nim

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[hashes, os, parseopt, streams, strutils, tables]
+
+import ../preserves, ./schema, ./schemaparse
+
+when isMainModule:
+  let outStream = newFileStream(stdout)
+  var
+    inputPath = ""
+    noBundle = false
+
+  for kind, key, arg in getopt():
+    case kind
+    of cmdEnd: discard
+    of cmdArgument:
+      if inputPath != "":
+        quit "only a single path may specified"
+      inputPath = key
+    of cmdLongOption:
+      if arg != "":
+        quit("flag does not take an argument: " & key & " " & arg)
+      case key
+      of "no-bundle": noBundle = true
+      else: quit(key & "flag not recognized")
+    else: quit(key & "flag not recognized")
+
+  if inputPath == "":
+    quit "input file(s) not specified"
+
+  if noBundle:
+    if not fileExists inputPath:
+      quit(inputPath & " does not exist or is not a file")
+    var schema = parsePreservesSchema(readFile(inputPath))
+    write(outStream, schema.toPreserves)
+
+  else:
+    var bundle: Bundle
+    if not dirExists inputPath:
+      quit "not a directory of schemas: " & inputPath
+    else:
+      for filePath in walkDirRec(inputPath, relative = true):
+        var (dirPath, fileName, fileExt) = splitFile(filePath)
+        if fileExt == ".prs":
+          var
+            scm = parsePreservesSchema(readFile(inputPath / filePath))
+            path: ModulePath
+          for e in split(dirPath, '/'):
+            if e != "": add(path, Symbol e)
+          add(path, Symbol fileName)
+          bundle.modules[path] = scm
+      if bundle.modules.len == 0:
+        quit "no schemas parsed"
+      else:
+        write(outStream, bundle.toPreserves)
+
+  close(outStream)

src/preserves/private/decoding.nim

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Unlicense
 
 import std/[endians, streams, strutils]
+import bigints
 import ./values
 
 proc readVarint(s: Stream): uint =
@@ -14,104 +15,119 @@ proc readVarint(s: Stream): uint =
     c = uint s.readUint8
   result = result or (c shl shift)
 
-proc decodePreserves*(s: Stream; E = void): Preserve[E] =
+proc decodePreserves*(s: Stream): Value =
   ## Decode a Preserves value from a binary-encoded stream.
   if s.atEnd: raise newException(IOError, "End of Preserves stream")
   const endMarker = 0x84
   let tag = s.readUint8()
   case tag
-  of 0x80: result = Preserve[E](kind: pkBoolean, bool: false)
+  of 0x80: result = Value(kind: pkBoolean, bool: false)
-  of 0x81: result = Preserve[E](kind: pkBoolean, bool: true)
+  of 0x81: result = Value(kind: pkBoolean, bool: true)
   of 0x85:
-    discard decodePreserves(s, E)
+    discard decodePreserves(s)
-    result = decodePreserves(s, E)
+    result = decodePreserves(s)
   of 0x86:
-    result = decodePreserves(s, E)
+    result = decodePreserves(s)
     result.embedded = true
   of 0x87:
-    let n = s.readUint8()
+    var N: int
+    let n = int s.readUint8()
     case n
     of 4:
-      when system.cpuEndian == bigEndian:
+      result = Value(kind: pkFloat)
-        result = Preserve[E](kind: pkFloat, float: s.readFloat32())
+      var buf: uint32
-      else:
+      N = s.readData(addr buf, sizeof(buf))
-        result = Preserve[E](kind: pkFloat)
+      bigEndian32(addr result.float, addr buf)
-        var be = s.readFloat32()
-        swapEndian32(result.float.addr, be.addr)
     of 8:
-      when system.cpuEndian == bigEndian:
+      result = Value(kind: pkDouble)
-        result = Preserve[E](kind: pkDouble, double: s.readFloat64())
+      var buf: uint64
-      else:
+      N = s.readData(addr buf, sizeof(buf))
-        result = Preserve[E](kind: pkDouble)
+      bigEndian64(addr result.double, addr buf)
-        var be = s.readFloat64()
-        swapEndian64(result.double.addr, be.addr)
     else:
       raise newException(IOError, "unhandled IEEE754 value of " & $n & " bytes")
-  of 0xb1:
+    if N != n: raise newException(IOError, "short read")
-    var data = newString(s.readVarint())
+  of 0xb0:
-    if data.len > 0:
+    var n = int s.readVarint()
-      let n = s.readData(unsafeAddr data[0], data.len)
+    if n <= sizeof(int):
-      if n != data.len:
+      result = Value(kind: pkRegister)
+      if n > 0:
+        var
+          buf: array[sizeof(int), byte]
+          off = buf.len - n
+        if s.readData(addr buf[off], n) != n:
+          raise newException(IOError, "short read")
+        if off > 0:
+          var fill: uint8 = if (buf[off] and 0x80) == 0x80'u8: 0xff else: 0x00'u8
+          for i in 0..<off: buf[i] = fill
+        when buf.len == 4:
+          bigEndian32(addr result.register, addr buf[0])
+        elif buf.len == 8:
+          bigEndian64(addr result.register, addr buf[0])
+        else: {.error: "int size " & $buf.len & " not supported here".}
+    else:
+      result = Value(kind: pkBigInt)
+      var buf = newSeq[byte](n)
+      if s.readData(addr buf[0], buf.len) != n:
+        raise newException(IOError, "short read")
+      if (buf[0] and 0x80) == 0x80:
+        for i, b in buf: buf[i] = not b
+        result.bigint.fromBytes(buf, bigEndian)
+        result.bigint = -(result.bigint.succ)
+      else:
+        result.bigint.fromBytes(buf, bigEndian)
+  of 0xb1:
+    result = Value(kind: pkString, string: newString(s.readVarint()))
+    if result.string.len > 0:
+      if s.readData(addr result.string[0], result.string.len) != result.string.len:
         raise newException(IOError, "short read")
-    result = Preserve[E](kind: pkString, string: data)
   of 0xb2:
     var data = newSeq[byte](s.readVarint())
     if data.len > 0:
       let n = s.readData(addr data[0], data.len)
       if n != data.len:
         raise newException(IOError, "short read")
-    result = Preserve[E](kind: pkByteString, bytes: data)
+    result = Value(kind: pkByteString, bytes: data)
   of 0xb3:
     var data = newString(s.readVarint())
     if data.len > 0:
       let n = s.readData(addr data[0], data.len)
       if n != data.len:
         raise newException(IOError, "short read")
-    result = Preserve[E](kind: pkSymbol, symbol: Symbol data)
+    result = Value(kind: pkSymbol, symbol: Symbol data)
   of 0xb4:
-    result = Preserve[E](kind: pkRecord)
+    result = Value(kind: pkRecord)
-    var label = decodePreserves(s, E)
+    var label = decodePreserves(s)
     while s.peekUint8() != endMarker:
-      result.record.add decodePreserves(s, E)
+      result.record.add decodePreserves(s)
     result.record.add(move label)
     discard s.readUint8()
   of 0xb5:
-    result = Preserve[E](kind: pkSequence)
+    result = Value(kind: pkSequence)
     while s.peekUint8() != endMarker:
-      result.sequence.add decodePreserves(s, E)
+      result.sequence.add decodePreserves(s)
     discard s.readUint8()
   of 0xb6:
-    result = Preserve[E](kind: pkSet)
+    result = Value(kind: pkSet)
     while s.peekUint8() != endMarker:
-      incl(result, decodePreserves(s, E))
+      incl(result, decodePreserves(s))
     discard s.readUint8()
   of 0xb7:
-    result = Preserve[E](kind: pkDictionary)
+    result = Value(kind: pkDictionary)
     while s.peekUint8() != endMarker:
-      result[decodePreserves(s, E)] = decodePreserves(s, E)
+      result[decodePreserves(s)] = decodePreserves(s)
     discard s.readUint8()
-  of 0xb0:
-    var len = s.readVarint()
-    result = Preserve[E](kind: pkSignedInteger)
-    if len > 0:
-      if (s.peekUint8() and 0x80) == 0x80:
-        result.int = BiggestInt -1
-      while len > 0:
-        result.int = (result.int shl 8) + s.readUint8().BiggestInt
-        dec(len)
-
   of endMarker:
     raise newException(ValueError, "invalid Preserves stream")
   else:
     raise newException(ValueError, "invalid Preserves tag byte 0x" & tag.toHex(2))
 
-proc decodePreserves*(s: string; E = void): Preserve[E] =
+proc decodePreserves*(s: string): Value =
   ## Decode a string of binary-encoded Preserves.
-  decodePreserves(s.newStringStream, E)
+  decodePreserves(s.newStringStream)
 
-proc decodePreserves*(s: seq[byte]; E = void): Preserve[E] =
+proc decodePreserves*(s: seq[byte]): Value =
   ## Decode a byte-string of binary-encoded Preserves.
-  decodePreserves(cast[string](s), E)
+  decodePreserves(cast[string](s))
 
 type BufferedDecoder* = object
   ## Type for buffering binary Preserves before decoding.
@@ -147,14 +163,14 @@ proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
   if data.len > 0:
     dec.feed(unsafeAddr data[0], data.len)
 
-proc decode*(dec: var BufferedDecoder; E = void): (bool, Preserve[E]) =
+proc decode*(dec: var BufferedDecoder): (bool, Value) =
   ## Decode from `dec`. If decoding fails the internal position of the
   ## decoder does not advance.
   if dec.appendPosition > 0:
     assert(dec.decodePosition < dec.appendPosition)
     dec.stream.setPosition(dec.decodePosition)
     try:
-      result[1] = decodePreserves(dec.stream, E)
+      result[1] = decodePreserves(dec.stream)
       result[0] = true
       dec.decodePosition = dec.stream.getPosition()
       if dec.decodePosition == dec.appendPosition:

src/preserves/private/encoding.nim

@@ -1,7 +1,8 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[endians, options, sets, sequtils, streams, tables, typetraits]
+import std/[endians, streams]
+import bigints
 import ./values
 
 proc writeVarint(s: Stream; n: Natural) =
@@ -11,7 +12,7 @@ proc writeVarint(s: Stream; n: Natural) =
     n = n shr 7
   s.write(uint8 n and 0x7f)
 
-proc write*[E](str: Stream; pr: Preserve[E]) =
+proc write*(str: Stream; pr: Value) =
   ## Write the binary-encoding of a Preserves value to a stream.
   if pr.embedded: str.write(0x86'u8)
   case pr.kind:
@@ -35,25 +36,44 @@ proc write*[E](str: Stream; pr: Preserve[E]) =
       var be: float64
       swapEndian64(be.addr, pr.double.unsafeAddr)
       str.write(be)
-  of pkSignedInteger:
+  of pkRegister:
-    if pr.int == 0:
+    if pr.register == 0: str.write("\xb0\x00")
-      str.write("\xb0\x00")
     else:
-      var bitCount = 1'u8
+      const bufLen = sizeof(int)
-      if pr.int < 0:
+      var buf: array[bufLen, byte]
-        while ((not pr.int) shr bitCount) != 0:
+      when bufLen == 4: bigEndian32(addr buf[0], addr pr.register)
-          inc(bitCount)
+      elif bufLen == 8: bigEndian64(addr buf[0], addr pr.register)
+      else: {.error: "int size " & $bufLen & " not supported here".}
+      if buf[0] != 0x00 and buf[0] != 0xff:
+        str.write(cast[string](buf)) # dumbass hex conversion
       else:
-        while (pr.int shr bitCount) != 0:
+        var start = 0
-          inc(bitCount)
+        while start < buf.high and buf[0] == buf[succ start]: inc start
-      var byteCount = (bitCount + 8) div 8
+        if start < buf.high and (buf[succ start] and 0x80) == (buf[0] and 0x80): inc start
-      str.write(0xb0'u8)
+        str.write('\xb0')
-      str.writeVarint(byteCount)
+        str.write(uint8(bufLen - start))
-      proc write(n: uint8; i: BiggestInt) =
+        str.write(cast[string](buf[start..<bufLen]))
-        if n > 1:
+  of pkBigInt:
-          write(n.pred, i shr 8)
+    if pr.bigint.isZero: str.write("\xb0\x00")
-        str.write(i.uint8)
+    elif pr.bigint.isNegative:
-      write(byteCount, pr.int)
+      var buf = pr.bigint.succ.toBytes(bigEndian)
+      for i, b in buf: buf[i] = not b
+      str.write('\xb0')
+      if (buf[0] and 0x80) != 0x80:
+        str.writeVarint(buf.len.succ)
+        str.write('\xff')
+      else:
+        str.writeVarint(buf.len)
+      str.write(cast[string](buf))
+    else:
+      var buf = pr.bigint.toBytes(bigEndian)
+      str.write('\xb0')
+      if (buf[0] and 0x80) != 0:
+        str.writeVarint(buf.len.succ)
+        str.write('\x00')
+      else:
+        str.writeVarint(buf.len)
+      str.write(cast[string](buf))
   of pkString:
     str.write(0xb1'u8)
     str.writeVarint(pr.string.len)
@@ -90,10 +110,10 @@ proc write*[E](str: Stream; pr: Preserve[E]) =
       str.write(value)
     str.write(0x84'u8)
   of pkEmbedded:
-    str.write(0x86'u8)
+    # str.write(0x86'u8)
-    str.write(pr.embed.toPreserve)
+    raise newException(ValueError, "cannot encode an embedded object")
 
-proc encode*[E](pr: Preserve[E]): seq[byte] =
+proc encode*(pr: Value): seq[byte] =
   ## Return the binary-encoding of a Preserves value.
   let s = newStringStream()
   s.write pr

src/preserves/private/parsing.nim

@@ -1,15 +1,15 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, parseutils, strutils, unicode]
+import std/[base64, options, parseutils, strutils, unicode]
 from std/sequtils import insert
 
-import npeg
+import bigints, npeg
+
 import ../pegs
 import ./decoding, ./values
 
 type
-  Value = Preserve[void]
   Frame = tuple[value: Value, pos: int]
   Stack = seq[Frame]
 
@@ -37,11 +37,30 @@ template unescape*(buf: var string; capture: string) =
       of 't': add(buf, char 0x09)
       of '"': add(buf, char 0x22)
       of 'u':
-        var r: int32
+        var short: uint16
         inc(i)
-        discard parseHex(capture, r, i, 4)
+        discard parseHex(capture, short, i, 4)
         inc(i, 3)
-        add(buf, Rune r)
+        if (short shr 15) == 0:
+          add(buf, Rune(short).toUtf8)
+        elif (short shr 10) == 0b110110:
+          if i+6 >= capture.len:
+            raise newException(ValueError, "Invalid UTF-16 surrogate pair")
+          var rune = uint32(short shl 10) + 0x10000
+          validate(capture[i+1] == '\\')
+          validate(capture[i+2] == 'u')
+          inc(i, 3)
+          discard parseHex(capture, short, i, 4)
+          if (short shr 10) != 0b110111:
+            raise newException(ValueError, "Invalid UTF-16 surrogate pair")
+          inc(i, 3)
+          rune = rune or (short and 0b1111111111)
+          #add(buf, Rune(rune).toUTF8)
+          let j = buf.len
+          buf.setLen(buf.len+4)
+          rune.Rune.fastToUTF8Copy(buf, j, false)
+        else:
+          raise newException(ValueError, "Invalid UTF-16 escape sequence " & capture)
       else:
         validate(false)
     else:
@@ -79,14 +98,12 @@ proc pushHexNibble[T](result: var T; c: char) =
     of '0'..'9': T(ord(c) - ord('0'))
     of 'a'..'f': T(ord(c) - ord('a') + 10)
     of 'A'..'F': T(ord(c) - ord('A') + 10)
-    else: 0
+    else: return
   result = (result shl 4) or n
 
-proc parsePreserves*(text: string): Preserve[void] =
+proc parsePreserves*(text: string): Value =
-  ## Parse a text-encoded Preserves `string` to a `Preserve` value.
+  ## Parse a text-encoded Preserves `string` to a Preserves `Value`.
-  runnableExamples:
+  let pegParser = peg("Document", stack: Stack):
-    assert parsePreserves"[ 1 2 3 ]" == [ 1, 2, 3 ].toPreserve
-  const pegParser = peg("Document", stack: Stack):
     # Override rules from pegs.nim
 
     Document <- Preserves.Document
@@ -157,11 +174,19 @@ proc parsePreserves*(text: string): Preserve[void] =
       pushStack Value(kind: pkDouble, double: cast[float64](reg))
 
     Preserves.SignedInteger <- Preserves.SignedInteger:
-      pushStack Value(kind: pkSignedInteger, int: parseInt($0))
+      var
+        big = initBigInt($0)
+        small = toInt[int](big)
+      if small.isSome:
+        pushStack Value(kind: pkRegister, register: small.get)
+      else:
+        pushStack Value(kind: pkBigInt, bigint: big)
 
     Preserves.String <- Preserves.String:
       var v = Value(kind: pkString, string: newStringOfCap(len($1)))
       unescape(v.string, $1)
+      if validateUtf8(v.string) != -1:
+        raise newException(ValueError, "Preserves text contains an invalid UTF-8 sequence")
       pushStack v
 
     Preserves.charByteString <- Preserves.charByteString:
@@ -176,7 +201,9 @@ proc parsePreserves*(text: string): Preserve[void] =
       pushStack Value(kind: pkByteString, bytes: cast[seq[byte]](base64.decode(joinWhitespace($1))))
 
     Preserves.Symbol <- Preserves.Symbol:
-      pushStack Value(kind: pkSymbol, symbol: Symbol $1)
+      var buf = newStringOfCap(len($1))
+      unescape(buf, $1)
+      pushStack Value(kind: pkSymbol, symbol: Symbol buf)
 
     Preserves.Embedded <- Preserves.Embedded:
       var v = stack.pop.value
@@ -189,7 +216,7 @@ proc parsePreserves*(text: string): Preserve[void] =
       pushStack val
 
     Preserves.Compact <- Preserves.Compact:
-      pushStack decodePreserves(stack.pop.value.bytes, void)
+      pushStack decodePreserves(stack.pop.value.bytes)
 
   var stack: Stack
   let match = pegParser.match(text, stack)
@@ -198,7 +225,65 @@ proc parsePreserves*(text: string): Preserve[void] =
   assert(stack.len == 1)
   stack.pop.value
 
-proc parsePreserves*(text: string; E: typedesc): Preserve[E] =
+proc parsePreservesAtom*(text: string): Atom =
-  ## Parse a text-encoded Preserves `string` to a `Preserve[E]` value for embedded type `E`.
+  ## Parse a text-encoded Preserves `string` to a Preserves `Atom`.
-  when E is void: parsePreserves(text)
+  let pegParser = peg("Atom", a: Atom):
-  else: mapEmbeds(parsePreserves(text), E)
+    # Override rules from pegs.nim
+
+    Atom <- ?"#!" * Preserves.Atom
+
+    Preserves.Boolean <- Preserves.Boolean:
+      case $0
+      of "#f": a = Atom(kind: pkBoolean)
+      of "#t": a = Atom(kind: pkBoolean, bool: true)
+      else: discard
+
+    Preserves.Float <- Preserves.Float:
+      a = Atom(kind: pkFloat, float: parseFloat($1))
+
+    Preserves.Double <- Preserves.Double:
+      a = Atom(kind: pkDouble)
+      discard parseBiggestFloat($0, a.double)
+
+    Preserves.FloatRaw <- Preserves.FloatRaw:
+      var reg: uint32
+      for c in $1: pushHexNibble(reg, c)
+      a = Atom(kind: pkFloat, float: cast[float32](reg))
+
+    Preserves.DoubleRaw <- Preserves.DoubleRaw:
+      var reg: uint64
+      for c in $1: pushHexNibble(reg, c)
+      a = Atom(kind: pkDouble, double: cast[float64](reg))
+
+    Preserves.SignedInteger <- Preserves.SignedInteger:
+      var
+        big = initBigInt($0)
+        small = toInt[int](big)
+      if small.isSome:
+        a = Atom(kind: pkRegister, register: small.get)
+      else:
+        a = Atom(kind: pkBigInt, bigint: big)
+
+    Preserves.String <- Preserves.String:
+      a = Atom(kind: pkString, string: newStringOfCap(len($1)))
+      unescape(a.string, $1)
+      if validateUtf8(a.string) != -1:
+        raise newException(ValueError, "Preserves text contains an invalid UTF-8 sequence")
+
+    Preserves.charByteString <- Preserves.charByteString:
+      a = Atom(kind: pkByteString, bytes: newSeqOfCap[byte](len($1)))
+      unescape(a.bytes, $1)
+
+    Preserves.hexByteString <- Preserves.hexByteString:
+      a = Atom(kind: pkByteString, bytes: cast[seq[byte]](parseHexStr(joinWhitespace($1))))
+
+    Preserves.b64ByteString <- Preserves.b64ByteString:
+      a = Atom(kind: pkByteString, bytes: cast[seq[byte]](base64.decode(joinWhitespace($1))))
+
+    Preserves.Symbol <- Preserves.Symbol:
+      var buf = newStringOfCap(len($1))
+      unescape(buf, $1)
+      a = Atom(kind: pkSymbol, symbol: Symbol buf)
+
+  if not pegParser.match(text, result).ok:
+    raise newException(ValueError, "failed to parse Preserves atom: " & text)

src/preserves/private/preserves_encode.nim

@@ -19,17 +19,17 @@ when isMainModule:
     of "preserves_from_json":
       let
         js = stdin.newFileStream.parseJson
-        pr = js.toPreserve
+        pr = js.toPreserves
       stdout.newFileStream.write(pr)
     of "preserves_from_xml":
       let
         xn = stdin.newFileStream.parseXml
-        pr = xn.toPreserveHook(void)
+        pr = xn.toPreservesHook()
       stdout.newFileStream.write(pr)
     of "preserves_to_json":
       let
         pr = stdin.readAll.decodePreserves
-        js = preserveTo(pr, JsonNode)
+        js = preservesTo(pr, JsonNode)
       if js.isSome:
         stdout.writeLine(get js)
       else:
@@ -37,7 +37,7 @@ when isMainModule:
     of "preserves_to_xml":
       let pr = stdin.readAll.decodePreserves
       var xn: XmlNode
-      if fromPreserve(xn, pr):
+      if fromPreserves(xn, pr):
         stdout.writeLine(xn)
       else:
         quit("Preserves not convertable to XML")

src/preserves/private/texts.nim

@@ -1,41 +1,48 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, json, options, sets, sequtils, streams, strutils, tables, typetraits]
+import std/[base64, endians, math, sequtils, streams, strutils]
+import bigints
 import ./values
 
-proc `$`*(s: Symbol): string =
+const hexAlphabet = "0123456789abcdef"
-  let sym = string s
-  if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
-    result = sym
-  else:
-    result = newStringOfCap(sym.len shl 1)
-    result.add('|')
-    for c in sym:
-      case c
-      of '\\':
-        result.add("\\\\")
-      of '/':
-        result.add("\\/")
-      of '\x08':
-        result.add("\\b")
-      of '\x0c':
-        result.add("\\f")
-      of '\x0a':
-        result.add("\\n")
-      of '\x0d':
-        result.add("\\r")
-      of '\x09':
-        result.add("\\t")
-      of '|':
-        result.add("\\|")
-      else:
-        result.add(c)
-    result.add('|')
 
 type TextMode* = enum textPreserves, textJson
 
-proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
+template writeEscaped(stream: Stream; text: string; delim: char) =
+  const escaped = { '"', '\\', '\b', '\f', '\n', '\r', '\t' }
+  var
+    i: int
+    c: char
+  while i < text.len:
+    c = text[i]
+    case c
+    of delim:
+      write(stream, '\\')
+      write(stream, delim)
+    of '\\': write(stream, "\\\\")
+    of '\b': write(stream, "\\b")
+    of '\f': write(stream, "\\f")
+    of '\n': write(stream, "\\n")
+    of '\r': write(stream, "\\r")
+    of '\t': write(stream, "\\t")
+    of { '\x00'..'\x1f', '\x7f' } - escaped:
+      # do not use \x__ notation because
+      # it is a subset of \u____.
+      write(stream, "\\u00")
+      write(stream, c.uint8.toHex(2))
+    else: write(stream, c)
+    inc i
+
+proc writeSymbol(stream: Stream; sym: string) =
+  if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
+    write(stream, sym)
+  else:
+    write(stream, '|')
+    writeEscaped(stream, sym, '|')
+    write(stream, '|')
+
+proc writeText*(stream: Stream; pr: Value; mode = textPreserves) =
   ## Encode Preserves to a `Stream` as text.
   if pr.embedded: write(stream, "#!")
   case pr.kind:
@@ -44,14 +51,38 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
     of false: write(stream, "#f")
     of true: write(stream, "#t")
   of pkFloat:
-    write(stream, $pr.float)
+    case pr.float.classify:
-    write(stream, 'f')
+    of fcNormal, fcZero, fcNegZero:
+      write(stream, $pr.float)
+      write(stream, 'f')
+    else:
+      var buf: array[4, byte]
+      bigEndian32(addr buf[0], addr pr.float)
+      write(stream, "#xf\"")
+      for b in buf:
+        write(stream, hexAlphabet[b shr 4])
+        write(stream, hexAlphabet[b and 0xf])
+      write(stream, '"')
   of pkDouble:
-    write(stream, $pr.double)
+    case pr.double.classify:
-  of pkSignedInteger:
+    of fcNormal, fcZero, fcNegZero:
-    write(stream, $pr.int)
+      write(stream, $pr.double)
+    else:
+      var buf: array[8, byte]
+      bigEndian64(addr buf[0], addr pr.double)
+      write(stream, "#xd\"")
+      for b in buf:
+        write(stream, hexAlphabet[b shr 4])
+        write(stream, hexAlphabet[b and 0xf])
+      write(stream, '"')
+  of pkRegister:
+    write(stream, $pr.register)
+  of pkBigInt:
+    write(stream, $pr.bigint)
   of pkString:
-    write(stream, escapeJson(pr.string))
+    write(stream, '"')
+    writeEscaped(stream, pr.string, '"')
+    write(stream, '"')
   of pkByteString:
     if pr.bytes.allIt(char(it) in {' '..'!', '#'..'~'}):
       write(stream, "#\"")
@@ -63,39 +94,13 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
         write(stream, base64.encode(pr.bytes))
         write(stream, ']')
       else:
-        const alphabet = "0123456789abcdef"
         write(stream, "#x\"")
         for b in pr.bytes:
-          write(stream, alphabet[int(b shr 4)])
+          write(stream, hexAlphabet[b.int shr 4])
-          write(stream, alphabet[int(b and 0xf)])
+          write(stream, hexAlphabet[b.int and 0xf])
         write(stream, '"')
   of pkSymbol:
-    let sym = pr.symbol.string
+    writeSymbol(stream, pr.symbol.string)
-    if sym.len > 0 and sym[0] in {'A'..'z'} and not sym.anyIt(char(it) in { '\x00'..'\x19', '"', '\\', '|' }):
-      write(stream, sym)
-    else:
-      write(stream, '|')
-      for c in sym:
-        case c
-        of '\\':
-          write(stream, "\\\\")
-        of '/':
-          write(stream, "\\/")
-        of '\x08':
-          write(stream, "\\b")
-        of '\x0c':
-          write(stream, "\\f")
-        of '\x0a':
-          write(stream, "\\n")
-        of '\x0d':
-          write(stream, "\\r")
-        of '\x09':
-          write(stream, "\\t")
-        of '|':
-          write(stream, "\\|")
-        else:
-          write(stream, c)
-      write(stream, '|')
   of pkRecord:
     assert(pr.record.len > 0)
     write(stream, '<')
@@ -147,13 +152,21 @@ proc writeText*[E](stream: Stream; pr: Preserve[E]; mode = textPreserves) =
         writeText(stream, value, mode)
     write(stream, '}')
   of pkEmbedded:
-    write(stream, "#!")
+    if not pr.embedded: write(stream, "#!")
-    when compiles($pr.embed) and not E is void:
+    if pr.embeddedRef.isNil:
-      write(stream, $pr.embed)
+      write(stream, "<null>")
     else:
-      write(stream, "…")
+      when compiles($pr.embed):
+        write(stream, $pr.embed)
+      else:
+        write(stream, "…")
 
-proc `$`*[E](pr: Preserve[E]): string =
+proc `$`*(sym: Symbol): string =
+  var stream = newStringStream()
+  writeSymbol(stream, sym.string)
+  result = move stream.data
+
+proc `$`*(pr: Value): string =
   ## Generate the textual representation of ``pr``.
   var stream = newStringStream()
   writeText(stream, pr, textPreserves)

src/preserves/private/values.nim

@@ -1,15 +1,17 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[hashes, options, sets, sequtils, tables]
+import std/[algorithm, hashes, math, options, sets, sequtils, tables]
+
+import bigints
 
 type
   PreserveKind* = enum
-    pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol,
+    pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol,
         pkRecord, pkSequence, pkSet, pkDictionary, pkEmbedded
 
 const
-  atomKinds* = {pkBoolean, pkFloat, pkDouble, pkSignedInteger, pkString, pkByteString, pkSymbol}
+  atomKinds* = {pkBoolean, pkFloat, pkDouble, pkRegister, pkBigInt, pkString, pkByteString, pkSymbol}
   compoundKinds* = {pkRecord, pkSequence, pkSet, pkDictionary}
 
 type Symbol* = distinct string
@@ -19,9 +21,9 @@ proc hash*(s: Symbol): Hash {.borrow.}
 proc len*(s: Symbol): int {.borrow.}
 
 type
-  Preserve*[E] = object
+  Atom* = object
-    embedded*: bool
+    ## Atomic Preserves value.
-      ## Flag to mark embedded Preserves
+    ## Useful when a `const Value` is required.
     case kind*: PreserveKind
     of pkBoolean:
       bool*: bool
@@ -29,8 +31,31 @@ type
       float*: float32
     of pkDouble:
       double*: float64
-    of pkSignedInteger:
+    of pkRegister:
-      int*: BiggestInt
+      register*: int
+    of pkBigInt:
+      bigint*: BigInt
+    of pkString:
+      string*: string
+    of pkByteString:
+      bytes*: seq[byte]
+    of pkSymbol:
+      symbol*: Symbol
+    else:
+      discard
+
+  Value* = object
+    case kind*: PreserveKind
+    of pkBoolean:
+      bool*: bool
+    of pkFloat:
+      float*: float32
+    of pkDouble:
+      double*: float64
+    of pkRegister:
+      register*: int
+    of pkBigInt:
+      bigint*: BigInt
     of pkString:
       string*: string
     of pkByteString:
@@ -38,32 +63,46 @@ type
     of pkSymbol:
       symbol*: Symbol
     of pkRecord:
-      record*: seq[Preserve[E]] # label is last
+      record*: seq[Value] # label is last
     of pkSequence:
-      sequence*: seq[Preserve[E]]
+      sequence*: seq[Value]
     of pkSet:
-      set*: seq[Preserve[E]]
+      set*: seq[Value]
         # TODO: HashSet
     of pkDictionary:
-      dict*: seq[DictEntry[E]]
+      dict*: seq[DictEntry]
         # TODO: Tables
     of pkEmbedded:
-      embed*: E
+      embeddedRef*: EmbeddedRef
+    embedded*: bool
+      ## Flag to mark embedded Preserves value
 
-  DictEntry*[E] = tuple[key: Preserve[E], val: Preserve[E]]
+  DictEntry* = tuple[key: Value, val: Value]
 
-func `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
+  EmbeddedRef* = ref RootObj
+  EmbeddedObj* = RootObj
+    ## Object refs embedded in Preserves `Value`s must inherit from `EmbeddedObj`.
+    ## At the moment this is just an alias to `RootObj` but this may change in the future.
+
+func `===`[T: SomeFloat](a, b: T): bool =
+  ## Compare where Nan == NaN.
+  let class = a.classify
+  (class == b.classify) and ((class notin {fcNormal,fcSubnormal}) or (a == b))
+
+func `==`*(x, y: Value): bool =
   ## Check `x` and `y` for equivalence.
   if x.kind == y.kind and x.embedded == y.embedded:
     case x.kind
     of pkBoolean:
       result = x.bool == y.bool
     of pkFloat:
-      result = x.float == y.float
+      result = x.float === y.float
     of pkDouble:
-      result = x.double == y.double
+      result = x.double === y.double
-    of pkSignedInteger:
+    of pkRegister:
-      result = x.int == y.int
+      result = x.register == y.register
+    of pkBigInt:
+      result = x.bigint == y.bigint
     of pkString:
       result = x.string == y.string
     of pkByteString:
@@ -92,11 +131,7 @@ func `==`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
           (x.dict[i].key == y.dict[i].key) and
           (x.dict[i].val == y.dict[i].val)
     of pkEmbedded:
-      when A is B:
+      result = x.embeddedRef == y.embeddedRef
-        when A is void:
-          result = true
-        else:
-          result = x.embed == y.embed
 
 proc `<`(x, y: string | seq[byte]): bool =
   for i in 0 .. min(x.high, y.high):
@@ -104,7 +139,7 @@ proc `<`(x, y: string | seq[byte]): bool =
     if x[i] != y[i]: return false
   x.len < y.len
 
-proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
+proc `<`*(x, y: Value): bool =
   ## Preserves have a total order over values. Check if `x` is ordered before `y`.
   if x.embedded != y.embedded:
     result = y.embedded
@@ -118,8 +153,10 @@ proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
       result = x.float < y.float
     of pkDouble:
       result = x.double < y.double
-    of pkSignedInteger:
+    of pkRegister:
-      result = x.int < y.int
+      result = x.register < y.register
+    of pkBigInt:
+      result = x.bigint < y.bigint
     of pkString:
       result = x.string < y.string
     of pkByteString:
@@ -150,19 +187,18 @@ proc `<`*[A, B](x: Preserve[A]; y: Preserve[B]): bool =
           if x.dict[i].val != y.dict[i].val: return false
       result = x.dict.len < y.dict.len
     of pkEmbedded:
-      when (not A is void) and (A is B):
+      result = x.embeddedRef < y.embeddedRef
-        result = x.embed < y.embed
 
-func cmp*[E](x, y: Preserve[E]): int =
+func cmp*(x, y: Value): int =
   ## Compare by Preserves total ordering.
   if x == y: 0
   elif x < y: -1
   else: 1
 
-proc sort*[E](pr: var Preserve[E]) = sort(pr.sequence, cmp)
+proc sort*(pr: var Value) = sort(pr.sequence, cmp)
   ## Sort a Preserves array by total ordering.
 
-proc hash*[E](pr: Preserve[E]): Hash =
+proc hash*(pr: Value): Hash =
   ## Produce a `Hash` of `pr` for use with a `HashSet` or `Table`.
   var h = hash(pr.kind.int) !& hash(pr.embedded)
   case pr.kind
@@ -172,8 +208,10 @@ proc hash*[E](pr: Preserve[E]): Hash =
     h = h !& hash(pr.float)
   of pkDouble:
     h = h !& hash(pr.double)
-  of pkSignedInteger:
+  of pkRegister:
-    h = h !& hash(pr.int)
+    h = h !& hash(pr.register)
+  of pkBigInt:
+    h = h !& hash(pr.bigint)
   of pkString:
     h = h !& hash(pr.string)
   of pkByteString:
@@ -193,16 +231,10 @@ proc hash*[E](pr: Preserve[E]): Hash =
     for (key, val) in pr.dict.items:
       h = h !& hash(key) !& hash(val)
   of pkEmbedded:
-    when E is void:
+    h = h !& hash(cast[uint](addr pr.embeddedRef[]))
-      h = h !& hash(pr.embed)
-    else:
-      if pr.embed.isNil:
-        h = h !& hash(false)
-      else:
-        h = h !& hash(pr.embed)
   !$h
 
-proc `[]`*(pr: Preserve; i: int): Preserve =
+proc `[]`*(pr: Value; i: int): Value =
   ## Select an indexed value from ``pr``.
   ## Only valid for records and sequences.
   case pr.kind
@@ -211,16 +243,16 @@ proc `[]`*(pr: Preserve; i: int): Preserve =
   else:
     raise newException(ValueError, "Preserves value is not indexable")
 
-proc `[]=`*(pr: var Preserve; i: Natural; val: Preserve) =
+proc `[]=`*(pr: var Value; i: Natural; val: Value) =
   ## Assign an indexed value into ``pr``.
   ## Only valid for records and sequences.
   case pr.kind
   of pkRecord: pr.record[i] = val
   of pkSequence: pr.sequence[i] = val
   else:
-    raise newException(ValueError, "`Preserves value is not indexable")
+    raise newException(ValueError, "Preserves value is not indexable")
 
-proc `[]=`*(pr: var Preserve; key, val: Preserve) =
+proc `[]=`*(pr: var Value; key, val: Value) =
   ## Insert `val` by `key` in the Preserves dictionary `pr`.
   for i in 0..pr.dict.high:
     if key < pr.dict[i].key:
@@ -231,7 +263,7 @@ proc `[]=`*(pr: var Preserve; key, val: Preserve) =
       return
   pr.dict.add((key, val, ))
 
-proc incl*(pr: var Preserve; key: Preserve) =
+proc incl*(pr: var Value; key: Value) =
   ## Include `key` in the Preserves set `pr`.
   for i in 0..pr.set.high:
     if key < pr.set[i]:
@@ -239,7 +271,7 @@ proc incl*(pr: var Preserve; key: Preserve) =
       return
   pr.set.add(key)
 
-proc excl*(pr: var Preserve; key: Preserve) =
+proc excl*(pr: var Value; key: Value) =
   ## Exclude `key` from the Preserves set `pr`.
   for i in 0..pr.set.high:
     if pr.set[i] == key:

src/preserves/schema.nim

@@ -8,23 +8,23 @@ type
     `name`*: Symbol
 
   ModulePath* = seq[Symbol]
-  Bundle* {.acyclic, preservesRecord: "bundle".} = ref object
+  Bundle* {.preservesRecord: "bundle".} = object
     `modules`*: Modules
 
   CompoundPatternKind* {.pure.} = enum
     `rec`, `tuple`, `tuplePrefix`, `dict`
-  CompoundPatternRec* {.acyclic, preservesRecord: "rec".} = ref object
+  CompoundPatternRec* {.preservesRecord: "rec".} = object
     `label`*: NamedPattern
     `fields`*: NamedPattern
 
-  CompoundPatternTuple* {.acyclic, preservesRecord: "tuple".} = ref object
+  CompoundPatternTuple* {.preservesRecord: "tuple".} = object
     `patterns`*: seq[NamedPattern]
 
-  CompoundPatternTuplePrefix* {.acyclic, preservesRecord: "tuplePrefix".} = ref object
+  CompoundPatternTuplePrefix* {.preservesRecord: "tuplePrefix".} = object
     `fixed`*: seq[NamedPattern]
     `variable`*: NamedSimplePattern
 
-  CompoundPatternDict* {.acyclic, preservesRecord: "dict".} = ref object
+  CompoundPatternDict* {.preservesRecord: "dict".} = object
     `entries`*: DictionaryEntries
 
   `CompoundPattern`* {.acyclic, preservesOr.} = ref object
@@ -54,11 +54,11 @@ type
         `ref`*: Ref
 
   
+  Definitions* = Table[Symbol, Definition]
   `AtomKind`* {.preservesOr, pure.} = enum
     `Boolean`, `Float`, `Double`, `SignedInteger`, `String`, `ByteString`,
     `Symbol`
-  Definitions* = Table[Symbol, Definition]
+  DictionaryEntries* = Table[Value, NamedSimplePattern]
-  DictionaryEntries* = Table[Preserve[void], NamedSimplePattern]
   NamedPatternKind* {.pure.} = enum
     `named`, `anonymous`
   `NamedPattern`* {.acyclic, preservesOr.} = ref object
@@ -75,19 +75,19 @@ type
   SimplePatternAtom* {.preservesRecord: "atom".} = object
     `atomKind`*: AtomKind
 
-  SimplePatternEmbedded* {.acyclic, preservesRecord: "embedded".} = ref object
+  SimplePatternEmbedded* {.preservesRecord: "embedded".} = object
     `interface`*: SimplePattern
 
   SimplePatternLit* {.preservesRecord: "lit".} = object
-    `value`*: Preserve[void]
+    `value`*: Value
 
-  SimplePatternSeqof* {.acyclic, preservesRecord: "seqof".} = ref object
+  SimplePatternSeqof* {.preservesRecord: "seqof".} = object
     `pattern`*: SimplePattern
 
-  SimplePatternSetof* {.acyclic, preservesRecord: "setof".} = ref object
+  SimplePatternSetof* {.preservesRecord: "setof".} = object
     `pattern`*: SimplePattern
 
-  SimplePatternDictof* {.acyclic, preservesRecord: "dictof".} = ref object
+  SimplePatternDictof* {.preservesRecord: "dictof".} = object
     `key`*: SimplePattern
     `value`*: SimplePattern
 
@@ -120,7 +120,7 @@ type
   
   NamedSimplePatternKind* {.pure.} = enum
     `named`, `anonymous`
-  `NamedSimplePattern`* {.acyclic, preservesOr.} = ref object
+  `NamedSimplePattern`* {.preservesOr.} = object
     case orKind*: NamedSimplePatternKind
     of NamedSimplePatternKind.`named`:
         `named`*: Binding
@@ -131,23 +131,23 @@ type
   
   DefinitionKind* {.pure.} = enum
     `or`, `and`, `Pattern`
-  DefinitionOrField0* {.acyclic, preservesTuple.} = ref object
+  DefinitionOrField0* {.preservesTuple.} = object
     `pattern0`*: NamedAlternative
     `pattern1`*: NamedAlternative
     `patternN`* {.preservesTupleTail.}: seq[NamedAlternative]
 
-  DefinitionOr* {.acyclic, preservesRecord: "or".} = ref object
+  DefinitionOr* {.preservesRecord: "or".} = object
     `field0`*: DefinitionOrField0
 
-  DefinitionAndField0* {.acyclic, preservesTuple.} = ref object
+  DefinitionAndField0* {.preservesTuple.} = object
     `pattern0`*: NamedPattern
     `pattern1`*: NamedPattern
     `patternN`* {.preservesTupleTail.}: seq[NamedPattern]
 
-  DefinitionAnd* {.acyclic, preservesRecord: "and".} = ref object
+  DefinitionAnd* {.preservesRecord: "and".} = object
     `field0`*: DefinitionAndField0
 
-  `Definition`* {.acyclic, preservesOr.} = ref object
+  `Definition`* {.preservesOr.} = object
     case orKind*: DefinitionKind
     of DefinitionKind.`or`:
         `or`*: DefinitionOr
@@ -159,16 +159,16 @@ type
         `pattern`*: Pattern
 
   
-  NamedAlternative* {.acyclic, preservesTuple.} = ref object
+  NamedAlternative* {.preservesTuple.} = object
     `variantLabel`*: string
     `pattern`*: Pattern
 
-  SchemaField0* {.acyclic, preservesDictionary.} = ref object
+  SchemaField0* {.preservesDictionary.} = object
     `definitions`*: Definitions
     `embeddedType`*: EmbeddedTypeName
     `version`* {.preservesLiteral: "1".}: tuple[]
 
-  Schema* {.acyclic, preservesRecord: "schema".} = ref object
+  Schema* {.preservesRecord: "schema".} = object
     `field0`*: SchemaField0
 
   PatternKind* {.pure.} = enum
@@ -182,7 +182,7 @@ type
         `compoundpattern`*: CompoundPattern
 
   
-  Binding* {.acyclic, preservesRecord: "named".} = ref object
+  Binding* {.preservesRecord: "named".} = object
     `name`*: Symbol
     `pattern`*: SimplePattern
 
@@ -198,7 +198,7 @@ proc `$`*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
     Schema |
     Pattern |
     Binding): string =
-  `$`(toPreserve(x))
+  `$`(toPreserves(x))
 
 proc encode*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
     EmbeddedTypeName |
@@ -212,4 +212,4 @@ proc encode*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
     Schema |
     Pattern |
     Binding): seq[byte] =
-  encode(toPreserve(x))
+  encode(toPreserves(x))

src/preserves/schemac.nim

@@ -1,47 +0,0 @@
-# SPDX-FileCopyrightText: ☭ Emery Hemingway
-# SPDX-License-Identifier: Unlicense
-
-import std/[hashes, options, os, parseopt, streams, strutils, tables]
-
-import ../preserves, ./schema, ./schemaparse
-
-when isMainModule:
-  let outStream = newFileStream(stdout)
-
-  for kind, key, inputPath in getopt():
-    case kind
-    of cmdEnd: discard
-    of cmdArgument:
-      quit "arguments must be prefixed by --schema: or --bundle:"
-    of cmdLongOption:
-      if inputPath == "":
-        quit "long command line options require a path argument"
-      case key
-
-      of "schema":
-        var schema = parsePreservesSchema(readFile(inputPath))
-        write(outStream, schema.toPreserve)
-
-      of "bundle":
-        let bundle = Bundle()
-        if not dirExists inputPath:
-          quit "not a directory of schemas: " & inputPath
-        else:
-          for filePath in walkDirRec(inputPath, relative = true):
-            var (dirPath, fileName, fileExt) = splitFile(filePath)
-            if fileExt == ".prs":
-              var
-                scm = parsePreservesSchema(readFile(inputPath / filePath))
-                path: ModulePath
-              for e in split(dirPath, '/'):
-                add(path, Symbol e)
-              add(path, Symbol fileName)
-              bundle.modules[path] = scm
-          if bundle.modules.len == 0:
-            quit "no schemas parsed"
-          else:
-            write(outStream, bundle.toPreserve)
-
-      else: quit("unhandled option " & key)
-    else: quit("unhandled option " & key)
-  close(outStream)

src/preserves/schemac.nim.cfg

@@ -1 +0,0 @@
-d:npegDotDir:"../.."

src/preserves/schemaparse.nim

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
 import std/[strutils, tables]
@@ -9,7 +9,6 @@ import npeg
 import ../preserves, ./schema, ./pegs
 
 type
-  Value = Preserve[void]
   Stack = seq[tuple[node: Value, pos: int]]
   ParseState = object
     schema: SchemaField0
@@ -57,20 +56,25 @@ template pushStack(n: Value) =
   assert(p.stack.len > 0, capture[0].s)
 
 proc toSymbolLit(s: string): Value =
-  initRecord[void](toSymbol"lit", toSymbol s)
+  initRecord(toSymbol"lit", toSymbol s)
 
 proc match(text: string; p: var ParseState)
 
 const parser = peg("Schema", p: ParseState):
 
-  Schema <- ?Annotation * S * +(Clause * S) * !1
+  Schema <- S * +Clause * !1
 
-  Clause <- (Version | EmbeddedTypeName | Include | Definition) * S * '.'
+  Clause <- (Version | EmbeddedTypeName | Include | Definition | +LineComment) * S * '.' * S
 
   Version <- "version" * S * >(*Digit):
     if parseInt($1) != 1: fail()
 
-  EmbeddedTypeName <- "embeddedType" * S * >("#f" | Ref)
+  EmbeddedTypeName <- "embeddedType" * S * ("#f" | Ref):
+    if capture.len == 1:
+      var r = popStack()
+      p.schema.embeddedType =
+        EmbeddedTypeName(orKind: EmbeddedTypeNameKind.Ref)
+      validate p.schema.embeddedType.`ref`.fromPreserves(r)
 
   Include <- "include" * S * '"' * >(+Preserves.char) * '"':
     var path: string
@@ -82,19 +86,20 @@ const parser = peg("Schema", p: ParseState):
     match(readFile path, state)
     p.schema = move state.schema
 
-  Definition <- >id * S * '=' * S * (OrPattern | AndPattern | Pattern):
+  Definition <- ?Annotation * *LineComment * id * '=' * S * (OrPattern | AndPattern | Pattern):
     if p.schema.definitions.hasKey(Symbol $1):
-      raise newException(ValueError, "duplicate definition of " & $1)
+      raise newException(ValueError, "duplicate definition of " & $0)
     var
       node = popStack()
       def: Definition
-    if not fromPreserve(def, node):
+    if not def.fromPreserves(node):
-      raise newException(ValueError, $1 & ": " & $node)
+      raise newException(ValueError, "failed to convert " & $1 & " to a Definition: " & $node)
     p.schema.definitions[Symbol $1] = def
     p.stack.setLen(0)
 
-  OrPattern <- ?('/' * S) * AltPattern * +(S * '/' * S * AltPattern):
+  OrDelim <-  *LineComment * '/' * S * *LineComment
-    var node = initRecord(toSymbol("or"), toPreserve takeStackAt())
+  OrPattern <- ?OrDelim * AltPattern * +(S * OrDelim * AltPattern):
+    var node = initRecord(toSymbol("or"), takeStackAt().toPreserves)
     pushStack node
 
   AltPattern <-
@@ -103,21 +108,22 @@ const parser = peg("Schema", p: ParseState):
       AltRef |
       AltLiteralPattern
 
-  AltNamed <- '@' * >id * S * Pattern:
+  AltNamed <- atId * ?Annotation * Pattern:
-    var n = toPreserve @[toPreserve $1] & takeStackAt()
+    var n = toPreserves(@[toPreserves $1] & takeStackAt())
     pushStack n
 
-  AltRecord <- '<' * >id * *(S * NamedPattern) * '>':
+  AltRecord <- '<' * id * *NamedPattern * '>':
-    var n = toPreserve @[
+    var n = toPreserves @[
-      toPreserve $1,
+      toPreserves $1,
       initRecord(
         toSymbol"rec",
         toSymbolLit $1,
-        initRecord(toSymbol"tuple", toPreserve takeStackAt()))]
+        initRecord(toSymbol"tuple", toPreserves takeStackAt()))]
     pushStack n
 
   AltRef <- Ref:
-    var n = toPreserve @[toPreserve $0] & takeStackAt()
+    var r = popStack()
+    var n = toPreserves @[r[1].symbol.string.toPreserves, r]
     pushStack n
 
   AltLiteralPattern <-
@@ -131,18 +137,18 @@ const parser = peg("Schema", p: ParseState):
     of "#f": "false"
     of "#t": "true"
     else: $1
-    var n = toPreserve @[
+    var n = toPreserves @[
-      toPreserve id,
+      toPreserves id,
       initRecord(toSymbol"lit", parsePreserves $1)]
     pushStack n
 
-  AndPattern <- ?('&' * S) * NamedPattern * +(S * '&' * S * NamedPattern):
+  AndPattern <- ?'&' * S * NamedPattern * +('&' * S * NamedPattern):
-    var node = initRecord(toSymbol("and"), toPreserve takeStackAt())
+    var node = initRecord(toSymbol("and"), toPreserves takeStackAt())
     pushStack node
 
   Pattern <- SimplePattern | CompoundPattern
 
-  SimplePattern <-
+  SimplePattern <- (
       AnyPattern |
       AtomKindPattern |
       EmbeddedPattern |
@@ -150,7 +156,7 @@ const parser = peg("Schema", p: ParseState):
       SequenceOfPattern |
       SetOfPattern |
       DictOfPattern |
-      Ref
+      Ref ) * S
 
   AnyPattern <- "any":
     pushStack toSymbol"any"
@@ -185,17 +191,17 @@ const parser = peg("Schema", p: ParseState):
   LiteralPattern <- ('=' * >symbol) | ("<<lit>" * >Preserves.Value * ">") | >nonSymbolAtom:
     pushStack initRecord(toSymbol"lit", parsePreserves($1))
 
-  SequenceOfPattern <- '[' * S * SimplePattern * S * "..." * S * ']':
+  SequenceOfPattern <- '[' * S * SimplePattern * "..." * S * ']':
     var n = initRecord(toSymbol"seqof", popStack())
     pushStack n
 
-  SetOfPattern <- "#{" * S * SimplePattern * S * '}':
+  SetOfPattern <- "#{" * S * SimplePattern * '}':
     var n = initRecord(toSymbol"setof", popStack())
     pushStack n
 
   DictOfPattern <-
-      '{' *
+      '{' * S *
-          S * SimplePattern * S * ':' * S * SimplePattern * S * "...:..." * S *
+          ?Annotation * SimplePattern * ':' * S * SimplePattern * "...:..." * S *
       '}':
     var
       val = popStack()
@@ -210,60 +216,68 @@ const parser = peg("Schema", p: ParseState):
     var n = initRecord(toSymbol"ref", path, name)
     pushStack n
 
-  CompoundPattern <-
+  CompoundPattern <- (
       RecordPattern |
+      VariableRecordPattern |
       TuplePattern |
       VariableTuplePattern |
-      DictionaryPattern
+      DictionaryPattern ) * S
 
   RecordPattern <-
-      ("<<rec>" * S * NamedPattern * *(S * NamedPattern) * '>') |
+      ("<<rec>" * S * NamedPattern * *NamedPattern * '>') |
       ('<' * >Value * *(S * NamedPattern) * '>'):
     if capture.len == 2:
       var n = initRecord(toSymbol"rec",
         toSymbolLit $1,
-        initRecord(toSymbol"tuple", toPreserve takeStackAfter()))
+        initRecord(toSymbol"tuple", toPreserves takeStackAfter()))
       pushStack n
     else:
       var n = initRecord(toSymbol"rec", takeStackAfter())
       pushStack n
 
-  TuplePattern <-
+  VariableRecordPattern <- '<' * >Value * S * *(NamedPattern) * "..." * S * '>':
-      '[' * S * *(NamedPattern * S) * ']':
-    var n = initRecord(toSymbol"tuple", toPreserve takeStackAfter())
-    pushStack n
-
-  VariableTuplePattern <-
-      '[' * S * *(NamedPattern * S) * ?(Pattern * S) * "..." * S * ']':
     var fields = takeStackAfter()
     var tail = fields.pop
     tail[1] = initRecord(toSymbol"seqof", tail[1])
-    var node = initRecord(toSymbol"tuplePrefix", toPreserve fields, tail)
+    var n = initRecord(
+        toSymbol"rec",
+        toSymbolLit $1,
+        initRecord(toSymbol"tuplePrefix", toPreserves fields, tail))
+    pushStack n
+
+  TuplePattern <-
+      '[' * S * *NamedPattern * ']':
+    var n = initRecord(toSymbol"tuple", toPreserves takeStackAfter())
+    pushStack n
+
+  VariableTuplePattern <-
+      '[' * S * *NamedPattern * ?Pattern * "..." * S * ']':
+    var fields = takeStackAfter()
+    var tail = fields.pop
+    tail[1] = initRecord(toSymbol"seqof", tail[1])
+    var node = initRecord(toSymbol"tuplePrefix", toPreserves fields, tail)
     pushStack node
 
-  DictionaryPattern <- '{' * S * *(>Value * S * ':' * S * NamedSimplePattern * ?',' * S) * '}':
+  DictionaryPattern <- '{' * *(S * >Value * S * ':' * S * NamedSimplePattern * ?',') * S * '}':
-    var dict = initDictionary(void)
+    var dict = initDictionary()
     for i in countDown(pred capture.len, 1):
       let key = toSymbol capture[i].s
       dict[key] =  initRecord("named", key, popStack())
     var n = initRecord(toSymbol"dict", dict)
     pushStack n
 
-  NamedPattern <- ('@' * >id * S * SimplePattern) | Pattern:
+  NamedPattern <- ((atId * ?Annotation * SimplePattern) | Pattern):
     if capture.len == 2:
       var n = initRecord(toSymbol"named", toSymbol $1, popStack())
       pushStack n
 
-  NamedSimplePattern <- ('@' * >id * S * SimplePattern) | SimplePattern:
+  NamedSimplePattern <- ((atId * ?Annotation * SimplePattern) | SimplePattern):
     if capture.len == 2:
       var n = initRecord(toSymbol"named", toSymbol $1, popStack())
       pushStack n
 
-  id <- Alpha * *Alnum
+  id <- >(Alpha * *Alnum) * S
-
+  atId <- ?Annotation * '@' * id
-  Comment <- '#' * @'\n'
-
-  S <- *(Space | Comment)
 
   symbol <- Preserves.Symbol
 
@@ -278,7 +292,11 @@ const parser = peg("Schema", p: ParseState):
   Value <- Preserves.Value:
     discard
 
-  Annotation <- '@' * Value
+  Annotation <- '@' * (Preserves.String | Preserves.Record) * S
+
+  S <- *{ ' ', '\t', '\r', '\n' }
+
+  LineComment <- '#' * @'\n' * S
 
 proc match(text: string; p: var ParseState) =
   let match = parser.match(text, p)
@@ -301,5 +319,5 @@ when isMainModule:
   if txt != "":
     let
       scm = parsePreservesSchema(txt)
-      pr = toPreserve scm
+      pr = toPreserves scm
     stdout.newFileStream.writeText(pr, textPreserves)

src/preserves/sprypreserves.nim

@@ -1,154 +0,0 @@
-# SPDX-FileCopyrightText: ☭ 2021 Emery Hemingway
-# SPDX-License-Identifier: Unlicense
-
-import std/[sequtils, tables]
-
-import spryvm/spryvm
-
-import ../preserves
-
-type
-  PreservesNode* = ref object of Value
-    preserve: Preserve[void]
-  EmbeddedNode* = ref object of PreservesNode
-  ByteStringNode* = ref object of StringVal
-  RecordNode* = ref object of Blok
-  SetNode = ref object of PreservesNode
-
-method eval*(self: PreservesNode; spry: Interpreter): Node =
-  self
-
-method `$`*(self: PreservesNode): string =
-  $self.preserve
-
-method typeName*(self: PreservesNode): string =
-  "preserves-value"
-
-method typeName*(self: EmbeddedNode): string =
-  "preserves-embedded-value"
-
-method typeName*(self: ByteStringNode): string =
-  "preserves-bytestring"
-
-method typeName*(self: RecordNode): string =
-  "preserves-record"
-
-method typeName*(self: SetNode): string =
-  "preserves-set"
-
-proc toSpry(pr: Preserve[void], spry: Interpreter): Node =
-  if pr.embedded:
-    result = EmbeddedNode(preserve: pr)
-      # TODO: need to be able to manipulate these
-  else:
-    case pr.kind
-    of pkBoolean:
-      result = boolVal(pr.bool, spry)
-    of pkFloat:
-      result = newValue(pr.float)
-    of pkDouble:
-      result = newValue(pr.double)
-    of pkSignedInteger:
-      result = newValue(int pr.int)
-    of pkString:
-      result = newValue(pr.string)
-    of pkByteString:
-      result = ByteStringNode(value: cast[string](pr.bytes))
-    of pkSymbol:
-      result =
-        if pr.symbol == Symbol"null": newNilVal()
-        else: newLitWord(spry, string pr.symbol)
-    of pkRecord:
-      var comp = RecordNode()
-      proc f(pr: Preserve[void]): Node = toSpry(pr, spry)
-      comp.nodes = map(pr.record, f)
-      result = comp
-    of pkSequence:
-      var blk = newBlok()
-      for e in pr.sequence: blk.add toSpry(e, spry)
-      result = blk
-    of pkSet:
-      result = SetNode(preserve: pr)
-    of pkDictionary:
-      var map = newMap()
-      for (key, val) in pr.dict.items:
-        map[toSpry(key, spry)] = toSpry(val, spry)
-      result = map
-    of pkEmbedded:
-      result = EmbeddedNode(preserve: pr)
-
-proc toPreserveHook*(node: Node; E: typedesc): Preserve[E] =
-  if node of PreservesNode:
-    result = PreservesNode(node).preserve
-  elif node of RecordNode:
-    result = Preserve[E](kind: pkRecord)
-    var comp = RecordNode(node)
-    proc f(child: Node): Preserve[void] = toPreserve(child, void)
-    result.record = map(comp.nodes, f)
-  elif node of ByteStringNode:
-    result = toPreserve(cast[seq[byte]](ByteStringNode(node).value), E)
-  elif node of Blok:
-    var blk = Blok(node)
-    result = initSequence(blk.nodes.len, E)
-    for i, child in blk.nodes: result.sequence[i] = toPreserve(child, E)
-  elif node of Map:
-    result = initDictionary(E)
-    for key, val in Map(node).bindings:
-      result[toPreserve(key, E)] = toPreserve(val, E)
-  elif node of StringVal:
-    result = toPreserve(StringVal(node).value, E)
-  elif node of LitWord:
-    result = toSymbol(LitWord(node).word, E)
-  elif node of IntVal:
-    result = toPreserve(IntVal(node).value, E)
-  elif node of FloatVal:
-    result = toPreserve(FloatVal(node).value, E)
-  elif node of BoolVal:
-    result = toPreserve(BoolVal(node).value, E)
-  else: # node of NilVal:
-    result = toSymbol("null", E)
-
-when isMainModule:
-  var
-    node: Node
-    pr = toPreserveHook(node, void)
-
-proc addPreserves*(spry: Interpreter) =
-  nimFunc("parsePreserves"):
-    let node = evalArg(spry)
-    if node of StringVal:
-      let str = StringVal(node).value
-      result = PreservesNode(preserve: parsePreserves(str))
-
-  nimFunc("decodePreserves"):
-    let node = evalArg(spry)
-    if node of StringVal:
-      let str = StringVal(node).value
-      result = PreservesNode(preserve: decodePreserves(cast[seq[byte]](str)))
-
-  nimMeth("encodePreserves"):
-    let node = evalArgInfix(spry)
-    if node of PreservesNode:
-      var bin = encode PreservesNode(node).preserve
-      result = newValue(cast[string](bin))
-
-  nimFunc("fromPreserves"):
-    let node = evalArg(spry)
-    if node of PreservesNode:
-      let pr = PreservesNode(node).preserve
-      return toSpry(pr, spry)
-
-  nimMeth("toPreserves"):
-    let node = evalArgInfix(spry)
-    PreservesNode(preserve: node.toPreserve)
-
-  nimMeth("arity"):
-    let node = evalArgInfix(spry)
-    if node of RecordNode:
-      return newValue(pred SeqComposite(node).nodes.len)
-
-  nimMeth("label"):
-    let node = evalArgInfix(spry)
-    if node of RecordNode:
-      let rec = RecordNode(node)
-      return rec.nodes[rec.nodes.high]

src/preserves/sprypreserves.nim.cfg

@@ -1,2 +0,0 @@
-multimethods:on
-nilseqs:on

src/preserves/xmlhooks.nim

@@ -1,36 +1,36 @@
-# SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[parseutils, strtabs, tables, xmltree]
+import std/[parseutils, strtabs, xmltree]
 import ../preserves
 
-proc toPreserveFromString*(s: string; E: typedesc): Preserve[E] =
+proc toPreservesFromString*(s: string): Value =
   # This is a bad and slow thing to do, but that is XML.
   case s
   of "false", "no", "off":
-    result = toPreserve(false, E)
+    result = toPreserves(false)
   of "true", "yes", "on":
-    result = toPreserve(true, E)
+    result = toPreserves(true)
   else:
     var
       n: BiggestInt
       f: BiggestFloat
     if parseBiggestInt(s, n) == s.len:
-      result = toPreserve(n, E)
+      result = toPreserves(n)
     elif parseHex(s, n) == s.len:
-      result = toPreserve(n, E)
+      result = toPreserves(n)
     elif parseFloat(s, f) == s.len:
-      result = toPreserve(f, E)
+      result = toPreserves(f)
     else:
-      result = toPreserve(s, E)
+      result = toPreserves(s)
 
-proc toPreserveHook*(xn: XmlNode; E: typedesc): Preserve[E] =
+proc toPreservesHook*(xn: XmlNode): Value =
   if xn.kind == xnElement:
-    result = Preserve[E](kind: pkRecord)
+    result = Value(kind: pkRecord)
     if not xn.attrs.isNil:
-      var attrs = initDictionary(E)
+      var attrs = initDictionary()
       for xk, xv in xn.attrs.pairs:
-        attrs[toSymbol(xk, E)] = toPreserveFromString(xv, E)
+        attrs[toSymbol(xk)] = toPreservesFromString(xv)
       result.record.add(attrs)
     var isText = xn.len > 0
       # escaped text is broken up and must be concatenated
@@ -39,20 +39,20 @@ proc toPreserveHook*(xn: XmlNode; E: typedesc): Preserve[E] =
         isText = false
         break
     if isText:
-      result.record.add(toPreserve(xn.innerText, E))
+      result.record.add(toPreserves(xn.innerText))
     else:
       for child in xn.items:
         case child.kind
         of xnElement:
-          result.record.add(toPreserveHook(child, E))
+          result.record.add(toPreservesHook(child))
         of xnText, xnVerbatimText, xnCData, xnEntity:
-          result.record.add(toPreserve(text(child), E))
+          result.record.add(toPreserves(text(child)))
         of xnComment:
           discard
-    result.record.add(toSymbol(xn.tag, E))
+    result.record.add(toSymbol(xn.tag))
       # record labels are stored after the fields
 
-proc toUnquotedString[E](pr: Preserve[E]): string {.inline.} =
+proc toUnquotedString(pr: Value): string {.inline.} =
   case pr.kind
   of pkString:
     pr.string
@@ -60,7 +60,7 @@ proc toUnquotedString[E](pr: Preserve[E]): string {.inline.} =
     if pr.bool: "true" else: "false"
   else: $pr
 
-proc fromPreserveHook*[E](xn: var XmlNode; pr: Preserve[E]): bool =
+proc fromPreservesHook*(xn: var XmlNode; pr: Value): bool =
   if pr.kind == pkRecord and pr.label.kind == pkSymbol:
     xn = newElement($pr.label)
     var i: int
@@ -74,7 +74,7 @@ proc fromPreserveHook*[E](xn: var XmlNode; pr: Preserve[E]): bool =
         xn.add newText(e.string)
       else:
         var child: XmlNode
-        result = fromPreserveHook(child, e)
+        result = fromPreservesHook(child, e)
         if not result: return
         xn.add child
       inc i
@@ -82,5 +82,5 @@ proc fromPreserveHook*[E](xn: var XmlNode; pr: Preserve[E]): bool =
 
 when isMainModule:
   var xn = newElement("foobar")
-  var pr = xn.toPreserveHook(void)
+  var pr = xn.toPreservesHook()
-  assert fromPreserveHook(xn, pr)
+  assert fromPreservesHook(xn, pr)

tests/Tupfile

@@ -1,3 +1,7 @@
 include_rules
 NIM_FLAGS_test_samples += -d:upstreamTestfile="$(TUP_CWD)/../../preserves/tests/samples.pr"
 : foreach t*.nim |> !nim_run |> | ../<test>
+
+: $(BIN_DIR)/<preserves-schemac> \
+	|> $(BIN_DIR)/preserves-schemac --no-bundle ../../preserves/doc/demo.prs | xxd > %o \
+	|> demo.xxd

tests/config.nims

@@ -1 +1 @@
-switch("path", "$projectDir/../src")
+switch("path", "$projectDir/../src")

tests/test_buffereddecoder.nim

@@ -9,7 +9,7 @@ suite "BufferedDecoder":
   test "half-string":
     var
       buf = newBufferedDecoder()
-      pr = Preserve[void](kind: pkByteString, bytes: newSeq[byte](23))
+      pr = Value(kind: pkByteString, bytes: newSeq[byte](23))
       ok: bool
     for i, _ in pr.bytes:
       pr.bytes[i] = byte(i)

tests/test_conversions.nim

@@ -6,44 +6,49 @@ import preserves, preserves/xmlhooks
 
 type
   Route {.preservesRecord: "route".} = object
-    `transports`*: seq[Preserve[void]]
+    `transports`*: seq[Value]
-    `pathSteps`* {.preservesTupleTail.}: seq[Preserve[void]]
+    `pathSteps`* {.preservesTupleTail.}: seq[Value]
 
 suite "conversions":
   test "dictionary":
     type Bar = tuple
       s: string
     type Foobar {.preservesDictionary.} = object
-      a, b: int
+      a: int
+      b: seq[int]
       c {.preservesEmbedded.}: Bar
+      d: Option[bool]
+      e: Option[bool]
     let
-      c = Foobar(a: 1, b: 2, c: ("ku", ))
+      c = Foobar(a: 1, b: @[2], c: ("ku", ), e: some(true))
-      b = toPreserve(c)
+      b = toPreserves(c)
-      a = preserveTo(b, Foobar)
+      a = preservesTo(b, Foobar)
-    check($b == """{a: 1 b: 2 c: #!["ku"]}""")
+    check($b == """{a: 1 b: [2] c: #!["ku"] e: #t}""")
-    check(a.isSome and (get(a) == c))
+    check(a.isSome)
+    if a.isSome: check(get(a) == c)
     check(b.kind == pkDictionary)
 
   test "records":
     type Bar {.preservesRecord: "bar".} = object
       s: string
     type Foobar {.preservesRecord: "foo".} = object
-      a, b: int
+      a: int
+      b: seq[int]
       c: Bar
     let
-      tup = Foobar(a: 1, b: 2, c: Bar(s: "ku", ))
+      tup = Foobar(a: 1, b: @[2], c: Bar(s: "ku", ))
-      prs = toPreserve(tup)
+      prs = toPreserves(tup)
     check(prs.kind == pkRecord)
-    check($prs == """<foo 1 2 <bar "ku">>""")
+    check($prs == """<foo 1 [2] <bar "ku">>""")
-    check(preserveTo(prs, Foobar) == some(tup))
+    check(preservesTo(prs, Foobar) == some(tup))
 
   test "tables":
     var a: Table[int, string]
     for i, s in ["a", "b", "c"]: a[i] = s
-    let b = toPreserve(a)
+    let b = toPreserves(a)
     check($b == """{0: "a" 1: "b" 2: "c"}""")
     var c: Table[int, string]
-    check(fromPreserve(c, b))
+    check(fromPreserves(c, b))
     check(a == c)
 
   test "XML":
@@ -61,17 +66,39 @@ suite "conversions":
         <rect x="1" y="1" width="998" height="298" fill="none" stroke="blue" stroke-width="2"/>
       </svg>
     """
-    var pr = toPreserve(b, void)
+    var pr = toPreserves(b)
     checkpoint $pr
-    check fromPreserve(a, pr)
+    check fromPreserves(a, pr)
 
   test "preservesTupleTail":
     let pr = parsePreserves """<route [<tcp "localhost" 1024>] <ref {oid: "syndicate" sig: #x"69ca300c1dbfa08fba692102dd82311a"}>>"""
     var route: Route
-    check route.fromPreserve(pr)
+    check route.fromPreserves(pr)
+
+  test "ebedded":
+    type
+      Foo {.preservesRecord: "foo".} = object
+        n: int
+        bar {.preservesEmbedded.}: Bar
+      Bar = ref object of RootObj
+        x: int
+      Baz = ref object of RootObj
+        x: int
+
+    let a = initRecord("foo", 9.toPreserves, embed Bar(x: 768))
+    checkpoint $a
+    check a.preservesTo(Foo).isSome
+
+    let b = initRecord("foo", 2.toPreserves, embed Baz(x: 999))
+    checkpoint $b
+    check not b.preservesTo(Foo).isSome
 
 suite "toPreserve":
-  template check(p: Preserve; s: string) =
+  template check(p: Value; s: string) =
     test s: check($p == s)
-  check false.toPreserve, "#f"
+  check false.toPreserves, "#f"
-  check [0, 1, 2, 3].toPreserve, "[0 1 2 3]"
+  check [0, 1, 2, 3].toPreserves, "[0 1 2 3]"
+
+  test "toRecord":
+    let r = toRecord(Symbol"foo", "üks", "kaks", "kolm", {4..7})
+    check $r == """<foo "üks" "kaks" "kolm" #{4 5 6 7}>"""

tests/test_parser.nim

@@ -27,15 +27,17 @@ suite "parse":
   for (txt, bin) in examples:
     test txt:
       checkpoint(txt)
-      let test = parsePreserves(txt, int)
+      let test = parsePreserves(txt)
       checkpoint($test)
       block:
         let
           a = test
-          b = decodePreserves(bin, int)
+          b = decodePreserves(bin)
         check(a == b)
       block:
         let
           a = encode test
           b = bin
         check(cast[string](a).toHex == b.toHex)
+      if test.isAtomic:
+        discard parsePreservesAtom(txt)

tests/test_parser.nim.cfg

@@ -1 +0,0 @@
-threads:off

tests/test_rfc8259.nim

@@ -57,7 +57,7 @@ for i, jsText in testVectors:
     checkpoint(jsText)
     let
       control = parseJson jsText
-      x = control.toPreserve
+      x = control.toPreserves
     checkpoint($x)
     var stream = newStringStream()
     stream.write(x)

tests/test_samples.nim

@@ -4,29 +4,27 @@
 import std/unittest
 import preserves
 
-type Value = Preserve[void]
-
 const upstreamTestfile {.strdefine.} = ""
 
-proc strip(pr: Preserve): Preserve = pr
+proc strip(pr: Value): Value = pr
 
 proc encodeBinary(pr: Value): Value =
-  result = encode(pr).toPreserve
+  result = encode(pr).toPreserves
   checkpoint("encoded binary: " & $result)
 
 proc looseEncodeBinary(pr: Value): Value =
-  result = encode(pr).toPreserve
+  result = encode(pr).toPreserves
   checkpoint("loose encoded binary: " & $result)
 
 proc annotatedBinary(pr: Value): Value =
-  result = encode(pr).toPreserve
+  result = encode(pr).toPreserves
   checkpoint("annotated binary: " & $result)
 
 proc decodeBinary(pr: Value): Value =
   result = decodePreserves(pr.bytes)
 
 proc encodeText(pr: Value): Value =
-  result = ($pr).toPreserve
+  result = ($pr).toPreserves
   checkpoint("encoded text: " & result.string)
 
 proc decodeText(pr: Value): Value =
@@ -34,7 +32,7 @@ proc decodeText(pr: Value): Value =
   checkpoint("decoded text " & $pr)
 
 if upstreamTestfile != "":
-  let samples = readFile(upstreamTestfile).parsePreserves(void)
+  let samples = readFile(upstreamTestfile).parsePreserves()
   assert samples.isRecord("TestCases")
 
   var binary, annotatedValue, stripped, text, bytes: Value

tests/test_step.nim

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[options, sequtils, unittest]
+import preserves
+
+suite "step":
+  var data = parsePreserves """
+      <foo "bar" [ 0.0 {a: #f, "b": #t } ] >
+    """
+
+  var o = some data
+  for i in [1.toPreserves, 1.toPreserves, "b".toPreserves]:
+    test $i:
+      o = step(get o, i)
+      check o.isSome