Schema: a collection of breaking changes

- Process all modules in a bundle simultanously so that it can be
   determined if imported types are recursive or take a parameter
- Different rules for deciding when an object must be a ref
- Use Preserve[void] if there is no EmbededTypeName defined
- Put embeddeded types directly into other types without a
  Preserve[E] container

preserves.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version = "20221205"
+version = "20221208"
 author        = "Emery Hemingway"
 description   = "data model and serialization format"
 license       = "Unlicense"

src/preserves/Tupfile

@@ -1,2 +1,3 @@
 include_rules
+NIM_PATH += --path:$(TUP_CWD)/..
 : foreach *.nim |> !nim_run |>

src/preserves/private/preserves_encode.nim

@@ -37,7 +37,7 @@ when isMainModule:
     of "preserves_to_xml":
       let pr = stdin.readAll.decodePreserves
       var xn: XmlNode
-      if fromPreserveHook(xn, pr):
+      if fromPreserve(xn, pr):
         stdout.writeLine(xn)
       else:
         quit("Preserves not convertable to XML")

src/preserves/schema.nim

@@ -1,5 +1,5 @@
 
-import ../preserves, std/typetraits, std/tables
+import ../preserves, std/tables
 
 type
   Ref* {.preservesRecord: "ref".} = object
@@ -7,89 +7,90 @@ type
     `name`*: Symbol
 
   ModulePath* = seq[Symbol]
-  Bundle*[E] {.preservesRecord: "bundle".} = ref object
-    `modules`*: Modules[E]
+  Bundle* {.preservesRecord: "bundle".} = object
+    `modules`*: Modules
 
   CompoundPatternKind* {.pure.} = enum
     `rec`, `tuple`, `tuplePrefix`, `dict`
-  CompoundPatternRec*[E] {.preservesRecord: "rec".} = ref object
-    `label`*: NamedPattern[E]
-    `fields`*: NamedPattern[E]
+  CompoundPatternRec* {.preservesRecord: "rec".} = ref object
+    `label`*: NamedPattern
+    `fields`*: NamedPattern
 
-  CompoundPatternTuple*[E] {.preservesRecord: "tuple".} = ref object
-    `patterns`*: seq[NamedPattern[E]]
+  CompoundPatternTuple* {.preservesRecord: "tuple".} = ref object
+    `patterns`*: seq[NamedPattern]
 
-  CompoundPatternTuplePrefix*[E] {.preservesRecord: "tuplePrefix".} = ref object
-    `fixed`*: seq[NamedPattern[E]]
-    `variable`*: NamedSimplePattern[E]
+  CompoundPatternTuplePrefix* {.preservesRecord: "tuplePrefix".} = ref object
+    `fixed`*: seq[NamedPattern]
+    `variable`*: NamedSimplePattern
 
-  CompoundPatternDict*[E] {.preservesRecord: "dict".} = ref object
-    `entries`*: DictionaryEntries[E]
+  CompoundPatternDict* {.preservesRecord: "dict".} = ref object
+    `entries`*: DictionaryEntries
 
-  `CompoundPattern`*[E] {.preservesOr.} = ref object
+  `CompoundPattern`* {.preservesOr.} = ref object
     case orKind*: CompoundPatternKind
     of CompoundPatternKind.`rec`:
-        `rec`*: CompoundPatternRec[E]
+        `rec`*: CompoundPatternRec
 
     of CompoundPatternKind.`tuple`:
-        `tuple`*: CompoundPatternTuple[E]
+        `tuple`*: CompoundPatternTuple
 
     of CompoundPatternKind.`tuplePrefix`:
-        `tupleprefix`*: CompoundPatternTuplePrefix[E]
+        `tupleprefix`*: CompoundPatternTuplePrefix
 
     of CompoundPatternKind.`dict`:
-        `dict`*: CompoundPatternDict[E]
+        `dict`*: CompoundPatternDict
 
-
-  Modules*[E] = Table[ModulePath, Schema[E]]
+  
+  Modules* = Table[ModulePath, Schema]
   EmbeddedTypeNameKind* {.pure.} = enum
-     `false`, `Ref`
+    `false`, `Ref`
   `EmbeddedTypeName`* {.preservesOr.} = object
     case orKind*: EmbeddedTypeNameKind
     of EmbeddedTypeNameKind.`false`:
         `false`* {.preservesLiteral: "#f".}: bool
+
     of EmbeddedTypeNameKind.`Ref`:
         `ref`*: Ref
 
-
+  
   `AtomKind`* {.preservesOr, pure.} = enum
     `Boolean`, `Float`, `Double`, `SignedInteger`, `String`, `ByteString`,
     `Symbol`
-  Definitions*[E] = Table[Symbol, Definition[E]]
-  DictionaryEntries*[E] = Table[Preserve[E], NamedSimplePattern[E]]
+  Definitions* = Table[Symbol, Definition]
+  DictionaryEntries* = Table[Preserve[void], NamedSimplePattern]
   NamedPatternKind* {.pure.} = enum
     `named`, `anonymous`
-  `NamedPattern`*[E] {.preservesOr.} = ref object
+  `NamedPattern`* {.preservesOr.} = ref object
     case orKind*: NamedPatternKind
     of NamedPatternKind.`named`:
-        `named`*: Binding[E]
+        `named`*: Binding
 
     of NamedPatternKind.`anonymous`:
-        `anonymous`*: Pattern[E]
-
+        `anonymous`*: Pattern
 
+  
   SimplePatternKind* {.pure.} = enum
     `any`, `atom`, `embedded`, `lit`, `seqof`, `setof`, `dictof`, `Ref`
   SimplePatternAtom* {.preservesRecord: "atom".} = object
     `atomKind`*: AtomKind
 
-  SimplePatternEmbedded*[E] {.preservesRecord: "embedded".} = ref object
-    `interface`*: SimplePattern[E]
+  SimplePatternEmbedded* {.preservesRecord: "embedded".} = ref object
+    `interface`*: SimplePattern
 
-  SimplePatternLit*[E] {.preservesRecord: "lit".} = ref object
-    `value`*: Preserve[E]
+  SimplePatternLit* {.preservesRecord: "lit".} = object
+    `value`*: Preserve[void]
 
-  SimplePatternSeqof*[E] {.preservesRecord: "seqof".} = ref object
-    `pattern`*: SimplePattern[E]
+  SimplePatternSeqof* {.preservesRecord: "seqof".} = ref object
+    `pattern`*: SimplePattern
 
-  SimplePatternSetof*[E] {.preservesRecord: "setof".} = ref object
-    `pattern`*: SimplePattern[E]
+  SimplePatternSetof* {.preservesRecord: "setof".} = ref object
+    `pattern`*: SimplePattern
 
-  SimplePatternDictof*[E] {.preservesRecord: "dictof".} = ref object
-    `key`*: SimplePattern[E]
-    `value`*: SimplePattern[E]
+  SimplePatternDictof* {.preservesRecord: "dictof".} = ref object
+    `key`*: SimplePattern
+    `value`*: SimplePattern
 
-  `SimplePattern`*[E] {.preservesOr.} = ref object
+  `SimplePattern`* {.preservesOr.} = ref object
     case orKind*: SimplePatternKind
     of SimplePatternKind.`any`:
         `any`* {.preservesLiteral: "any".}: bool
@@ -98,119 +99,116 @@ type
         `atom`*: SimplePatternAtom
 
     of SimplePatternKind.`embedded`:
-        `embedded`*: SimplePatternEmbedded[E]
+        `embedded`*: SimplePatternEmbedded
 
     of SimplePatternKind.`lit`:
-        `lit`*: SimplePatternLit[E]
+        `lit`*: SimplePatternLit
 
     of SimplePatternKind.`seqof`:
-        `seqof`*: SimplePatternSeqof[E]
+        `seqof`*: SimplePatternSeqof
 
     of SimplePatternKind.`setof`:
-        `setof`*: SimplePatternSetof[E]
+        `setof`*: SimplePatternSetof
 
     of SimplePatternKind.`dictof`:
-        `dictof`*: SimplePatternDictof[E]
+        `dictof`*: SimplePatternDictof
 
     of SimplePatternKind.`Ref`:
         `ref`*: Ref
 
-
+  
   NamedSimplePatternKind* {.pure.} = enum
     `named`, `anonymous`
-  `NamedSimplePattern`*[E] {.preservesOr.} = ref object
+  `NamedSimplePattern`* {.preservesOr.} = ref object
     case orKind*: NamedSimplePatternKind
     of NamedSimplePatternKind.`named`:
-        `named`*: Binding[E]
+        `named`*: Binding
 
     of NamedSimplePatternKind.`anonymous`:
-        `anonymous`*: SimplePattern[E]
-
+        `anonymous`*: SimplePattern
 
+  
   DefinitionKind* {.pure.} = enum
     `or`, `and`, `Pattern`
-  DefinitionOrData*[E] {.preservesTuple.} = ref object
-    `pattern0`*: NamedAlternative[E]
-    `pattern1`*: NamedAlternative[E]
-    `patternN`* {.preservesTupleTail.}: seq[NamedAlternative[E]]
+  DefinitionOrData* {.preservesTuple.} = ref object
+    `pattern0`*: NamedAlternative
+    `pattern1`*: NamedAlternative
+    `patternN`* {.preservesTupleTail.}: seq[NamedAlternative]
 
-  DefinitionOr*[E] {.preservesRecord: "or".} = ref object
-    `data`*: DefinitionOrData[E]
+  DefinitionOr* {.preservesRecord: "or".} = ref object
+    `data`*: DefinitionOrData
 
-  DefinitionAndData*[E] {.preservesTuple.} = ref object
-    `pattern0`*: NamedPattern[E]
-    `pattern1`*: NamedPattern[E]
-    `patternN`* {.preservesTupleTail.}: seq[NamedPattern[E]]
+  DefinitionAndData* {.preservesTuple.} = ref object
+    `pattern0`*: NamedPattern
+    `pattern1`*: NamedPattern
+    `patternN`* {.preservesTupleTail.}: seq[NamedPattern]
 
-  DefinitionAnd*[E] {.preservesRecord: "and".} = ref object
-    `data`*: DefinitionAndData[E]
+  DefinitionAnd* {.preservesRecord: "and".} = ref object
+    `data`*: DefinitionAndData
 
-  `Definition`*[E] {.preservesOr.} = ref object
+  `Definition`* {.preservesOr.} = ref object
     case orKind*: DefinitionKind
     of DefinitionKind.`or`:
-        `or`*: DefinitionOr[E]
+        `or`*: DefinitionOr
 
     of DefinitionKind.`and`:
-        `and`*: DefinitionAnd[E]
-          # doesn't build with C++
+        `and`*: DefinitionAnd
 
     of DefinitionKind.`Pattern`:
-        `pattern`*: Pattern[E]
+        `pattern`*: Pattern
 
-
-  NamedAlternative*[E] {.preservesTuple.} = ref object
+  
+  NamedAlternative* {.preservesTuple.} = ref object
     `variantLabel`*: string
-    `pattern`*: Pattern[E]
+    `pattern`*: Pattern
 
-  SchemaData*[E] {.preservesDictionary.} = ref object
+  SchemaData* {.preservesDictionary.} = object
     `version`* {.preservesLiteral: "1".}: bool
     `embeddedType`*: EmbeddedTypeName
-    `definitions`*: Definitions[E]
+    `definitions`*: Definitions
 
-  Schema*[E] {.preservesRecord: "schema".} = ref object
-    `data`*: SchemaData[E]
+  Schema* {.preservesRecord: "schema".} = object
+    `data`*: SchemaData
 
   PatternKind* {.pure.} = enum
     `SimplePattern`, `CompoundPattern`
-  `Pattern`*[E] {.preservesOr.} = ref object
+  `Pattern`* {.preservesOr.} = ref object
     case orKind*: PatternKind
     of PatternKind.`SimplePattern`:
-        `simplepattern`*: SimplePattern[E]
+        `simplepattern`*: SimplePattern
 
     of PatternKind.`CompoundPattern`:
-        `compoundpattern`*: CompoundPattern[E]
+        `compoundpattern`*: CompoundPattern
 
-
-  Binding*[E] {.preservesRecord: "named".} = ref object
+  
+  Binding* {.preservesRecord: "named".} = ref object
     `name`*: Symbol
-    `pattern`*: SimplePattern[E]
+    `pattern`*: SimplePattern
 
-proc `$`*[E](x: Bundle[E] | CompoundPattern[E] | Modules[E] | Definitions[E] |
-    DictionaryEntries[E] |
-    NamedPattern[E] |
-    SimplePattern[E] |
-    NamedSimplePattern[E] |
-    Definition[E] |
-    NamedAlternative[E] |
-    Schema[E] |
-    Pattern[E] |
-    Binding[E]): string =
-  `$`(toPreserve(x, E))
-
-proc encode*[E](x: Bundle[E] | CompoundPattern[E] | Modules[E] | Definitions[E] |
-    DictionaryEntries[E] |
-    NamedPattern[E] |
-    SimplePattern[E] |
-    NamedSimplePattern[E] |
-    Definition[E] |
-    NamedAlternative[E] |
-    Schema[E] |
-    Pattern[E] |
-    Binding[E]): seq[byte] =
-  encode(toPreserve(x, E))
-
-proc `$`*(x: Ref | ModulePath | EmbeddedTypeName): string =
+proc `$`*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
+    EmbeddedTypeName |
+    Definitions |
+    DictionaryEntries |
+    NamedPattern |
+    SimplePattern |
+    NamedSimplePattern |
+    Definition |
+    NamedAlternative |
+    Schema |
+    Pattern |
+    Binding): string =
   `$`(toPreserve(x))
 
-proc encode*(x: Ref | ModulePath | EmbeddedTypeName): seq[byte] =
+proc encode*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
+    EmbeddedTypeName |
+    Definitions |
+    DictionaryEntries |
+    NamedPattern |
+    SimplePattern |
+    NamedSimplePattern |
+    Definition |
+    NamedAlternative |
+    Schema |
+    Pattern |
+    Binding): seq[byte] =
   encode(toPreserve(x))

src/preserves/schemac.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2022 ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[hashes, options, os, parseopt, streams, strutils, sets, tables]
+import std/[hashes, options, os, parseopt, streams, strutils, tables]
 
 import ../preserves, ./schema, ./schemaparse
 
@@ -23,7 +23,7 @@ when isMainModule:
         write(outStream, schema.toPreserve)
 
       of "bundle":
-        var bundle: Bundle[void]
+        var bundle: Bundle
         if not dirExists inputPath:
           quit "not a directory of schemas: " & inputPath
         else:

src/preserves/schemaparse.nim

@@ -10,9 +10,6 @@ import ../preserves, ./schema, ./pegs
 
 type
   Value = Preserve[void]
-  Definition = schema.Definition[void]
-  SchemaData = schema.SchemaData[void]
-  Schema = schema.Schema[void]
   Stack = seq[tuple[node: Value, pos: int]]
   ParseState = object
     schema: SchemaData
@@ -60,7 +57,7 @@ template pushStack(n: Value) =
   assert(p.stack.len > 0, capture[0].s)
 
 proc toSymbolLit(s: string): Value =
-  initRecord(toSymbol"lit", toSymbol s)
+  initRecord[void](toSymbol"lit", toSymbol s)
 
 proc match(text: string; p: var ParseState) {.gcsafe.}
 
@@ -205,10 +202,10 @@ const parser = peg("Schema", p: ParseState):
     pushStack n
 
   Ref <- >(Alpha * *Alnum) * *('.' * >(*Alnum)):
-    var path: seq[string]
-    for i in 1..<capture.len: path.add capture[i].s
-    let name = pop path
-    var n = initRecord(toSymbol"ref", toPreserve path, toSymbol name)
+    var path = initSequence[void]()
+    for i in 1..<capture.len: path.sequence.add(toSymbol capture[i].s)
+    var name = pop(path.sequence)
+    var n = initRecord(toSymbol"ref", path, name)
     pushStack n
 
   CompoundPattern <-
@@ -291,6 +288,7 @@ proc parsePreservesSchema*(text: string; directory = getCurrentDir()): Schema =
   ##
   ## Schemas in binary encoding should instead be parsed as Preserves
   ## and converted to `Schema` with `fromPreserve` or `preserveTo`.
+  assert directory != ""
   var p = ParseState(schema: SchemaData(), directory: directory)
   match(text, p)
   Schema(data: p.schema)
@@ -300,6 +298,6 @@ when isMainModule:
   let txt = readAll stdin
   if txt != "":
     let
-      scm = parsePreservesSchema(readAll stdin)
+      scm = parsePreservesSchema(txt)
       pr = toPreserve scm
-    stdout.newFileStream.write pr
+    stdout.newFileStream.writeText pr

tests/test_schemas.nim

@@ -1,40 +0,0 @@
-# SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
-# SPDX-License-Identifier: Unlicense
-
-import std/[tables, options, os, unittest]
-import preserves, preserves/schema, preserves/schemaparse
-
-if dirExists "tests": setCurrentDir "tests"
-  # nimble runs tests in the directory above
-
-suite "schema":
-
-  const
-    binPath = "../schema.bin"
-    textPath = "../schema.prs"
-
-  test "convertability":
-    if not fileExists(binPath): skip()
-    else:
-      var
-        b = decodePreserves readFile(binPath)
-        scm = preserveTo(b, Schema[void])
-      check scm.isSome
-      if scm.isSome:
-        var a = toPreserve(get scm)
-        check(a == b)
-
-  test "parser":
-    if not fileExists(binPath): skip()
-    else:
-      var
-        b = decodePreserves readFile(binPath)
-        scm = preserveTo(b, Schema[void])
-      check scm.isSome
-      if scm.isSome:
-        var a = toPreserve parsePreservesSchema(readFile textPath, textPath)
-        let aDefs = a[0][toSymbol "definitions"]
-        let bDefs = b[0][toSymbol "definitions"]
-        for (key, val) in aDefs.pairs:
-          check(bDefs[key] == val)
-        check(a == b)