ehmry
/
preserves-nim
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Schema: a collection of breaking changes 

- Process all modules in a bundle simultanously so that it can be
   determined if imported types are recursive or take a parameter
- Different rules for deciding when an object must be a ref
- Use Preserve[void] if there is no EmbededTypeName defined
- Put embeddeded types directly into other types without a
  Preserve[E] container
This commit is contained in:
Emery Hemingway 2022-12-08 01:53:11 -06:00
parent cc0703c343
commit d5909e4df0
8 changed files with 545 additions and 497 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
preserves.nimble
View File

@ -1,6 +1,6 @@
# Package # Package
version = "20221205" version = "20221208"
author = "Emery Hemingway" author = "Emery Hemingway"
description = "data model and serialization format" description = "data model and serialization format"
license = "Unlicense" license = "Unlicense"

1
src/preserves/Tupfile
View File

@ -1,2 +1,3 @@
include_rules include_rules
NIM_PATH += --path:$(TUP_CWD)/..
: foreach *.nim |> !nim_run |> : foreach *.nim |> !nim_run |>

765
src/preserves/preserves_schema_nim.nim
View File

File diff suppressed because it is too large Load Diff

2
src/preserves/private/preserves_encode.nim
View File

@ -37,7 +37,7 @@ when isMainModule:
of "preserves_to_xml": of "preserves_to_xml":
let pr = stdin.readAll.decodePreserves let pr = stdin.readAll.decodePreserves
var xn: XmlNode var xn: XmlNode
if fromPreserveHook(xn, pr): if fromPreserve(xn, pr):
stdout.writeLine(xn) stdout.writeLine(xn)
else: else:
quit("Preserves not convertable to XML") quit("Preserves not convertable to XML")

210
src/preserves/schema.nim
View File

@ -1,5 +1,5 @@
import ../preserves, std/typetraits, std/tables import ../preserves, std/tables
type type
Ref* {.preservesRecord: "ref".} = object Ref* {.preservesRecord: "ref".} = object
@ -7,89 +7,90 @@ type
`name`*: Symbol `name`*: Symbol
ModulePath* = seq[Symbol] ModulePath* = seq[Symbol]
Bundle*[E] {.preservesRecord: "bundle".} = ref object Bundle* {.preservesRecord: "bundle".} = object
`modules`*: Modules[E] `modules`*: Modules
CompoundPatternKind* {.pure.} = enum CompoundPatternKind* {.pure.} = enum
`rec`, `tuple`, `tuplePrefix`, `dict` `rec`, `tuple`, `tuplePrefix`, `dict`
CompoundPatternRec*[E] {.preservesRecord: "rec".} = ref object CompoundPatternRec* {.preservesRecord: "rec".} = ref object
`label`*: NamedPattern[E] `label`*: NamedPattern
`fields`*: NamedPattern[E] `fields`*: NamedPattern
CompoundPatternTuple*[E] {.preservesRecord: "tuple".} = ref object CompoundPatternTuple* {.preservesRecord: "tuple".} = ref object
`patterns`*: seq[NamedPattern[E]] `patterns`*: seq[NamedPattern]
CompoundPatternTuplePrefix*[E] {.preservesRecord: "tuplePrefix".} = ref object CompoundPatternTuplePrefix* {.preservesRecord: "tuplePrefix".} = ref object
`fixed`*: seq[NamedPattern[E]] `fixed`*: seq[NamedPattern]
`variable`*: NamedSimplePattern[E] `variable`*: NamedSimplePattern
CompoundPatternDict*[E] {.preservesRecord: "dict".} = ref object CompoundPatternDict* {.preservesRecord: "dict".} = ref object
`entries`*: DictionaryEntries[E] `entries`*: DictionaryEntries
`CompoundPattern`*[E] {.preservesOr.} = ref object `CompoundPattern`* {.preservesOr.} = ref object
case orKind*: CompoundPatternKind case orKind*: CompoundPatternKind
of CompoundPatternKind.`rec`: of CompoundPatternKind.`rec`:
`rec`*: CompoundPatternRec[E] `rec`*: CompoundPatternRec
of CompoundPatternKind.`tuple`: of CompoundPatternKind.`tuple`:
`tuple`*: CompoundPatternTuple[E] `tuple`*: CompoundPatternTuple
of CompoundPatternKind.`tuplePrefix`: of CompoundPatternKind.`tuplePrefix`:
`tupleprefix`*: CompoundPatternTuplePrefix[E] `tupleprefix`*: CompoundPatternTuplePrefix
of CompoundPatternKind.`dict`: of CompoundPatternKind.`dict`:
`dict`*: CompoundPatternDict[E] `dict`*: CompoundPatternDict
Modules*[E] = Table[ModulePath, Schema[E]] Modules* = Table[ModulePath, Schema]
EmbeddedTypeNameKind* {.pure.} = enum EmbeddedTypeNameKind* {.pure.} = enum
`false`, `Ref` `false`, `Ref`
`EmbeddedTypeName`* {.preservesOr.} = object `EmbeddedTypeName`* {.preservesOr.} = object
case orKind*: EmbeddedTypeNameKind case orKind*: EmbeddedTypeNameKind
of EmbeddedTypeNameKind.`false`: of EmbeddedTypeNameKind.`false`:
`false`* {.preservesLiteral: "#f".}: bool `false`* {.preservesLiteral: "#f".}: bool
of EmbeddedTypeNameKind.`Ref`: of EmbeddedTypeNameKind.`Ref`:
`ref`*: Ref `ref`*: Ref
`AtomKind`* {.preservesOr, pure.} = enum `AtomKind`* {.preservesOr, pure.} = enum
`Boolean`, `Float`, `Double`, `SignedInteger`, `String`, `ByteString`, `Boolean`, `Float`, `Double`, `SignedInteger`, `String`, `ByteString`,
`Symbol` `Symbol`
Definitions*[E] = Table[Symbol, Definition[E]] Definitions* = Table[Symbol, Definition]
DictionaryEntries*[E] = Table[Preserve[E], NamedSimplePattern[E]] DictionaryEntries* = Table[Preserve[void], NamedSimplePattern]
NamedPatternKind* {.pure.} = enum NamedPatternKind* {.pure.} = enum
`named`, `anonymous` `named`, `anonymous`
`NamedPattern`*[E] {.preservesOr.} = ref object `NamedPattern`* {.preservesOr.} = ref object
case orKind*: NamedPatternKind case orKind*: NamedPatternKind
of NamedPatternKind.`named`: of NamedPatternKind.`named`:
`named`*: Binding[E] `named`*: Binding
of NamedPatternKind.`anonymous`: of NamedPatternKind.`anonymous`:
`anonymous`*: Pattern[E] `anonymous`*: Pattern
SimplePatternKind* {.pure.} = enum SimplePatternKind* {.pure.} = enum
`any`, `atom`, `embedded`, `lit`, `seqof`, `setof`, `dictof`, `Ref` `any`, `atom`, `embedded`, `lit`, `seqof`, `setof`, `dictof`, `Ref`
SimplePatternAtom* {.preservesRecord: "atom".} = object SimplePatternAtom* {.preservesRecord: "atom".} = object
`atomKind`*: AtomKind `atomKind`*: AtomKind
SimplePatternEmbedded*[E] {.preservesRecord: "embedded".} = ref object SimplePatternEmbedded* {.preservesRecord: "embedded".} = ref object
`interface`*: SimplePattern[E] `interface`*: SimplePattern
SimplePatternLit*[E] {.preservesRecord: "lit".} = ref object SimplePatternLit* {.preservesRecord: "lit".} = object
`value`*: Preserve[E] `value`*: Preserve[void]
SimplePatternSeqof*[E] {.preservesRecord: "seqof".} = ref object SimplePatternSeqof* {.preservesRecord: "seqof".} = ref object
`pattern`*: SimplePattern[E] `pattern`*: SimplePattern
SimplePatternSetof*[E] {.preservesRecord: "setof".} = ref object SimplePatternSetof* {.preservesRecord: "setof".} = ref object
`pattern`*: SimplePattern[E] `pattern`*: SimplePattern
SimplePatternDictof*[E] {.preservesRecord: "dictof".} = ref object SimplePatternDictof* {.preservesRecord: "dictof".} = ref object
`key`*: SimplePattern[E] `key`*: SimplePattern
`value`*: SimplePattern[E] `value`*: SimplePattern
`SimplePattern`*[E] {.preservesOr.} = ref object `SimplePattern`* {.preservesOr.} = ref object
case orKind*: SimplePatternKind case orKind*: SimplePatternKind
of SimplePatternKind.`any`: of SimplePatternKind.`any`:
`any`* {.preservesLiteral: "any".}: bool `any`* {.preservesLiteral: "any".}: bool
@ -98,119 +99,116 @@ type
`atom`*: SimplePatternAtom `atom`*: SimplePatternAtom
of SimplePatternKind.`embedded`: of SimplePatternKind.`embedded`:
`embedded`*: SimplePatternEmbedded[E] `embedded`*: SimplePatternEmbedded
of SimplePatternKind.`lit`: of SimplePatternKind.`lit`:
`lit`*: SimplePatternLit[E] `lit`*: SimplePatternLit
of SimplePatternKind.`seqof`: of SimplePatternKind.`seqof`:
`seqof`*: SimplePatternSeqof[E] `seqof`*: SimplePatternSeqof
of SimplePatternKind.`setof`: of SimplePatternKind.`setof`:
`setof`*: SimplePatternSetof[E] `setof`*: SimplePatternSetof
of SimplePatternKind.`dictof`: of SimplePatternKind.`dictof`:
`dictof`*: SimplePatternDictof[E] `dictof`*: SimplePatternDictof
of SimplePatternKind.`Ref`: of SimplePatternKind.`Ref`:
`ref`*: Ref `ref`*: Ref
NamedSimplePatternKind* {.pure.} = enum NamedSimplePatternKind* {.pure.} = enum
`named`, `anonymous` `named`, `anonymous`
`NamedSimplePattern`*[E] {.preservesOr.} = ref object `NamedSimplePattern`* {.preservesOr.} = ref object
case orKind*: NamedSimplePatternKind case orKind*: NamedSimplePatternKind
of NamedSimplePatternKind.`named`: of NamedSimplePatternKind.`named`:
`named`*: Binding[E] `named`*: Binding
of NamedSimplePatternKind.`anonymous`: of NamedSimplePatternKind.`anonymous`:
`anonymous`*: SimplePattern[E] `anonymous`*: SimplePattern
DefinitionKind* {.pure.} = enum DefinitionKind* {.pure.} = enum
`or`, `and`, `Pattern` `or`, `and`, `Pattern`
DefinitionOrData*[E] {.preservesTuple.} = ref object DefinitionOrData* {.preservesTuple.} = ref object
`pattern0`*: NamedAlternative[E] `pattern0`*: NamedAlternative
`pattern1`*: NamedAlternative[E] `pattern1`*: NamedAlternative
`patternN`* {.preservesTupleTail.}: seq[NamedAlternative[E]] `patternN`* {.preservesTupleTail.}: seq[NamedAlternative]
DefinitionOr*[E] {.preservesRecord: "or".} = ref object DefinitionOr* {.preservesRecord: "or".} = ref object
`data`*: DefinitionOrData[E] `data`*: DefinitionOrData
DefinitionAndData*[E] {.preservesTuple.} = ref object DefinitionAndData* {.preservesTuple.} = ref object
`pattern0`*: NamedPattern[E] `pattern0`*: NamedPattern
`pattern1`*: NamedPattern[E] `pattern1`*: NamedPattern
`patternN`* {.preservesTupleTail.}: seq[NamedPattern[E]] `patternN`* {.preservesTupleTail.}: seq[NamedPattern]
DefinitionAnd*[E] {.preservesRecord: "and".} = ref object DefinitionAnd* {.preservesRecord: "and".} = ref object
`data`*: DefinitionAndData[E] `data`*: DefinitionAndData
`Definition`*[E] {.preservesOr.} = ref object `Definition`* {.preservesOr.} = ref object
case orKind*: DefinitionKind case orKind*: DefinitionKind
of DefinitionKind.`or`: of DefinitionKind.`or`:
`or`*: DefinitionOr[E] `or`*: DefinitionOr
of DefinitionKind.`and`: of DefinitionKind.`and`:
`and`*: DefinitionAnd[E] `and`*: DefinitionAnd
# doesn't build with C++
of DefinitionKind.`Pattern`: of DefinitionKind.`Pattern`:
`pattern`*: Pattern[E] `pattern`*: Pattern
NamedAlternative*[E] {.preservesTuple.} = ref object NamedAlternative* {.preservesTuple.} = ref object
`variantLabel`*: string `variantLabel`*: string
`pattern`*: Pattern[E] `pattern`*: Pattern
SchemaData*[E] {.preservesDictionary.} = ref object SchemaData* {.preservesDictionary.} = object
`version`* {.preservesLiteral: "1".}: bool `version`* {.preservesLiteral: "1".}: bool
`embeddedType`*: EmbeddedTypeName `embeddedType`*: EmbeddedTypeName
`definitions`*: Definitions[E] `definitions`*: Definitions
Schema*[E] {.preservesRecord: "schema".} = ref object Schema* {.preservesRecord: "schema".} = object
`data`*: SchemaData[E] `data`*: SchemaData
PatternKind* {.pure.} = enum PatternKind* {.pure.} = enum
`SimplePattern`, `CompoundPattern` `SimplePattern`, `CompoundPattern`
`Pattern`*[E] {.preservesOr.} = ref object `Pattern`* {.preservesOr.} = ref object
case orKind*: PatternKind case orKind*: PatternKind
of PatternKind.`SimplePattern`: of PatternKind.`SimplePattern`:
`simplepattern`*: SimplePattern[E] `simplepattern`*: SimplePattern
of PatternKind.`CompoundPattern`: of PatternKind.`CompoundPattern`:
`compoundpattern`*: CompoundPattern[E] `compoundpattern`*: CompoundPattern
Binding*[E] {.preservesRecord: "named".} = ref object Binding* {.preservesRecord: "named".} = ref object
`name`*: Symbol `name`*: Symbol
`pattern`*: SimplePattern[E] `pattern`*: SimplePattern
proc `$`*[E](x: Bundle[E] | CompoundPattern[E] | Modules[E] | Definitions[E] | proc `$`*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
DictionaryEntries[E] | EmbeddedTypeName |
NamedPattern[E] | Definitions |
SimplePattern[E] | DictionaryEntries |
NamedSimplePattern[E] | NamedPattern |
Definition[E] | SimplePattern |
NamedAlternative[E] | NamedSimplePattern |
Schema[E] | Definition |
Pattern[E] | NamedAlternative |
Binding[E]): string = Schema |
`$`(toPreserve(x, E)) Pattern |
Binding): string =
proc encode*[E](x: Bundle[E] | CompoundPattern[E] | Modules[E] | Definitions[E] |
DictionaryEntries[E] |
NamedPattern[E] |
SimplePattern[E] |
NamedSimplePattern[E] |
Definition[E] |
NamedAlternative[E] |
Schema[E] |
Pattern[E] |
Binding[E]): seq[byte] =
encode(toPreserve(x, E))
proc `$`*(x: Ref | ModulePath | EmbeddedTypeName): string =
`$`(toPreserve(x)) `$`(toPreserve(x))
proc encode*(x: Ref | ModulePath | EmbeddedTypeName): seq[byte] = proc encode*(x: Ref | ModulePath | Bundle | CompoundPattern | Modules |
EmbeddedTypeName |
Definitions |
DictionaryEntries |
NamedPattern |
SimplePattern |
NamedSimplePattern |
Definition |
NamedAlternative |
Schema |
Pattern |
Binding): seq[byte] =
encode(toPreserve(x)) encode(toPreserve(x))

4
src/preserves/schemac.nim
View File

@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: 2022 ☭ Emery Hemingway # SPDX-FileCopyrightText: 2022 ☭ Emery Hemingway
# SPDX-License-Identifier: Unlicense # SPDX-License-Identifier: Unlicense
import std/[hashes, options, os, parseopt, streams, strutils, sets, tables] import std/[hashes, options, os, parseopt, streams, strutils, tables]
import ../preserves, ./schema, ./schemaparse import ../preserves, ./schema, ./schemaparse
@ -23,7 +23,7 @@ when isMainModule:
write(outStream, schema.toPreserve) write(outStream, schema.toPreserve)
of "bundle": of "bundle":
var bundle: Bundle[void] var bundle: Bundle
if not dirExists inputPath: if not dirExists inputPath:
quit "not a directory of schemas: " & inputPath quit "not a directory of schemas: " & inputPath
else: else:

18
src/preserves/schemaparse.nim
View File

@ -10,9 +10,6 @@ import ../preserves, ./schema, ./pegs
type type
Value = Preserve[void] Value = Preserve[void]
Definition = schema.Definition[void]
SchemaData = schema.SchemaData[void]
Schema = schema.Schema[void]
Stack = seq[tuple[node: Value, pos: int]] Stack = seq[tuple[node: Value, pos: int]]
ParseState = object ParseState = object
schema: SchemaData schema: SchemaData
@ -60,7 +57,7 @@ template pushStack(n: Value) =
assert(p.stack.len > 0, capture[0].s) assert(p.stack.len > 0, capture[0].s)
proc toSymbolLit(s: string): Value = proc toSymbolLit(s: string): Value =
initRecord(toSymbol"lit", toSymbol s) initRecord[void](toSymbol"lit", toSymbol s)
proc match(text: string; p: var ParseState) {.gcsafe.} proc match(text: string; p: var ParseState) {.gcsafe.}
@ -205,10 +202,10 @@ const parser = peg("Schema", p: ParseState):
pushStack n pushStack n
Ref <- >(Alpha * *Alnum) * *('.' * >(*Alnum)): Ref <- >(Alpha * *Alnum) * *('.' * >(*Alnum)):
var path: seq[string] var path = initSequence[void]()
for i in 1..<capture.len: path.add capture[i].s for i in 1..<capture.len: path.sequence.add(toSymbol capture[i].s)
let name = pop path var name = pop(path.sequence)
var n = initRecord(toSymbol"ref", toPreserve path, toSymbol name) var n = initRecord(toSymbol"ref", path, name)
pushStack n pushStack n
CompoundPattern <- CompoundPattern <-
@ -291,6 +288,7 @@ proc parsePreservesSchema*(text: string; directory = getCurrentDir()): Schema =
## ##
## Schemas in binary encoding should instead be parsed as Preserves ## Schemas in binary encoding should instead be parsed as Preserves
## and converted to `Schema` with `fromPreserve` or `preserveTo`. ## and converted to `Schema` with `fromPreserve` or `preserveTo`.
assert directory != ""
var p = ParseState(schema: SchemaData(), directory: directory) var p = ParseState(schema: SchemaData(), directory: directory)
match(text, p) match(text, p)
Schema(data: p.schema) Schema(data: p.schema)
@ -300,6 +298,6 @@ when isMainModule:
let txt = readAll stdin let txt = readAll stdin
if txt != "": if txt != "":
let let
scm = parsePreservesSchema(readAll stdin) scm = parsePreservesSchema(txt)
pr = toPreserve scm pr = toPreserve scm
stdout.newFileStream.write pr stdout.newFileStream.writeText pr

40
tests/test_schemas.nim
View File

@ -1,40 +0,0 @@
# SPDX-FileCopyrightText: 2021 ☭ Emery Hemingway
# SPDX-License-Identifier: Unlicense
import std/[tables, options, os, unittest]
import preserves, preserves/schema, preserves/schemaparse
if dirExists "tests": setCurrentDir "tests"
# nimble runs tests in the directory above
suite "schema":
const
binPath = "../schema.bin"
textPath = "../schema.prs"
test "convertability":
if not fileExists(binPath): skip()
else:
var
b = decodePreserves readFile(binPath)
scm = preserveTo(b, Schema[void])
check scm.isSome
if scm.isSome:
var a = toPreserve(get scm)
check(a == b)
test "parser":
if not fileExists(binPath): skip()
else:
var
b = decodePreserves readFile(binPath)
scm = preserveTo(b, Schema[void])
check scm.isSome
if scm.isSome:
var a = toPreserve parsePreservesSchema(readFile textPath, textPath)
let aDefs = a[0][toSymbol "definitions"]
let bDefs = b[0][toSymbol "definitions"]
for (key, val) in aDefs.pairs:
check(bDefs[key] == val)
check(a == b)