Replace Nimble with an SBOM

Tupfile

@@ -1,2 +1,2 @@
 include_rules
-: lock.json |> !nim_cfg |> | ./<lock>
+: sbom.json |> !sbom-to-nix |> | ./<lock>

default.nix

@@ -0,0 +1,26 @@
+{
+  pkgs ? import <nixpkgs> { },
+}:
+let
+  inherit (pkgs) lib buildNimPackage nim;
+in
+buildNimPackage {
+  pname = "preserves-nim";
+  version = "unstable";
+
+  lockFile = ./lock.json;
+
+  src = if lib.inNixShell then null else lib.cleanSource ./.;
+
+  nimFlags = [ "--path:${nim.passthru.nim}/nim" ];
+  # Path to the compiler/ast library.
+
+  postInstall = ''
+    pushd $out/bin
+    for link in preserves_decode preserves_from_json preserves_to_json;
+      do ln -s  preserves_encode $link
+    done
+    mv preserves_schemac preserves-schemac
+    popd
+  '';
+}

preserves.nimble

@@ -1,14 +1,61 @@
-# Package
+# Emulate Nimble from CycloneDX data at sbom.json.
 
-version = "20240208"
-author        = "Emery Hemingway"
-description   = "data model and serialization format"
-license       = "Unlicense"
-srcDir        = "src"
+import std/json
 
-bin           = @["preserves/preserves_schema_nim", "preserves/private/preserves_encode", "preserves/schemac"]
+proc lookupComponent(sbom: JsonNode; bomRef: string): JsonNode =
+  for c in sbom{"components"}.getElems.items:
+    if c{"bom-ref"}.getStr == bomRef:
+      return c
+  result = newJNull()
 
+let
+  sbom = "sbom.json".readFile.parseJson
+  comp = sbom{"metadata", "component"}
+  bomRef = comp{"bom-ref"}.getStr
 
-# Dependencies
+version = comp{"version"}.getStr
+author = comp{"authors"}[0]{"name"}.getStr
+description = comp{"description"}.getStr
+license = comp{"licenses"}[0]{"license", "id"}.getStr
 
-requires "nim >= 2.0.0", "compiler >= 2.0.0", "https://github.com/zevv/npeg.git >= 1.2.1", "https://github.com/ehmry/nim-bigints.git >= 20231006"
+for prop in comp{"properties"}.getElems.items:
+  let (key, val) = (prop{"name"}.getStr, prop{"value"}.getStr)
+  case key
+  of "nim:skipDirs:":
+    add(skipDirs, val)
+  of "nim:skipFiles:":
+    add(skipFiles, val)
+  of "nim:skipExt":
+    add(skipExt, val)
+  of "nim:installDirs":
+    add(installDirs, val)
+  of "nim:installFiles":
+    add(installFiles, val)
+  of "nim:installExt":
+    add(installExt, val)
+  of "nim:binDir":
+    add(binDir, val)
+  of "nim:srcDir":
+    add(srcDir, val)
+  of "nim:backend":
+    add(backend, val)
+  else:
+    if key.startsWith "nim:bin:":
+      namedBin[key[8..key.high]] = val
+
+for depend in sbom{"dependencies"}.items:
+  if depend{"ref"}.getStr == bomRef:
+    for depRef in depend{"dependsOn"}.items:
+      let dep = sbom.lookupComponent(depRef.getStr)
+      var spec = dep{"name"}.getStr
+      for extRef in dep{"externalReferences"}.elems:
+        if extRef{"type"}.getStr == "vcs":
+          spec = extRef{"url"}.getStr
+          break
+      let ver = dep{"version"}.getStr
+      if ver != "":
+        if ver.allCharsInSet {'0'..'9', '.'}: spec.add " == "
+        else: spec.add '#'
+        spec.add ver
+      requires spec
+    break

sbom.json

@@ -0,0 +1,162 @@
+{
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.6",
+  "metadata": {
+    "component": {
+      "type": "application",
+      "bom-ref": "pkg:nim/preserves",
+      "name": "preserves",
+      "description": "data model and serialization format",
+      "version": "20240523",
+      "authors": [
+        {
+          "name": "Emery Hemingway"
+        }
+      ],
+      "licenses": [
+        {
+          "license": {
+            "id": "Unlicense"
+          }
+        }
+      ],
+      "properties": [
+        {
+          "name": "nim:skipExt",
+          "value": "nim"
+        },
+        {
+          "name": "nim:bin:preserves/private/preserves_encode",
+          "value": "preserves/private/preserves_encode"
+        },
+        {
+          "name": "nim:bin:preserves/preserves_schema_nim",
+          "value": "preserves/preserves_schema_nim"
+        },
+        {
+          "name": "nim:bin:preserves/preserves_schemac",
+          "value": "preserves/preserves_schemac"
+        },
+        {
+          "name": "nim:srcDir",
+          "value": "src"
+        },
+        {
+          "name": "nim:backend",
+          "value": "c"
+        }
+      ]
+    }
+  },
+  "components": [
+    {
+      "type": "library",
+      "bom-ref": "pkg:nim/npeg",
+      "name": "npeg",
+      "version": "1.2.2",
+      "externalReferences": [
+        {
+          "url": "https://github.com/zevv/npeg/archive/ec0cc6e64ea4c62d2aa382b176a4838474238f8d.tar.gz",
+          "type": "source-distribution"
+        },
+        {
+          "url": "https://github.com/zevv/npeg.git",
+          "type": "vcs"
+        }
+      ],
+      "properties": [
+        {
+          "name": "nix:fod:method",
+          "value": "fetchzip"
+        },
+        {
+          "name": "nix:fod:path",
+          "value": "/nix/store/xpn694ibgipj8xak3j4bky6b3k0vp7hh-source"
+        },
+        {
+          "name": "nix:fod:rev",
+          "value": "ec0cc6e64ea4c62d2aa382b176a4838474238f8d"
+        },
+        {
+          "name": "nix:fod:sha256",
+          "value": "1fi9ls3xl20bmv1ikillxywl96i9al6zmmxrbffx448gbrxs86kg"
+        },
+        {
+          "name": "nix:fod:url",
+          "value": "https://github.com/zevv/npeg/archive/ec0cc6e64ea4c62d2aa382b176a4838474238f8d.tar.gz"
+        },
+        {
+          "name": "nix:fod:ref",
+          "value": "1.2.2"
+        },
+        {
+          "name": "nix:fod:srcDir",
+          "value": "src"
+        }
+      ]
+    },
+    {
+      "type": "library",
+      "bom-ref": "pkg:nim/bigints",
+      "name": "bigints",
+      "version": "20231006",
+      "externalReferences": [
+        {
+          "url": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz",
+          "type": "source-distribution"
+        },
+        {
+          "url": "https://github.com/ehmry/nim-bigints.git",
+          "type": "vcs"
+        }
+      ],
+      "properties": [
+        {
+          "name": "nix:fod:method",
+          "value": "fetchzip"
+        },
+        {
+          "name": "nix:fod:path",
+          "value": "/nix/store/jvrm392g8adfsgf36prgwkbyd7vh5jsw-source"
+        },
+        {
+          "name": "nix:fod:rev",
+          "value": "86ea14d31eea9275e1408ca34e6bfe9c99989a96"
+        },
+        {
+          "name": "nix:fod:sha256",
+          "value": "15pcpmnk1bnw3k8769rjzcpg00nahyrypwbxs88jnwr4aczp99j4"
+        },
+        {
+          "name": "nix:fod:url",
+          "value": "https://github.com/ehmry/nim-bigints/archive/86ea14d31eea9275e1408ca34e6bfe9c99989a96.tar.gz"
+        },
+        {
+          "name": "nix:fod:ref",
+          "value": "20231006"
+        },
+        {
+          "name": "nix:fod:srcDir",
+          "value": "src"
+        }
+      ]
+    }
+  ],
+  "dependencies": [
+    {
+      "ref": "pkg:nim/preserves",
+      "dependsOn": [
+        "pkg:nim/npeg",
+        "pkg:nim/bigints"
+      ]
+    },
+    {
+      "ref": "pkg:nim/npeg",
+      "dependsOn": []
+    },
+    {
+      "ref": "pkg:nim/bigints",
+      "dependsOn": []
+    }
+  ]
+}

src/preserves.nim

@@ -1,13 +1,13 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[options, sets, sequtils, strutils, tables, typetraits]
+import std/[assertions, options, sets, sequtils, strutils, tables, typetraits]
 from std/algorithm import sort
 from std/json import escapeJson, escapeJsonUnquoted
 import bigints
-import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values]
+import ./preserves/private/[buffering, encoding, decoding, dot, macros, parsing, texts, values]
 
-export encoding, decoding, parsing, texts, values
+export buffering, encoding, decoding, parsing, texts, values
 
 when defined(tracePreserves):
   when defined(posix):
@@ -335,6 +335,9 @@ template unpreservable*() {.pragma.}
   ## as its native type.
   ## Unpreservability is asserted at runtime.
 
+converter preserve*(i: SomeInteger): Value =
+  Value(kind: pkRegister, register: BiggestInt i)
+
 proc toPreserves*[T](x: T): Value {.gcsafe.} =
   ## Serializes ``x`` to Preserves. Can be customized by defining
   ## ``toPreservesHook(x: T; E: typedesc)`` in the calling scope.
@@ -551,7 +554,7 @@ proc fromAtom*[T](v: var T; a: ATom): bool =
   elif T is distinct:
     result = fromAtom(v.distinctBase, a)
 
-proc fromPreserves*[T](v: var T; pr: Value): bool {.gcsafe.} =
+proc fromPreserves*[T](v: var T; pr: Value): bool =
   ## Inplace version of `preservesTo`. Returns ``true`` on
   ## a complete match, otherwise returns ``false``.
   ## Can be customized with `fromPreservesHook(x: T; var pr: Value): bool`.
@@ -565,7 +568,7 @@ proc fromPreserves*[T](v: var T; pr: Value): bool {.gcsafe.} =
     type Foo {.preservesRecord: "foo".} = object
       x, y: int
     var foo: Foo
-    assert(fromPreserve(foo, parsePreserves("""<foo 1 2>""")))
+    assert(fromPreserves(foo, parsePreserves("""<foo 1 2>""")))
     assert(foo.x == 1)
     assert(foo.y == 2)
   when T is Value:
@@ -843,9 +846,9 @@ func step*(pr: Value; path: varargs[Value, toPreserves]): Option[Value] =
   ## Works for sequences, records, and dictionaries.
   runnableExamples:
     import std/options
-    assert step(parsePreserves("""<foo 1 2>"""), 1.toPreserve) == some(2.toPreserve)
-    assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserve)
-    assert step(parsePreserves("""[ ]"""), 1.toPreserve) == none(Value)
+    assert step(parsePreserves("""<foo 1 2>"""), 1.toPreserves) == some(2.toPreserves)
+    assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserves)
+    assert step(parsePreserves("""[ ]"""), 1.toPreserves) == none(Value)
   result = some(pr)
   for index in path:
     if result.isSome:
@@ -878,7 +881,7 @@ proc apply*(result: var Value; op: proc(_: var Value) {.gcsafe.}) {.gcsafe.} =
       recurse(e.val)
   cannonicalize(result)
 
-proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value {.gcsafe.}): Value {.gcsafe.} =
+proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value): Value =
   ## Process all embeds in a `Value`.
   case pr.kind
   of pkBoolean, pkFloat, pkRegister, pkBigInt,

src/preserves/expressions.nim

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import
+  npeg,
+  ../preserves, ./pegs
+
+type
+  Frame = tuple[value: Value, pos: int]
+  Stack = seq[Frame]
+
+proc shrink(stack: var Stack; n: int) = stack.setLen(stack.len - n)
+
+template pushStack(v: Value) = stack.add((v, capture[0].si))
+
+template collectEntries(result: var seq[Value]; stack: var Stack) =
+  for frame in stack.mitems:
+    if frame.pos > capture[0].si:
+      result.add frame.value.move
+  stack.shrink result.len
+
+proc parseExpressions*(text: string): seq[Value] =
+  let parser = peg("Document", stack: Stack):
+
+    ws <- *{ ' ', '\t', '\r', '\n' }
+
+    Document <- *Expr * ws * !1
+
+    Annotation <-
+      ('@' * SimpleExpr) |
+      ('#' * {'\x20', '\x09', '\x21'} * @{'\r','\n'})
+
+    Trailer <- *(ws * Annotation)
+
+    Expr <- ws * (Punct | SimpleExpr) * Trailer
+
+    Punct <- {',', ';'} | +':':
+      pushStack initRecord("p", toSymbol $0)
+
+    SimpleExpr <-
+        Atom |
+        Compound |
+        Embedded |
+        Annotated
+
+    Embedded <- "#:" * SimpleExpr:
+      pushstack stack.pop.value.embed
+
+    Annotated <- Annotation * SimpleExpr
+
+    Compound <- Sequence | Record | Block | Group | Set
+
+    Sequence <-  '[' * *Expr * ws * ']':
+      var pr = Value(kind: pkSequence)
+      collectEntries(pr.sequence, stack)
+      pushStack pr
+
+    Record <- '<' * *Expr * ws * '>':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"r"
+      pushStack pr
+
+    Block <- '{' * *Expr * ws * '}':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"b"
+      pushStack pr
+
+    Group <- '(' * *Expr * ws * ')':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"g"
+      pushStack pr
+
+    Set <- "#{" * *Expr * ws * '}':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"s"
+      pushStack pr
+
+    Atom <- Preserves.Atom:
+      pushStack parsePreserves($0)
+
+  var stack: Stack
+  let match = parser.match(text, stack)
+  if not match.ok:
+    raise newException(ValueError, "failed to parse Preserves Expressions:\n" & text[match.matchMax..text.high])
+
+  result.setLen stack.len
+  for i, _ in result:
+    result[i] = move stack[i].value

src/preserves/pegs.nim

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Unlicense
 
 ## NPEG rules for Preserves.
+## For an explanation of the syntax see https://github.com/zevv/npeg/blob/master/README.md#syntax.
 
 import npeg, npeg/lib/utf8
 
@@ -11,7 +12,11 @@ grammar "Preserves":
 
   ws <- *{ ' ', '\t', '\r', '\n' }
   commas <- *(ws * ',') * ws
-  delimiter <- { ' ', '\t', '\r', '\n', '<', '>', '[', ']', '{', '}', '#', ':', '"', '|', '@', ';', ',' } | !1
+  delimiter <- {
+      ' ', '\t', '\r', '\n',
+      '<', '>', '[', ']', '{', '}', '(', ')',
+      '#', ':', '"', '|', '@', ';', ','
+    } | !1
 
   Document <- Value * ws * !1
 
@@ -19,10 +24,10 @@ grammar "Preserves":
 
   Collection <- Sequence | Dictionary | Set
 
-  Value <-
-      (ws * (Record | Collection | Atom | Embedded | Compact)) |
-      (ws * Annotation) |
-      (ws * '#' * @'\n' * Value)
+  Value <- ws * (
+      Record | Collection | Atom | Embedded | Compact |
+      Annotation |
+      ('#' * @'\n' * Value) )
 
   Record <- '<' * +Value * ws * '>'
 

src/preserves/private/buffering.nim

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[assertions, endians, options, streams, strutils]
+import bigints
+import ./decoding, ./parsing, ./values
+
+type BufferedDecoder* = object
+  ## Type for buffering binary Preserves before decoding.
+  stream: StringStream
+  appendPosition, decodePosition, maxSize: int
+
+proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
+  ## Create a new `newBufferedDecoder`.
+  runnableExamples:
+    var
+      buf = newBufferedDecoder()
+      bin = encode(parsePreserves("<foobar>"))
+    buf.feed(bin[0..2])
+    buf.feed(bin[3..bin.high])
+    var (success, pr) = decode(buf)
+    assert success
+    assert $pr == "<foobar>"
+  BufferedDecoder(
+      stream: newStringStream(newStringOfCap(maxSize)),
+      maxSize: maxSize,
+    )
+
+proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
+  assert len > 0
+  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
+    raise newException(IOError, "BufferedDecoder at maximum buffer size")
+  dec.stream.setPosition(dec.appendPosition)
+  dec.stream.writeData(buf, len)
+  inc(dec.appendPosition, len)
+  assert dec.appendPosition == dec.stream.getPosition()
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
+  if data.len > 0:
+    dec.feed(addr data[0], data.len)
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]; slice: Slice[int]) =
+  let n = slice.b + 1 - slice.a
+  if n > 0:
+    dec.feed(addr data[slice.a], n)
+
+proc decode*(dec: var BufferedDecoder): Option[Value] =
+  ## Decode from `dec`. If decoding fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.decodePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError:
+      discard
+
+proc parse*(dec: var BufferedDecoder): Option[Value] =
+  ## Parse from `dec`. If parsing fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.readAll.parsePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError, ValueError:
+      discard

src/preserves/private/decoding.nim

@@ -15,14 +15,18 @@ proc readVarint(s: Stream): uint =
     c = uint s.readUint8
   result = result or (c shl shift)
 
-proc decodePreserves*(s: Stream): Value =
+proc decodePreserves*(s: Stream): Value {.gcsafe.}
+
+proc decodePreserves(s: Stream; tag: uint8): Value =
   ## Decode a Preserves value from a binary-encoded stream.
-  if s.atEnd: raise newException(IOError, "End of Preserves stream")
   const endMarker = 0x84
-  let tag = s.readUint8()
   case tag
-  of 0x80: result = Value(kind: pkBoolean, bool: false)
-  of 0x81: result = Value(kind: pkBoolean, bool: true)
+  of 0x80: return Value(kind: pkBoolean, bool: false)
+  of 0x81: return Value(kind: pkBoolean, bool: true)
+  else: discard
+  if s.atEnd:
+    raise newException(IOError, "End of Preserves stream")
+  case tag
   of 0x85:
     discard decodePreserves(s)
     result = decodePreserves(s)
@@ -99,30 +103,38 @@ proc decodePreserves*(s: Stream): Value =
   of 0xb4:
     result = Value(kind: pkRecord)
     var label = decodePreserves(s)
-    while s.peekUint8() != endMarker:
-      result.record.add decodePreserves(s)
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result.record.add decodePreserves(s, tag)
+      tag = s.readUint8()
     result.record.add(move label)
-    discard s.readUint8()
   of 0xb5:
     result = Value(kind: pkSequence)
-    while s.peekUint8() != endMarker:
-      result.sequence.add decodePreserves(s)
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result.sequence.add decodePreserves(s, tag)
+      tag = s.readUint8()
   of 0xb6:
     result = Value(kind: pkSet)
-    while s.peekUint8() != endMarker:
-      incl(result, decodePreserves(s))
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      incl(result, decodePreserves(s, tag))
+      tag = s.readUint8()
   of 0xb7:
     result = Value(kind: pkDictionary)
-    while s.peekUint8() != endMarker:
-      result[decodePreserves(s)] = decodePreserves(s)
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result[decodePreserves(s, tag)] = decodePreserves(s)
+      tag = s.readUint8()
   of endMarker:
     raise newException(ValueError, "invalid Preserves stream")
   else:
     raise newException(ValueError, "invalid Preserves tag byte 0x" & tag.toHex(2))
 
+proc decodePreserves*(s: Stream): Value {.gcsafe.} =
+  ## Decode a Preserves value from a binary-encoded stream.
+  s.decodePreserves s.readUint8()
+
 proc decodePreserves*(s: string): Value =
   ## Decode a string of binary-encoded Preserves.
   decodePreserves(s.newStringStream)
@@ -130,54 +142,3 @@ proc decodePreserves*(s: string): Value =
 proc decodePreserves*(s: seq[byte]): Value =
   ## Decode a byte-string of binary-encoded Preserves.
   decodePreserves(cast[string](s))
-
-type BufferedDecoder* = object
-  ## Type for buffering binary Preserves before decoding.
-  stream: StringStream
-  appendPosition, decodePosition, maxSize: int
-
-proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
-  ## Create a new `newBufferedDecoder`.
-  runnableExamples:
-    var
-      buf = newBufferedDecoder()
-      bin = encode(parsePreserves("<foobar>"))
-    buf.feed(bin[0..2])
-    buf.feed(bin[3..bin.high])
-    var (success, pr) = decode(buf)
-    assert success
-    assert $pr == "<foobar>"
-  BufferedDecoder(
-      stream: newStringStream(newStringOfCap(maxSize)),
-      maxSize: maxSize,
-    )
-
-proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
-  assert len > 0
-  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
-    raise newException(IOError, "BufferedDecoder at maximum buffer size")
-  dec.stream.setPosition(dec.appendPosition)
-  dec.stream.writeData(buf, len)
-  inc(dec.appendPosition, len)
-  assert dec.appendPosition == dec.stream.getPosition()
-
-proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
-  if data.len > 0:
-    dec.feed(unsafeAddr data[0], data.len)
-
-proc decode*(dec: var BufferedDecoder): Option[Value] =
-  ## Decode from `dec`. If decoding fails the internal position of the
-  ## decoder does not advance.
-  if dec.appendPosition > 0:
-    assert(dec.decodePosition < dec.appendPosition)
-    dec.stream.setPosition(dec.decodePosition)
-    try:
-      result = dec.stream.decodePreserves.some
-      dec.decodePosition = dec.stream.getPosition()
-      if dec.decodePosition == dec.appendPosition:
-        dec.stream.setPosition(0)
-        dec.stream.data.setLen(0)
-        dec.appendPosition = 0
-        dec.decodePosition = 0
-    except IOError:
-      discard

src/preserves/private/encoding.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[endians, streams]
+import std/[algorithm, assertions, endians, streams]
 import bigints
 import ./values
 
@@ -96,10 +96,22 @@ proc write*(str: Stream; pr: Value) =
       str.write(val)
     str.write(0x84'u8)
   of pkDictionary:
+    var
+      keyIndices = newSeqOfCap[(string, int)](pr.dict.len)
+      keyBuffer = newStringStream()
+    for i in 0..pr.dict.high:
+      keyBuffer.write(pr.dict[i][0])
+      keyIndices.add((keyBuffer.data.move, i))
+      keyBuffer.setPosition(0)
+        # add each encoded key and its index to the seq
+    sort(keyIndices) do (a, b: (string, int)) -> int:
+      cmp(a[0], b[0])
+        # sort the seq by encoded keys
     str.write(0xb7'u8)
-    for (key, value) in pr.dict.items:
-      str.write(key)
-      str.write(value)
+    for (keyBytes, i) in keyIndices:
+      str.write(keyBytes)
+      str.write(pr.dict[i][1])
+        # encode the values in sorted key order
     str.write(0x84'u8)
   of pkEmbedded:
     # str.write(0x86'u8)

src/preserves/private/macros.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import std/macros
+import std/[assertions, macros]
 
 const
   nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit}

src/preserves/private/parsing.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, options, parseutils, strutils, unicode]
+import std/[assertions, base64, options, parseutils, strutils, unicode]
 from std/sequtils import insert
 
 import bigints, npeg

src/preserves/private/texts.nim

@@ -1,7 +1,9 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, endians, math, sequtils, streams, strutils]
+import std/[assertions, base64, endians, sequtils, streams, strutils]
+when not defined(nimNoLibc):
+  import std/math
 import bigints
 import ./values
 
@@ -42,6 +44,15 @@ proc writeSymbol(stream: Stream; sym: string) =
     writeEscaped(stream, sym, '|')
     write(stream, '|')
 
+proc writeFloatBytes(stream: Stream; f: float) =
+  var buf: array[8, byte]
+  bigEndian64(addr buf[0], addr f)
+  write(stream, "#xd\"")
+  for b in buf:
+    write(stream, hexAlphabet[b shr 4])
+    write(stream, hexAlphabet[b and 0xf])
+  write(stream, '"')
+
 proc writeText*(stream: Stream; pr: Value; mode = textPreserves) =
   ## Encode Preserves to a `Stream` as text.
   if pr.embedded: write(stream, "#:")
@@ -51,17 +62,14 @@ proc writeText*(stream: Stream; pr: Value; mode = textPreserves) =
     of false: write(stream, "#f")
     of true: write(stream, "#t")
   of pkFloat:
-    case pr.float.classify:
-    of fcNormal, fcZero, fcNegZero:
-      write(stream, $pr.float)
+    when defined(nimNoLibc):
+      writeFloatBytes(stream, pr.float)
+        # IEE754-to-decimal is non-trivial
     else:
-      var buf: array[8, byte]
-      bigEndian64(addr buf[0], addr pr.float)
-      write(stream, "#xd\"")
-      for b in buf:
-        write(stream, hexAlphabet[b shr 4])
-        write(stream, hexAlphabet[b and 0xf])
-      write(stream, '"')
+      if pr.float.classify in {fcNormal, fcZero, fcNegZero}:
+        write(stream, $pr.float)
+      else:
+        writeFloatBytes(stream, pr.float)
   of pkRegister:
     write(stream, $pr.register)
   of pkBigInt:

src/preserves/private/values.nim

@@ -1,8 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[algorithm, hashes, math, options, sets, sequtils, tables]
-
+import std/[algorithm, hashes, options, sets, sequtils, tables]
 import bigints
 
 type
@@ -80,11 +79,6 @@ type
     ## Object refs embedded in Preserves `Value`s must inherit from `EmbeddedObj`.
     ## At the moment this is just an alias to `RootObj` but this may change in the future.
 
-func `===`[T: SomeFloat](a, b: T): bool =
-  ## Compare where Nan == NaN.
-  let class = a.classify
-  (class == b.classify) and ((class notin {fcNormal,fcSubnormal}) or (a == b))
-
 func `==`*(x, y: Value): bool =
   ## Check `x` and `y` for equivalence.
   if x.kind == y.kind and x.embedded == y.embedded:
@@ -92,7 +86,7 @@ func `==`*(x, y: Value): bool =
     of pkBoolean:
       result = x.bool == y.bool
     of pkFloat:
-      result = x.float === y.float
+      result = cast[uint64](x.float) == cast[uint64](y.float)
     of pkRegister:
       result = x.register == y.register
     of pkBigInt:
@@ -255,10 +249,13 @@ proc `[]=`*(pr: var Value; key, val: Value) =
 
 proc incl*(pr: var Value; key: Value) =
   ## Include `key` in the Preserves set `pr`.
+  # TODO: binary search
   for i in 0..pr.set.high:
     if key < pr.set[i]:
       insert(pr.set, [key], i)
       return
+    elif key == pr.set[i]:
+      return
   pr.set.add(key)
 
 proc excl*(pr: var Value; key: Value) =

src/preserves/schema.nim

@@ -1,6 +1,7 @@
 
 import
-  preserves, std/tables
+  std/tables,
+  ../preserves
 
 type
   Ref* {.preservesRecord: "ref".} = object

src/preserves/schemaparse.nim

@@ -86,7 +86,7 @@ const parser = peg("Schema", p: ParseState):
     match(readFile path, state)
     p.schema = move state.schema
 
-  Definition <- ?Annotation * *LineComment * id * '=' * S * (OrPattern | AndPattern | Pattern):
+  Definition <- *LineComment * ?Annotation * id * '=' * S * (OrPattern | AndPattern | Pattern):
     if p.schema.definitions.hasKey(Symbol $1):
       raise newException(ValueError, "duplicate definition of " & $0)
     var
@@ -254,7 +254,7 @@ const parser = peg("Schema", p: ParseState):
     var node = initRecord(toSymbol"tuplePrefix", toPreserves fields, tail)
     pushStack node
 
-  DictionaryPattern <- '{' * *(S * >Value * S * ':' * S * NamedSimplePattern * ?',') * S * '}':
+  DictionaryPattern <- '{' * S * *(*LineComment * >Value * S * ':' * S * NamedSimplePattern * ?',' * S) * '}':
     var dict = initDictionary()
     for i in countDown(pred capture.len, 1):
       let key = toSymbol capture[i].s
@@ -263,12 +263,12 @@ const parser = peg("Schema", p: ParseState):
     pushStack n
 
   NamedPattern <- ((atId * ?Annotation * SimplePattern) | Pattern):
-    if capture.len == 2:
+    if capture.len > 1:
       var n = initRecord(toSymbol"named", toSymbol $1, popStack())
       pushStack n
 
   NamedSimplePattern <- ((atId * ?Annotation * SimplePattern) | SimplePattern):
-    if capture.len == 2:
+    if capture.len > 1:
       var n = initRecord(toSymbol"named", toSymbol $1, popStack())
       pushStack n
 
@@ -287,7 +287,8 @@ const parser = peg("Schema", p: ParseState):
   Value <- Preserves.Value:
     discard
 
-  Annotation <- '@' * (Preserves.String | Preserves.Record) * S
+  Annotation <- '@' * (Preserves.String | Preserves.Record) * S:
+    discard
 
   S <- *{ ' ', '\t', '\r', '\n' }
 

src/preserves/sugar.nim

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import ../preserves, ./private/macros
+
+proc `%`*(v: bool|SomeFloat|SomeInteger|string|seq[byte]|Symbol): Value {.inline.} = v.toPreserves
+  # Preserve an atomic Nim value.

tests/test_p_exprs.nim

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import
+  std/unittest,
+  preserves, preserves/expressions
+
+template testExpr(name, code, cntrl: string) {.dirty.} =
+  test name:
+    checkpoint code
+    let
+      pr = parsePreserves cntrl
+      exprs = parseExpressions code
+    checkpoint $(exprs.toPreserves)
+    check exprs.len == 1
+    let px = exprs[0]
+    check px == pr
+
+suite "expression":
+
+  testExpr "date", """
+      <date 1821 (lookup-month "February") 3>
+    """, """
+      <r date 1821 <g lookup-month "February"> 3>
+    """
+
+  testExpr "r", "<>", "<r>"
+
+  testExpr "begin",
+    """(begin (println! (+ 1 2)) (+ 3 4))""",
+    """<g begin <g println! <g + 1 2>> <g + 3 4>>"""
+
+  testExpr "g",
+    """()""", """<g>"""
+
+  testExpr "groups",
+    """[() () ()]""", """[<g>, <g>, <g>]"""
+
+  testExpr "loop", """
+      {
+        setUp();
+        # Now enter the loop
+        loop: {
+            greet("World");
+        }
+        tearDown();
+      }
+    """,  """
+      <b
+      setUp <g> <p |;|>
+      # Now enter the loop
+      loop <p |:|> <b
+          greet <g "World"> <p |;|>
+      >
+      tearDown <g> <p |;|>
+    >
+  """
+
+  testExpr "+", """
+      [1 + 2.0, print "Hello", predicate: #t, foo, #:remote, bar]
+    """, """
+      [1 + 2.0 <p |,|> print "Hello" <p |,|> predicate <p |:|> #t <p |,|>
+   foo <p |,|> #:remote <p |,|> bar]
+   """
+
+  testExpr "set",
+    """#{1 2 3}""",  """<s 1 2 3>"""
+
+  testExpr "group-set",
+    """#{(read) (read) (read)}""",
+    """<s <g read> <g read> <g read>>"""
+
+  testExpr "block", """
+      {
+        optional name: string,
+        address: Address,
+      }
+    """, """
+      <b
+        optional name <p |:|> string <p |,|>
+        address <p |:|> Address <p |,|>
+      >
+    """