Fix set inclusion

preserves.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version = "20240208"
+version = "20240506"
 author        = "Emery Hemingway"
 description   = "data model and serialization format"
 license       = "Unlicense"

src/preserves.nim

@@ -1,13 +1,13 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[options, sets, sequtils, strutils, tables, typetraits]
+import std/[assertions, options, sets, sequtils, strutils, tables, typetraits]
 from std/algorithm import sort
 from std/json import escapeJson, escapeJsonUnquoted
 import bigints
-import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values]
+import ./preserves/private/[buffering, encoding, decoding, dot, macros, parsing, texts, values]
 
-export encoding, decoding, parsing, texts, values
+export buffering, encoding, decoding, parsing, texts, values
 
 when defined(tracePreserves):
   when defined(posix):
@@ -335,6 +335,9 @@ template unpreservable*() {.pragma.}
   ## as its native type.
   ## Unpreservability is asserted at runtime.
 
+converter preserve*(i: SomeInteger): Value =
+  Value(kind: pkRegister, register: BiggestInt i)
+
 proc toPreserves*[T](x: T): Value {.gcsafe.} =
   ## Serializes ``x`` to Preserves. Can be customized by defining
   ## ``toPreservesHook(x: T; E: typedesc)`` in the calling scope.
@@ -551,7 +554,7 @@ proc fromAtom*[T](v: var T; a: ATom): bool =
   elif T is distinct:
     result = fromAtom(v.distinctBase, a)
 
-proc fromPreserves*[T](v: var T; pr: Value): bool {.gcsafe.} =
+proc fromPreserves*[T](v: var T; pr: Value): bool =
   ## Inplace version of `preservesTo`. Returns ``true`` on
   ## a complete match, otherwise returns ``false``.
   ## Can be customized with `fromPreservesHook(x: T; var pr: Value): bool`.
@@ -565,7 +568,7 @@ proc fromPreserves*[T](v: var T; pr: Value): bool {.gcsafe.} =
     type Foo {.preservesRecord: "foo".} = object
       x, y: int
     var foo: Foo
-    assert(fromPreserve(foo, parsePreserves("""<foo 1 2>""")))
+    assert(fromPreserves(foo, parsePreserves("""<foo 1 2>""")))
     assert(foo.x == 1)
     assert(foo.y == 2)
   when T is Value:
@@ -843,9 +846,9 @@ func step*(pr: Value; path: varargs[Value, toPreserves]): Option[Value] =
   ## Works for sequences, records, and dictionaries.
   runnableExamples:
     import std/options
-    assert step(parsePreserves("""<foo 1 2>"""), 1.toPreserve) == some(2.toPreserve)
-    assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserve)
-    assert step(parsePreserves("""[ ]"""), 1.toPreserve) == none(Value)
+    assert step(parsePreserves("""<foo 1 2>"""), 1.toPreserves) == some(2.toPreserves)
+    assert step(parsePreserves("""{ foo: 1 bar: 2}"""), "foo".toSymbol) == some(1.toPreserves)
+    assert step(parsePreserves("""[ ]"""), 1.toPreserves) == none(Value)
   result = some(pr)
   for index in path:
     if result.isSome:
@@ -878,7 +881,7 @@ proc apply*(result: var Value; op: proc(_: var Value) {.gcsafe.}) {.gcsafe.} =
       recurse(e.val)
   cannonicalize(result)
 
-proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value {.gcsafe.}): Value {.gcsafe.} =
+proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value): Value =
   ## Process all embeds in a `Value`.
   case pr.kind
   of pkBoolean, pkFloat, pkRegister, pkBigInt,

src/preserves/expressions.nim

@@ -0,0 +1,92 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import
+  npeg,
+  ../preserves, ./pegs
+
+type
+  Frame = tuple[value: Value, pos: int]
+  Stack = seq[Frame]
+
+proc shrink(stack: var Stack; n: int) = stack.setLen(stack.len - n)
+
+template pushStack(v: Value) = stack.add((v, capture[0].si))
+
+template collectEntries(result: var seq[Value]; stack: var Stack) =
+  for frame in stack.mitems:
+    if frame.pos > capture[0].si:
+      result.add frame.value.move
+  stack.shrink result.len
+
+proc parseExpressions*(text: string): seq[Value] =
+  let parser = peg("Document", stack: Stack):
+
+    ws <- *{ ' ', '\t', '\r', '\n' }
+
+    Document <- *Expr * ws * !1
+
+    Annotation <-
+      ('@' * SimpleExpr) |
+      ('#' * {'\x20', '\x09', '\x21'} * @{'\r','\n'})
+
+    Trailer <- *(ws * Annotation)
+
+    Expr <- ws * (Punct | SimpleExpr) * Trailer
+
+    Punct <- {',', ';'} | +':':
+      pushStack initRecord("p", toSymbol $0)
+
+    SimpleExpr <-
+        Atom |
+        Compound |
+        Embedded |
+        Annotated
+
+    Embedded <- "#:" * SimpleExpr:
+      pushstack stack.pop.value.embed
+
+    Annotated <- Annotation * SimpleExpr
+
+    Compound <- Sequence | Record | Block | Group | Set
+
+    Sequence <-  '[' * *Expr * ws * ']':
+      var pr = Value(kind: pkSequence)
+      collectEntries(pr.sequence, stack)
+      pushStack pr
+
+    Record <- '<' * *Expr * ws * '>':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"r"
+      pushStack pr
+
+    Block <- '{' * *Expr * ws * '}':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"b"
+      pushStack pr
+
+    Group <- '(' * *Expr * ws * ')':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"g"
+      pushStack pr
+
+    Set <- "#{" * *Expr * ws * '}':
+      var pr = Value(kind: pkRecord)
+      collectEntries(pr.record, stack)
+      pr.record.add toSymbol"s"
+      pushStack pr
+
+    Atom <- Preserves.Atom:
+      pushStack parsePreserves($0)
+
+  var stack: Stack
+  let match = parser.match(text, stack)
+  if not match.ok:
+    raise newException(ValueError, "failed to parse Preserves Expressions:\n" & text[match.matchMax..text.high])
+
+  result.setLen stack.len
+  for i, _ in result:
+    result[i] = move stack[i].value

src/preserves/pegs.nim

@@ -11,7 +11,11 @@ grammar "Preserves":
 
   ws <- *{ ' ', '\t', '\r', '\n' }
   commas <- *(ws * ',') * ws
-  delimiter <- { ' ', '\t', '\r', '\n', '<', '>', '[', ']', '{', '}', '#', ':', '"', '|', '@', ';', ',' } | !1
+  delimiter <- {
+      ' ', '\t', '\r', '\n',
+      '<', '>', '[', ']', '{', '}', '(', ')',
+      '#', ':', '"', '|', '@', ';', ','
+    } | !1
 
   Document <- Value * ws * !1
 
@@ -19,10 +23,10 @@ grammar "Preserves":
 
   Collection <- Sequence | Dictionary | Set
 
-  Value <-
-      (ws * (Record | Collection | Atom | Embedded | Compact)) |
-      (ws * Annotation) |
-      (ws * '#' * @'\n' * Value)
+  Value <- ws * (
+      Record | Collection | Atom | Embedded | Compact |
+      Annotation |
+      ('#' * @'\n' * Value) )
 
   Record <- '<' * +Value * ws * '>'
 

src/preserves/private/buffering.nim

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[assertions, endians, options, streams, strutils]
+import bigints
+import ./decoding, ./parsing, ./values
+
+type BufferedDecoder* = object
+  ## Type for buffering binary Preserves before decoding.
+  stream: StringStream
+  appendPosition, decodePosition, maxSize: int
+
+proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
+  ## Create a new `newBufferedDecoder`.
+  runnableExamples:
+    var
+      buf = newBufferedDecoder()
+      bin = encode(parsePreserves("<foobar>"))
+    buf.feed(bin[0..2])
+    buf.feed(bin[3..bin.high])
+    var (success, pr) = decode(buf)
+    assert success
+    assert $pr == "<foobar>"
+  BufferedDecoder(
+      stream: newStringStream(newStringOfCap(maxSize)),
+      maxSize: maxSize,
+    )
+
+proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
+  assert len > 0
+  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
+    raise newException(IOError, "BufferedDecoder at maximum buffer size")
+  dec.stream.setPosition(dec.appendPosition)
+  dec.stream.writeData(buf, len)
+  inc(dec.appendPosition, len)
+  assert dec.appendPosition == dec.stream.getPosition()
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
+  if data.len > 0:
+    dec.feed(addr data[0], data.len)
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]; slice: Slice[int]) =
+  let n = slice.b + 1 - slice.a
+  if n > 0:
+    dec.feed(addr data[slice.a], n)
+
+proc decode*(dec: var BufferedDecoder): Option[Value] =
+  ## Decode from `dec`. If decoding fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.decodePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError:
+      discard
+
+proc parse*(dec: var BufferedDecoder): Option[Value] =
+  ## Parse from `dec`. If parsing fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.readAll.parsePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError, ValueError:
+      discard

src/preserves/private/decoding.nim

@@ -15,11 +15,12 @@ proc readVarint(s: Stream): uint =
     c = uint s.readUint8
   result = result or (c shl shift)
 
-proc decodePreserves*(s: Stream): Value =
+proc decodePreserves*(s: Stream): Value {.gcsafe.}
+
+proc decodePreserves(s: Stream; tag: uint8): Value =
   ## Decode a Preserves value from a binary-encoded stream.
   if s.atEnd: raise newException(IOError, "End of Preserves stream")
   const endMarker = 0x84
-  let tag = s.readUint8()
   case tag
   of 0x80: result = Value(kind: pkBoolean, bool: false)
   of 0x81: result = Value(kind: pkBoolean, bool: true)
@@ -99,30 +100,38 @@ proc decodePreserves*(s: Stream): Value =
   of 0xb4:
     result = Value(kind: pkRecord)
     var label = decodePreserves(s)
-    while s.peekUint8() != endMarker:
-      result.record.add decodePreserves(s)
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result.record.add decodePreserves(s, tag)
+      tag = s.readUint8()
     result.record.add(move label)
-    discard s.readUint8()
   of 0xb5:
     result = Value(kind: pkSequence)
-    while s.peekUint8() != endMarker:
-      result.sequence.add decodePreserves(s)
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result.sequence.add decodePreserves(s, tag)
+      tag = s.readUint8()
   of 0xb6:
     result = Value(kind: pkSet)
-    while s.peekUint8() != endMarker:
-      incl(result, decodePreserves(s))
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      incl(result, decodePreserves(s, tag))
+      tag = s.readUint8()
   of 0xb7:
     result = Value(kind: pkDictionary)
-    while s.peekUint8() != endMarker:
-      result[decodePreserves(s)] = decodePreserves(s)
-    discard s.readUint8()
+    var tag = s.readUint8()
+    while tag != endMarker:
+      result[decodePreserves(s, tag)] = decodePreserves(s)
+      tag = s.readUint8()
   of endMarker:
     raise newException(ValueError, "invalid Preserves stream")
   else:
     raise newException(ValueError, "invalid Preserves tag byte 0x" & tag.toHex(2))
 
+proc decodePreserves*(s: Stream): Value {.gcsafe.} =
+  ## Decode a Preserves value from a binary-encoded stream.
+  s.decodePreserves s.readUint8()
+
 proc decodePreserves*(s: string): Value =
   ## Decode a string of binary-encoded Preserves.
   decodePreserves(s.newStringStream)
@@ -130,54 +139,3 @@ proc decodePreserves*(s: string): Value =
 proc decodePreserves*(s: seq[byte]): Value =
   ## Decode a byte-string of binary-encoded Preserves.
   decodePreserves(cast[string](s))
-
-type BufferedDecoder* = object
-  ## Type for buffering binary Preserves before decoding.
-  stream: StringStream
-  appendPosition, decodePosition, maxSize: int
-
-proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
-  ## Create a new `newBufferedDecoder`.
-  runnableExamples:
-    var
-      buf = newBufferedDecoder()
-      bin = encode(parsePreserves("<foobar>"))
-    buf.feed(bin[0..2])
-    buf.feed(bin[3..bin.high])
-    var (success, pr) = decode(buf)
-    assert success
-    assert $pr == "<foobar>"
-  BufferedDecoder(
-      stream: newStringStream(newStringOfCap(maxSize)),
-      maxSize: maxSize,
-    )
-
-proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
-  assert len > 0
-  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
-    raise newException(IOError, "BufferedDecoder at maximum buffer size")
-  dec.stream.setPosition(dec.appendPosition)
-  dec.stream.writeData(buf, len)
-  inc(dec.appendPosition, len)
-  assert dec.appendPosition == dec.stream.getPosition()
-
-proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
-  if data.len > 0:
-    dec.feed(unsafeAddr data[0], data.len)
-
-proc decode*(dec: var BufferedDecoder): Option[Value] =
-  ## Decode from `dec`. If decoding fails the internal position of the
-  ## decoder does not advance.
-  if dec.appendPosition > 0:
-    assert(dec.decodePosition < dec.appendPosition)
-    dec.stream.setPosition(dec.decodePosition)
-    try:
-      result = dec.stream.decodePreserves.some
-      dec.decodePosition = dec.stream.getPosition()
-      if dec.decodePosition == dec.appendPosition:
-        dec.stream.setPosition(0)
-        dec.stream.data.setLen(0)
-        dec.appendPosition = 0
-        dec.decodePosition = 0
-    except IOError:
-      discard

src/preserves/private/encoding.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[endians, streams]
+import std/[assertions, endians, streams]
 import bigints
 import ./values
 

src/preserves/private/macros.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import std/macros
+import std/[assertions, macros]
 
 const
   nnkPragmaCallKinds = {nnkExprColonExpr, nnkCall, nnkCallStrLit}

src/preserves/private/parsing.nim

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, options, parseutils, strutils, unicode]
+import std/[assertions, base64, options, parseutils, strutils, unicode]
 from std/sequtils import insert
 
 import bigints, npeg

src/preserves/private/texts.nim

@@ -1,7 +1,9 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[base64, endians, math, sequtils, streams, strutils]
+import std/[assertions, base64, endians, sequtils, streams, strutils]
+when not defined(nimNoLibc):
+  import std/math
 import bigints
 import ./values
 
@@ -42,6 +44,15 @@ proc writeSymbol(stream: Stream; sym: string) =
     writeEscaped(stream, sym, '|')
     write(stream, '|')
 
+proc writeFloatBytes(stream: Stream; f: float) =
+  var buf: array[8, byte]
+  bigEndian64(addr buf[0], addr f)
+  write(stream, "#xd\"")
+  for b in buf:
+    write(stream, hexAlphabet[b shr 4])
+    write(stream, hexAlphabet[b and 0xf])
+  write(stream, '"')
+
 proc writeText*(stream: Stream; pr: Value; mode = textPreserves) =
   ## Encode Preserves to a `Stream` as text.
   if pr.embedded: write(stream, "#:")
@@ -51,17 +62,14 @@ proc writeText*(stream: Stream; pr: Value; mode = textPreserves) =
     of false: write(stream, "#f")
     of true: write(stream, "#t")
   of pkFloat:
-    case pr.float.classify:
-    of fcNormal, fcZero, fcNegZero:
-      write(stream, $pr.float)
+    when defined(nimNoLibc):
+      writeFloatBytes(stream, pr.float)
+        # IEE754-to-decimal is non-trivial
     else:
-      var buf: array[8, byte]
-      bigEndian64(addr buf[0], addr pr.float)
-      write(stream, "#xd\"")
-      for b in buf:
-        write(stream, hexAlphabet[b shr 4])
-        write(stream, hexAlphabet[b and 0xf])
-      write(stream, '"')
+      if pr.float.classify in {fcNormal, fcZero, fcNegZero}:
+        write(stream, $pr.float)
+      else:
+        writeFloatBytes(stream, pr.float)
   of pkRegister:
     write(stream, $pr.register)
   of pkBigInt:

src/preserves/private/values.nim

@@ -1,8 +1,7 @@
 # SPDX-FileCopyrightText: ☭ Emery Hemingway
 # SPDX-License-Identifier: Unlicense
 
-import std/[algorithm, hashes, math, options, sets, sequtils, tables]
-
+import std/[algorithm, hashes, options, sets, sequtils, tables]
 import bigints
 
 type
@@ -80,11 +79,6 @@ type
     ## Object refs embedded in Preserves `Value`s must inherit from `EmbeddedObj`.
     ## At the moment this is just an alias to `RootObj` but this may change in the future.
 
-func `===`[T: SomeFloat](a, b: T): bool =
-  ## Compare where Nan == NaN.
-  let class = a.classify
-  (class == b.classify) and ((class notin {fcNormal,fcSubnormal}) or (a == b))
-
 func `==`*(x, y: Value): bool =
   ## Check `x` and `y` for equivalence.
   if x.kind == y.kind and x.embedded == y.embedded:
@@ -92,7 +86,7 @@ func `==`*(x, y: Value): bool =
     of pkBoolean:
       result = x.bool == y.bool
     of pkFloat:
-      result = x.float === y.float
+      result = cast[uint64](x.float) == cast[uint64](y.float)
     of pkRegister:
       result = x.register == y.register
     of pkBigInt:
@@ -255,10 +249,13 @@ proc `[]=`*(pr: var Value; key, val: Value) =
 
 proc incl*(pr: var Value; key: Value) =
   ## Include `key` in the Preserves set `pr`.
+  # TODO: binary search
   for i in 0..pr.set.high:
     if key < pr.set[i]:
       insert(pr.set, [key], i)
       return
+    elif key == pr.set[i]:
+      return
   pr.set.add(key)
 
 proc excl*(pr: var Value; key: Value) =

src/preserves/sugar.nim

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import ../preserves, ./private/macros
+
+proc `%`*(n: SomeInteger): Value {.inline.} = n.toPreserves

tests/test_p_exprs.nim

@@ -0,0 +1,83 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import
+  std/unittest,
+  preserves, preserves/expressions
+
+template testExpr(name, code, cntrl: string) {.dirty.} =
+  test name:
+    checkpoint code
+    let
+      pr = parsePreserves cntrl
+      exprs = parseExpressions code
+    checkpoint $(exprs.toPreserves)
+    check exprs.len == 1
+    let px = exprs[0]
+    check px == pr
+
+suite "expression":
+
+  testExpr "date", """
+      <date 1821 (lookup-month "February") 3>
+    """, """
+      <r date 1821 <g lookup-month "February"> 3>
+    """
+
+  testExpr "r", "<>", "<r>"
+
+  testExpr "begin",
+    """(begin (println! (+ 1 2)) (+ 3 4))""",
+    """<g begin <g println! <g + 1 2>> <g + 3 4>>"""
+
+  testExpr "g",
+    """()""", """<g>"""
+
+  testExpr "groups",
+    """[() () ()]""", """[<g>, <g>, <g>]"""
+
+  testExpr "loop", """
+      {
+        setUp();
+        # Now enter the loop
+        loop: {
+            greet("World");
+        }
+        tearDown();
+      }
+    """,  """
+      <b
+      setUp <g> <p |;|>
+      # Now enter the loop
+      loop <p |:|> <b
+          greet <g "World"> <p |;|>
+      >
+      tearDown <g> <p |;|>
+    >
+  """
+
+  testExpr "+", """
+      [1 + 2.0, print "Hello", predicate: #t, foo, #:remote, bar]
+    """, """
+      [1 + 2.0 <p |,|> print "Hello" <p |,|> predicate <p |:|> #t <p |,|>
+   foo <p |,|> #:remote <p |,|> bar]
+   """
+
+  testExpr "set",
+    """#{1 2 3}""",  """<s 1 2 3>"""
+
+  testExpr "group-set",
+    """#{(read) (read) (read)}""",
+    """<s <g read> <g read> <g read>>"""
+
+  testExpr "block", """
+      {
+        optional name: string,
+        address: Address,
+      }
+    """, """
+      <b
+        optional name <p |:|> string <p |,|>
+        address <p |:|> Address <p |,|>
+      >
+    """