Add text parsing to BufferedDecoder

preserves.nimble

@@ -1,6 +1,6 @@
 # Package
 
-version = "20240208"
+version = "20240312"
 author        = "Emery Hemingway"
 description   = "data model and serialization format"
 license       = "Unlicense"

src/preserves.nim

@@ -5,9 +5,9 @@ import std/[options, sets, sequtils, strutils, tables, typetraits]
 from std/algorithm import sort
 from std/json import escapeJson, escapeJsonUnquoted
 import bigints
-import ./preserves/private/[encoding, decoding, dot, macros, parsing, texts, values]
+import ./preserves/private/[buffering, encoding, decoding, dot, macros, parsing, texts, values]
 
-export encoding, decoding, parsing, texts, values
+export buffering, encoding, decoding, parsing, texts, values
 
 when defined(tracePreserves):
   when defined(posix):
@@ -551,7 +551,7 @@ proc fromAtom*[T](v: var T; a: ATom): bool =
   elif T is distinct:
     result = fromAtom(v.distinctBase, a)
 
-proc fromPreserves*[T](v: var T; pr: Value): bool {.gcsafe.} =
+proc fromPreserves*[T](v: var T; pr: Value): bool =
   ## Inplace version of `preservesTo`. Returns ``true`` on
   ## a complete match, otherwise returns ``false``.
   ## Can be customized with `fromPreservesHook(x: T; var pr: Value): bool`.
@@ -878,7 +878,7 @@ proc apply*(result: var Value; op: proc(_: var Value) {.gcsafe.}) {.gcsafe.} =
       recurse(e.val)
   cannonicalize(result)
 
-proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value {.gcsafe.}): Value {.gcsafe.} =
+proc mapEmbeds*(pr: sink Value; op: proc (x: Value): Value): Value =
   ## Process all embeds in a `Value`.
   case pr.kind
   of pkBoolean, pkFloat, pkRegister, pkBigInt,

src/preserves/private/buffering.nim

@@ -0,0 +1,79 @@
+# SPDX-FileCopyrightText: ☭ Emery Hemingway
+# SPDX-License-Identifier: Unlicense
+
+import std/[endians, options, streams, strutils]
+import bigints
+import ./decoding, ./parsing, ./values
+
+type BufferedDecoder* = object
+  ## Type for buffering binary Preserves before decoding.
+  stream: StringStream
+  appendPosition, decodePosition, maxSize: int
+
+proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
+  ## Create a new `newBufferedDecoder`.
+  runnableExamples:
+    var
+      buf = newBufferedDecoder()
+      bin = encode(parsePreserves("<foobar>"))
+    buf.feed(bin[0..2])
+    buf.feed(bin[3..bin.high])
+    var (success, pr) = decode(buf)
+    assert success
+    assert $pr == "<foobar>"
+  BufferedDecoder(
+      stream: newStringStream(newStringOfCap(maxSize)),
+      maxSize: maxSize,
+    )
+
+proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
+  assert len > 0
+  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
+    raise newException(IOError, "BufferedDecoder at maximum buffer size")
+  dec.stream.setPosition(dec.appendPosition)
+  dec.stream.writeData(buf, len)
+  inc(dec.appendPosition, len)
+  assert dec.appendPosition == dec.stream.getPosition()
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
+  if data.len > 0:
+    dec.feed(addr data[0], data.len)
+
+proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]; slice: Slice[int]) =
+  let n = slice.b + 1 - slice.a
+  if n > 0:
+    dec.feed(addr data[slice.a], n)
+
+proc decode*(dec: var BufferedDecoder): Option[Value] =
+  ## Decode from `dec`. If decoding fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.decodePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError:
+      discard
+
+proc parse*(dec: var BufferedDecoder): Option[Value] =
+  ## Parse from `dec`. If parsing fails the internal position of the
+  ## decoder does not advance.
+  if dec.appendPosition > 0:
+    assert(dec.decodePosition < dec.appendPosition)
+    dec.stream.setPosition(dec.decodePosition)
+    try:
+      result = dec.stream.readAll.parsePreserves.some
+      dec.decodePosition = dec.stream.getPosition()
+      if dec.decodePosition == dec.appendPosition:
+        dec.stream.setPosition(0)
+        dec.stream.data.setLen(0)
+        dec.appendPosition = 0
+        dec.decodePosition = 0
+    except IOError, ValueError:
+      discard

src/preserves/private/decoding.nim

@@ -130,54 +130,3 @@ proc decodePreserves*(s: string): Value =
 proc decodePreserves*(s: seq[byte]): Value =
   ## Decode a byte-string of binary-encoded Preserves.
   decodePreserves(cast[string](s))
-
-type BufferedDecoder* = object
-  ## Type for buffering binary Preserves before decoding.
-  stream: StringStream
-  appendPosition, decodePosition, maxSize: int
-
-proc newBufferedDecoder*(maxSize = 4096): BufferedDecoder =
-  ## Create a new `newBufferedDecoder`.
-  runnableExamples:
-    var
-      buf = newBufferedDecoder()
-      bin = encode(parsePreserves("<foobar>"))
-    buf.feed(bin[0..2])
-    buf.feed(bin[3..bin.high])
-    var (success, pr) = decode(buf)
-    assert success
-    assert $pr == "<foobar>"
-  BufferedDecoder(
-      stream: newStringStream(newStringOfCap(maxSize)),
-      maxSize: maxSize,
-    )
-
-proc feed*(dec: var BufferedDecoder; buf: pointer; len: int) =
-  assert len > 0
-  if dec.maxSize > 0 and dec.maxSize < (dec.appendPosition + len):
-    raise newException(IOError, "BufferedDecoder at maximum buffer size")
-  dec.stream.setPosition(dec.appendPosition)
-  dec.stream.writeData(buf, len)
-  inc(dec.appendPosition, len)
-  assert dec.appendPosition == dec.stream.getPosition()
-
-proc feed*[T: byte|char](dec: var BufferedDecoder; data: openarray[T]) =
-  if data.len > 0:
-    dec.feed(unsafeAddr data[0], data.len)
-
-proc decode*(dec: var BufferedDecoder): Option[Value] =
-  ## Decode from `dec`. If decoding fails the internal position of the
-  ## decoder does not advance.
-  if dec.appendPosition > 0:
-    assert(dec.decodePosition < dec.appendPosition)
-    dec.stream.setPosition(dec.decodePosition)
-    try:
-      result = dec.stream.decodePreserves.some
-      dec.decodePosition = dec.stream.getPosition()
-      if dec.decodePosition == dec.appendPosition:
-        dec.stream.setPosition(0)
-        dec.stream.data.setLen(0)
-        dec.appendPosition = 0
-        dec.decodePosition = 0
-    except IOError:
-      discard